diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml new file mode 100644 index 00000000000..36c6c790b84 --- /dev/null +++ b/.github/actions/cache-cargo-build/action.yml @@ -0,0 +1,31 @@ +name: "Cache the Rust build" +description: >- + Cache the Cargo registry and target directory the root package's build needs, + so only the first job on a given Cargo.lock compiles the bridge from scratch. + + litellm builds through maturin, which compiles litellm-rust/crates/python-bridge + in release mode before it can produce a wheel. `uv sync` therefore pays a full + build in every job that installs the workspace: measured at 2m40s per unit shard + on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught + it, because the uv cache holds wheels uv downloads rather than wheels it builds, + and a path dependency whose source moves every commit could never hit that cache + anyway. Cargo rebuilds only what changed when its target directory survives, so a + warm job pays for the bridge crate alone. + + The key namespace is separate from test-rust.yml's. Both cache the same directory, + but that workflow fills it with debug and clippy artifacts, which a release build + cannot reuse, and a shared key would let whichever ran first deny the other a save. + +runs: + using: composite + steps: + - name: Restore the Cargo registry and target directory + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + litellm-rust/target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-release- diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index ff8fa864d4a..918589f84d1 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -48,16 +48,6 @@ test_paths: choice it informed is settled paths: - tests/code_coverage_tests/test_aio_http_image_conversion.py - - reason: >- - The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its - other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging - their bodies into the live file of the same name. This one cannot follow either route yet: - its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no - counterpart while 25 assertions fail against today's code, so what survives that rewrite - is a judgement about the endpoints, not a merge. Revisit by deciding which of the five - behaviours still hold - paths: - - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py - reason: >- No job invokes this suite and its files mix pure transformation tests with ones driving live vendor vector stores, so assigning them needs a per-file decision diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 54f50524a39..b7d185bd0b9 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -27,7 +27,7 @@ on: default: 20 job-timeout-minutes: description: >- - Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for + Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for the per-step ceilings on the setup steps below, and 5 for the runner overhead the job clock charges but no step owns (job init, step transitions, post-job cleanup). That headroom is what makes the test @@ -36,7 +36,7 @@ on: arithmetic, so the sum is passed in rather than computed. required: false type: number - default: 55 + default: 60 max-failures: description: "Stop after this many failures" required: false @@ -103,6 +103,11 @@ jobs: restore-keys: | ${{ runner.os }}-uv- + - name: Cache the Rust build + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 5 + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies if: steps.changes.outputs.decision != 'skip' timeout-minutes: 8 diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml index dbd663a2efa..285676a0ddd 100644 --- a/.github/workflows/check-ui-api-types.yml +++ b/.github/workflows/check-ui-api-types.yml @@ -67,6 +67,10 @@ jobs: restore-keys: | ${{ runner.os }}-uv- + - name: Cache the Rust build + if: steps.changes.outputs.relevant == 'true' + uses: ./.github/actions/cache-cargo-build + - name: Install backend dependencies if: steps.changes.outputs.relevant == 'true' run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router diff --git a/.github/workflows/mutation-test.yml b/.github/workflows/mutation-test.yml index 68317d5dd12..602c26a3e98 100644 --- a/.github/workflows/mutation-test.yml +++ b/.github/workflows/mutation-test.yml @@ -53,6 +53,9 @@ jobs: restore-keys: | ${{ runner.os }}-uv- + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml diff --git a/.github/workflows/publish-basedpyright-base-counts.yml b/.github/workflows/publish-basedpyright-base-counts.yml index 71e196d8361..cd443a8e9db 100644 --- a/.github/workflows/publish-basedpyright-base-counts.yml +++ b/.github/workflows/publish-basedpyright-base-counts.yml @@ -43,6 +43,9 @@ jobs: with: version: "0.10.9" + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + - name: Cache Prisma binaries uses: ./.github/actions/cache-prisma-binaries diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index d0ac0b6fdee..c112bf2bb22 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -56,6 +56,9 @@ jobs: restore-keys: | ${{ runner.os }}-uv- + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies run: uv sync --frozen --all-groups --all-extras diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index e031ba46773..ccb58f5cc9c 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -78,6 +78,10 @@ jobs: run: | uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) + - name: Cache the Rust build + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies if: steps.changes.outputs.decision != 'skip' run: | diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 95187ef2835..6ea814dc2de 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -47,6 +47,10 @@ jobs: with: version: "0.10.9" + - name: Cache the Rust build + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies if: steps.changes.outputs.decision != 'skip' run: | diff --git a/.github/workflows/test-terraform-provider.yml b/.github/workflows/test-terraform-provider.yml index 7ea22825f4f..e46432e0e31 100644 --- a/.github/workflows/test-terraform-provider.yml +++ b/.github/workflows/test-terraform-provider.yml @@ -88,6 +88,9 @@ jobs: restore-keys: | ${{ runner.os }}-uv- + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index cb8035aafa1..90b6b28374e 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -67,6 +67,10 @@ jobs: restore-keys: | ${{ runner.os }}-uv- + - name: Cache the Rust build + if: steps.changes.outputs.decision != 'skip' + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies if: steps.changes.outputs.decision != 'skip' run: | diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 3d6fffe7304..71eb0958bec 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -55,7 +55,7 @@ jobs: workers: 2 reruns: 1 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: enterprise-routing artifact-name: enterprise-routing @@ -67,7 +67,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: integrations artifact-name: integrations @@ -75,7 +75,7 @@ jobs: workers: 2 reruns: 3 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: Vertex AI artifact-name: llm-vertex-ai @@ -83,7 +83,7 @@ jobs: workers: 1 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: All Other Providers artifact-name: llm-other-providers @@ -91,7 +91,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: misc artifact-name: misc @@ -122,7 +122,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: proxy-auth artifact-name: proxy-auth @@ -134,7 +134,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: proxy-endpoints artifact-name: proxy-endpoints @@ -171,7 +171,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: proxy-server artifact-name: proxy-server @@ -179,7 +179,7 @@ jobs: workers: 4 reruns: 2 timeout-minutes: 60 - job-timeout-minutes: 95 + job-timeout-minutes: 100 - shard: proxy-infra artifact-name: proxy-infra @@ -198,7 +198,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 - shard: responses-caching-types artifact-name: responses-caching-types @@ -209,7 +209,7 @@ jobs: workers: 2 reruns: 2 timeout-minutes: 20 - job-timeout-minutes: 55 + job-timeout-minutes: 60 uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} diff --git a/.github/workflows/weekly_load_anomaly.yml b/.github/workflows/weekly_load_anomaly.yml index 2dffc889d0e..3e1fca89645 100644 --- a/.github/workflows/weekly_load_anomaly.yml +++ b/.github/workflows/weekly_load_anomaly.yml @@ -47,6 +47,9 @@ jobs: with: version: "0.10.9" + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + - name: Install dependencies run: | .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 8bbde7f3764..ccfe7eda5e2 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.58" +version = "0.1.59" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.58" +version = "0.1.59" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 26d42a33b29..98a3d8d535e 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.88" +version = "0.4.89" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.88" +version = "0.4.89" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/_logging.py b/litellm/_logging.py index e55c6bc40a8..36fd51206c2 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -88,6 +88,24 @@ def redact_secrets(value: str) -> str: return _redact_string(value) +def _substituted_color_message(record: logging.LogRecord) -> str | None: + """Render a record's ``color_message`` against its args, or None if absent. + + uvicorn's colorized formatter re-renders `color_message` against + record.args at emit time (see uvicorn.logging.ColourizedFormatter) instead + of using the already-formatted record.msg, so it has to be substituted + before args are cleared or it is later formatted with no args and prints + the raw "%s://%s:%d" placeholders instead of the URL. + """ + color_message: Final = record.__dict__.get("color_message") + if not isinstance(color_message, str) or not record.args: + return None + try: + return color_message % record.args + except TypeError: + return color_message + + class SecretRedactionFilter(logging.Filter): """Scrubs known secret/credential patterns from log records.""" @@ -97,6 +115,12 @@ class SecretRedactionFilter(logging.Filter): if not _ENABLE_SECRET_REDACTION: return True + # Runs before args are cleared, and before the extra-field loop below + # that redacts the substituted result. + substituted_color_message: Final = _substituted_color_message(record) + if substituted_color_message is not None: + record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place + try: record.msg = _redact_string(record.getMessage()) record.args = None diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 0cf22d82ca6..6eb13d2cba7 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -296,6 +296,32 @@ def calculate_vertex_ai_batch_cost_and_usage( ) +def _provider_output_file_id(output_file_id: str) -> str: + """ + Resolve the file id the provider actually knows: unified ids yield their embedded + llm_output_file_id, model-encoded ids decode to the raw provider id, raw ids pass through. + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + get_original_file_id, + ) + + unified_file_id: Final = _is_base64_encoded_unified_file_id(output_file_id) + if not unified_file_id: + return get_original_file_id(output_file_id) + try: + extracted: Final = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + except (IndexError, AttributeError) as e: + verbose_logger.error( + "Failed to extract LLM output file ID from unified file ID: %s, error: %s", + output_file_id, + e, + ) + return output_file_id + verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", extracted) + return extracted + + async def _fetch_batch_output_file_content( batch: Batch, custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai", @@ -311,23 +337,11 @@ async def _fetch_batch_output_file_content( Required for Azure and other providers that need authentication """ from litellm.files.main import afile_content - from litellm.proxy.openai_files_endpoints.common_utils import ( - _is_base64_encoded_unified_file_id, - ) if batch.output_file_id is None: raise ValueError("Output file id is None cannot retrieve file content") - file_id = batch.output_file_id - is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(file_id) - if is_base64_unified_file_id: - try: - file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id) - except (IndexError, AttributeError) as e: - verbose_logger.error( - "Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e - ) + file_id: Final = _provider_output_file_id(batch.output_file_id) # Build kwargs for afile_content with credentials from litellm_params file_content_kwargs: Final = { diff --git a/litellm/constants.py b/litellm/constants.py index 11904d5e9c2..c33e5a53b76 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -783,6 +783,7 @@ openai_compatible_endpoints: Final[list] = [ "https://pinstripes.io/v1", "https://api.meta.ai/v1", "https://api.cognition.ai/v1", + "https://api.scx.ai/v1", ] @@ -851,6 +852,7 @@ openai_compatible_providers: Final[list] = [ "darkbloom", "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", + "scx-ai", ] openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions` "together_ai", diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index ab07a6af1b3..2618aee9afa 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Final from litellm.types.utils import ProviderSpecificHeader @@ -6,13 +7,17 @@ from litellm.types.utils import ProviderSpecificHeader class ProviderSpecificHeaderUtils: @staticmethod def get_provider_specific_headers( - provider_specific_header: ProviderSpecificHeader | None, + provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, custom_llm_provider: str | None, ) -> dict: """ Get the provider specific headers for the given custom llm provider. - Supports comma-separated provider lists for headers that work across multiple providers. + Accepts either a single ProviderSpecificHeader or a sequence of them. Each entry + carries its own comma-separated provider list, so headers that are safe for several + providers and headers that are safe for exactly one can travel on the same request + without sharing a scope. Entries whose provider list does not contain + `custom_llm_provider` contribute nothing. Returns: Dict: The provider specific headers for the given custom llm provider @@ -20,10 +25,15 @@ class ProviderSpecificHeaderUtils: if provider_specific_header is None or custom_llm_provider is None: return {} - stored_providers: Final = provider_specific_header.get("custom_llm_provider", "") - provider_list: Final = [p.strip() for p in stored_providers.split(",")] + scoped_headers: Final = ( + (provider_specific_header,) if isinstance(provider_specific_header, dict) else provider_specific_header + ) - if custom_llm_provider in provider_list: - return provider_specific_header.get("extra_headers", {}) + matched_headers: Final = {} + for scoped_header in scoped_headers: + stored_providers = scoped_header.get("custom_llm_provider", "") + provider_list = [p.strip() for p in stored_providers.split(",")] + if custom_llm_provider in provider_list: + matched_headers.update(scoped_header.get("extra_headers", {})) - return {} + return matched_headers diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9b7707eabe1..c14dd6c3d8b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5615,6 +5615,37 @@ def _extract_response_obj_and_hidden_params( return response_obj, hidden_params +def _autorouter_savings_for_payload( + request_metadata: Mapping[str, object], + model: str | None, + custom_llm_provider: str | None, + model_id: str | None, + usage_object: Mapping[str, object] | None, + cost_breakdown: Mapping[str, object] | None, +) -> float | None: + """The auto-router savings figure for the payload, or ``None`` when there is none. + + Lazy proxy import: the savings module lives with the spend trackers that own the + math, and SDK-only installs have no proxy package to import. + """ + try: + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload + except Exception: # noqa: BLE001 # SDK-only install: no savings driver to run + return None + try: + return autorouter_savings_for_logging_payload( + request_metadata=request_metadata, + model=model, + custom_llm_provider=custom_llm_provider, + model_id=model_id, + usage_object=usage_object, + cost_breakdown=cost_breakdown, + ) + except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging + verbose_logger.debug("autorouter savings skipped on logging payload: %s", e) + return None + + def get_standard_logging_object_payload( kwargs: dict | None, init_response_obj: Any | BaseModel | dict, @@ -5772,6 +5803,16 @@ def get_standard_logging_object_payload( ): model_name = response_model_name + request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost) + autorouter_savings: Final = _autorouter_savings_for_payload( + request_metadata=metadata, + model=model_name, + custom_llm_provider=custom_llm_provider, + model_id=_model_id, + usage_object=usage_dict, + cost_breakdown=request_cost_breakdown, + ) + payload: Final[StandardLoggingPayload] = StandardLoggingPayload( id=str(id), litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), @@ -5802,7 +5843,8 @@ def get_standard_logging_object_payload( metadata=clean_metadata, cache_key=clean_hidden_params["cache_key"], response_cost=response_cost, - cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost), + cost_breakdown=request_cost_breakdown, + autorouter_savings=autorouter_savings, total_tokens=usage_dict.get("total_tokens", 0), prompt_tokens=usage_dict.get("prompt_tokens", 0), completion_tokens=usage_dict.get("completion_tokens", 0), @@ -5998,6 +6040,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: call_type="completion", stream=False, response_cost=response_cost, + autorouter_savings=None, response_cost_failure_debug_info=None, status="success", total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT), diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index 2e73719cf52..021210d9175 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -124,6 +124,14 @@ def ptu_identity_error( return None +PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to") + + +def declares_ptu(model_info: Mapping[str, object]) -> bool: + """Whether any PTU field is set here, including one too malformed to charge.""" + return any(model_info.get(field) is not None for field in PTU_MODEL_INFO_FIELDS) + + def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None: """Why this PTU configuration cannot be honoured, else None. diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 89066e33cbc..9d61701d26d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import ) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, + local_model_name, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -358,9 +359,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: except Exception: pass - if isinstance(model, str) and model and not model.startswith("responses/"): - # Prefix model with "responses/" to route to OpenAI Responses API - completion_kwargs["model"] = f"responses/{model}" + if isinstance(model, str) and model and "responses/" not in model: + local_model: Final = model.removeprefix(f"{custom_llm_provider}/") + completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}" auto_summary: Final = is_reasoning_auto_summary_enabled() @@ -616,7 +617,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( completion_response, - model=model, + model=local_model_name(model, kwargs.get("custom_llm_provider")), tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=True, @@ -750,7 +751,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( completion_response, - model=model, + model=local_model_name(model, kwargs.get("custom_llm_provider")), tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=False, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 26aef666172..f4d24bb933c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -42,15 +42,46 @@ from .utils import AnthropicMessagesRequestUtils, mock_response _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) -def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool: - """Return True when the provider should use the Responses API path. +def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool: + from litellm.main import responses_api_bridge_check + + model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider) + return model_info.get("mode") == "responses" + + +def _responses_mode_is_lost_by_prefix_strip( + requested_model: str, resolved_model: str, custom_llm_provider: str +) -> bool: + """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped. + + ``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a + deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the + chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions. + """ + if requested_model == resolved_model: + return False + return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api( + resolved_model, custom_llm_provider + ) + + +def _should_route_to_responses_api( + custom_llm_provider: str | None, + requested_model: str | None = None, + resolved_model: str | None = None, +) -> bool: + """Return True when the request should use the Responses API path. Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to opt out and route OpenAI/Azure requests through chat/completions instead. """ if litellm.use_chat_completions_url_for_anthropic_messages: return False - return custom_llm_provider in _RESPONSES_API_PROVIDERS + if custom_llm_provider in _RESPONSES_API_PROVIDERS: + return True + if custom_llm_provider is None or requested_model is None or resolved_model is None: + return False + return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider) def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: @@ -533,7 +564,7 @@ def anthropic_messages_handler( _shared_kwargs: Final = dict( max_tokens=max_tokens, messages=messages, - model=model, + model=original_model, metadata=metadata, stop_sequences=stop_sequences, stream=stream, @@ -551,7 +582,7 @@ def anthropic_messages_handler( custom_llm_provider=custom_llm_provider, **kwargs, ) - if _should_route_to_responses_api(custom_llm_provider): + if _should_route_to_responses_api(custom_llm_provider, original_model, model): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs) # The in-gateway context_management polyfill runs inside diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index d0dc5d527fe..02d82887dde 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -44,6 +44,7 @@ class AnthropicMessagesRequestUtils: filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None} if model is not None: from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.llms.anthropic.common_utils import AnthropicModelInfo AnthropicConfig._maybe_drop_speed_param( model=model, @@ -51,6 +52,16 @@ class AnthropicMessagesRequestUtils: drop_params=drop_params, custom_llm_provider=custom_llm_provider, ) + for param in ("temperature", "top_p", "top_k"): + if param in filtered_params: + AnthropicModelInfo._apply_sampling_param( # pyright: ignore[reportPrivateUsage] # same gating the /chat/completions path applies; forking it would drift + optional_params=filtered_params, + model=model, + param=param, + value=filtered_params.pop(param), + drop_params=drop_params, + output_key=param, + ) return cast(AnthropicMessagesRequestOptionalParams, filtered_params) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 843cda249c5..c1ea39fd72c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import ResponsesAPIResponse +from ..utils import local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -179,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler: result: Final = await litellm.aresponses(**responses_kwargs) if stream: - wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + wrapper: Final = AnthropicResponsesStreamWrapper( + responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")) + ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): @@ -257,7 +260,9 @@ class LiteLLMMessagesToResponsesAPIHandler: result: Final = litellm.responses(**responses_kwargs) if stream: - wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + wrapper: Final = AnthropicResponsesStreamWrapper( + responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")) + ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index c5abcf8c04c..29661572b73 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -13,6 +13,11 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None: return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None +def local_model_name(model: str, custom_llm_provider: object) -> str: + """The id the provider itself knows, for reporting back to the caller in ``message_start``.""" + return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model + + def is_reasoning_auto_summary_enabled() -> bool: """Check whether the default 'summary: detailed' injection is enabled (opt-in).""" return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8c98c526da1..369e150f6bd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2056,7 +2056,7 @@ class BaseLLMHTTPHandler: # Prepare headers kwargs = kwargs or {} provider_specific_header: Final = cast( - litellm.types.utils.ProviderSpecificHeader | None, + litellm.types.utils.ProviderSpecificHeader | Sequence[litellm.types.utils.ProviderSpecificHeader] | None, kwargs.get("provider_specific_header", None), ) provider_specific_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 5f57aaa78d8..a458a209ea9 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -188,5 +188,17 @@ "max_completion_tokens": "max_tokens" }, "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"] + }, + "scx-ai": { + "base_url": "https://api.scx.ai/v1", + "api_key_env": "SCX_API_KEY", + "api_base_env": "SCX_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "constraints": { + "temperature_max": 1.99 + }, + "supported_endpoints": ["/v1/chat/completions"] } } diff --git a/litellm/main.py b/litellm/main.py index 7cfd322f3d0..52785e7a393 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5091,14 +5091,16 @@ def completion( model_info: Final = kwargs.get("model_info", None) proxy_server_request: Final = kwargs.get("proxy_server_request", None) fallbacks = kwargs.get("fallbacks", None) - provider_specific_header: Final = cast(ProviderSpecificHeader | None, kwargs.get("provider_specific_header", None)) + provider_specific_header: Final = cast( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, + kwargs.get("provider_specific_header", None), + ) headers = kwargs.get("headers", None) or extra_headers ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None) user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None) assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None) - if headers is None: - headers = {} + headers = {} if headers is None else dict(headers) if extra_headers is not None: headers.update(extra_headers) # Inject proxy auth headers if configured diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a1961136d11..3af7d9e5019 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4881,6 +4881,38 @@ "supports_tool_choice": true, "supports_vision": false }, + "azure/gpt-audio-mini": { + "deprecation_date": "2027-04-06", + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/gpt-audio-mini-2025-10-06": { "deprecation_date": "2027-04-06", "input_cost_per_audio_token": 1e-05, @@ -5094,6 +5126,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/gpt-realtime-mini": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, @@ -19498,106 +19562,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "gemini/gemini-3.1-flash-lite-image": { - "rpm": 1000, - "tpm": 4000000, - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "vertex_ai/gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, "gemini-3.1-flash-image": { "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -19675,6 +19639,44 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -21505,6 +21507,42 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.1-flash-lite-image": { + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_vision": true, + "tpm": 4000000 + }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -26041,33 +26079,33 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06, - "cache_creation_input_token_cost_flex": 3.125e-06, - "cache_creation_input_token_cost_priority": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, - "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_above_272k_tokens_flex": 5e-06, - "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_flex": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_flex": 2.5e-06, + "cache_creation_input_token_cost_priority": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_flex": 2e-07, + "cache_read_input_token_cost_priority": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_flex": 2e-06, + "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, - "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_flex": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_flex": 1e-05, + "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -26104,33 +26142,33 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06, - "cache_creation_input_token_cost_flex": 3.125e-06, - "cache_creation_input_token_cost_priority": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, - "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_above_272k_tokens_flex": 5e-06, - "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_flex": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_flex": 2.5e-06, + "cache_creation_input_token_cost_priority": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_flex": 2e-07, + "cache_read_input_token_cost_priority": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_flex": 2e-06, + "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, - "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_flex": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_flex": 1e-05, + "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -26372,19 +26410,19 @@ "supports_parallel_function_calling": true }, "daybreak-blue-latest": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -31147,6 +31185,23 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "moonshot", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://platform.kimi.ai/docs/pricing/chat-k3", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, "moonshot/kimi-latest": { "cache_read_input_token_cost": 1.5e-07, "deprecation_date": "2026-01-28", @@ -36731,6 +36786,40 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, + "scx-ai/GLM-5.2": { + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 6.1e-07, + "litellm_provider": "scx-ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-06, + "source": "https://scx.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "scx-ai/Qwen3.8-Max": { + "cache_read_input_token_cost": 2.1e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "scx-ai", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.99e-06, + "source": "https://scx.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, @@ -41015,6 +41104,44 @@ "supports_reasoning": false, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" }, + "vertex_ai/gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -48554,6 +48681,156 @@ "supports_tool_choice": true, "supports_vision": true }, + "us.openai.gpt-5.6-sol": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "global.openai.gpt-5.6-sol": { + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "output_cost_per_token": 3e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "us.openai.gpt-5.6-terra": { + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, + "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, + "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "global.openai.gpt-5.6-terra": { + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_272k_tokens": 1.8e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "us.openai.gpt-5.6-luna": { + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, + "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, + "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "global.openai.gpt-5.6-luna": { + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-07, + "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_above_272k_tokens": 1.8e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index b4d635c0fba..86c14fb4cd8 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2027,6 +2027,23 @@ "interactions": true } }, + "scx-ai": { + "display_name": "SCX.ai (`scx-ai`)", + "url": "https://docs.litellm.ai/docs/providers/scx_ai", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "snowflake": { "display_name": "Snowflake (`snowflake`)", "url": "https://docs.litellm.ai/docs/providers/snowflake", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index c1248cafac5..7d85f3c4908 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -39,6 +39,7 @@ from litellm.proxy._types import ( SpecialMCPServerName, SpecialMCPServerNames, UserAPIKeyAuth, + user_api_key_has_admin_view, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import ( @@ -1785,11 +1786,14 @@ class MCPRequestHandler: global_mcp_server_manager, ) - # An OPEN channel (allow_all_keys, the user's own BYOM) makes the server REACHABLE through the - # user, though no grant source names it — without this the union returns [], listable but - # uninvokable. Reachability is ALL it confers, NOT a ceiling waiver: the user's own - # mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do on an allow_all server. - reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(auth) + # An OPEN channel (allow_all_keys, the user's own BYOM, an unscoped admin-view role) makes the + # server REACHABLE through the user, though no grant source names it — without this the union + # returns [], listable but uninvokable. Reachability is ALL it confers, NOT a ceiling waiver: + # the user's own mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do + # on an allow_all server or an admin key's do on any server. + reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids( + auth + ) or await MCPRequestHandler.admin_view_unscoped(auth) allowed: Final[set[str]] = set() for source, granted in await MCPRequestHandler.admitted_source_grants(auth): @@ -2723,6 +2727,32 @@ class MCPRequestHandler: entitled_servers: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_user(user_api_key_auth) return entitled_servers is None or len(entitled_servers) > 0 + @staticmethod + async def admin_view_unscoped(user_api_key_auth: UserAPIKeyAuth | None = None) -> bool: + """Whether this principal's admin-view role grants the unscoped MCP resolution, whatever + credential carries it (admin key, dashboard session, or OAuth-admitted session subject). + + Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit + ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An + admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is + [] by DB default, so for that shape the row binds through the entitlement ceiling instead + (any non-empty entitlement, or an unresolved one, disqualifies), exactly as + ``operator_open_server_ids`` reads the same row. The one owner of this predicate: the + server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open + channel in ``_resolve_admitted_subject_tools`` both consult it, so the two axes cannot + disagree.""" + if user_api_key_auth is None or not user_api_key_has_admin_view(user_api_key_auth): + return False + object_permission: Final = user_api_key_auth.object_permission + credential_scoped: Final = ( + not _is_mcp_admitted_user_subject(user_api_key_auth) + and object_permission is not None + and object_permission.mcp_servers is not None + ) + if credential_scoped: + return False + return not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth) + @staticmethod async def _apply_user_tool_ceiling( allowed_tools: Sequence[str] | None, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index dbe97dd5bce..7ab26db0f3e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2943,17 +2943,14 @@ class MCPServerManager: 2. If admin and no object_permission, return all servers 3. Otherwise, use standard permission checks """ - from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view - allow_all_server_ids: Final = self.get_allow_all_keys_server_ids() # A keyless admitted subject is resolved per grant source, and channel decisions that are # absolute for a scoped KEY credential are not absolute for it: its own opt-out silences its - # own source (handled per source in the resolver), never its teams' grants, and its admin - # role does not swallow the grant model — a session bearer is a third-party client - # credential, not the dashboard, so an admin signing in through the connect flow gets their - # grants like anyone else rather than handing the client the full registry ahead of every - # per-team org ceiling. + # own source (handled per source in the resolver), never its teams' grants. Its admin role + # rides the HUMAN, not the credential: an admin's session resolves the same registry their + # dashboard shows (connect-page parity), bounded like an admin key by explicit + # object_permission scope, the entitlement ceiling, and the session resource scope below. is_admitted_subject: Final = _is_mcp_admitted_user_subject(user_api_key_auth) # The key explicitly opted out of every MCP server. Return zero before @@ -2982,26 +2979,16 @@ class MCPServerManager: ) try: - # If admin but NO explicit object permission, get all servers (never for an admitted - # subject — see is_admitted_subject above) - if ( - user_api_key_auth - and not is_admitted_subject - and _user_has_admin_view(user_api_key_auth) - and not has_explicit_object_permission - # An entitlement attached to the HUMAN binds them whatever their role: it is the - # person's scope, not the credential's, so an admin role is not a waiver of it. An - # UNRESOLVED entitlement also skips the shortcut, so the resolver denies rather than - # handing over the whole registry on a transient fault. - and not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth) - ): - verbose_logger.debug("Admin user without explicit object_permission - returning all servers") - return list(self.get_registry().keys()) - - # Get allowed servers from object permissions (respects object_permission even for admins) - allowed_mcp_servers: Final = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) - verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", allowed_mcp_servers) - combined_servers: Final = set(allowed_mcp_servers) + # Admin view with no explicit object permission and no entitlement ceiling resolves the + # whole registry, for keys AND admitted session subjects alike (one predicate owns the + # question). Seeded into the union rather than returned early so the session resource + # scope below still bounds a per-server envelope held by an admin. + combined_servers: Final = ( + set(self.get_registry().keys()) + if await MCPRequestHandler.admin_view_unscoped(user_api_key_auth) + else set(await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth)) + ) + verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", combined_servers) combined_servers.update( await self.operator_open_server_ids( user_api_key_auth, diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index a30b5ee9e49..1ca2ffc703d 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -180,6 +180,22 @@ def well_known_root_suffix() -> str: return "" if root == "/" else root +def get_route_relative_request_path(scope: Scope) -> str: + """The request path the MCP route shapes are written against: the raw ASGI path with the + deployment's ``root_path`` removed. + + ``scope["path"]`` and ``_original_path`` are both raw request-line paths, so on a sub-path + deployment they still carry the ``SERVER_ROOT_PATH`` prefix (``/litellm/{server}/mcp``) while + every route shape compared against them is root-relative. Mirrors the segment-boundary strip in + :func:`litellm.proxy.auth.auth_utils.get_request_route`, which the rest of the MCP auth path + already routes through, so ``/litellmfoo`` is not truncated under ``root_path=/litellm``.""" + raw_path = str(scope.get("_original_path") or scope.get("path", "") or "") + root_path = str(scope.get("app_root_path") or scope.get("root_path") or "").rstrip("/") + if root_path and (raw_path == root_path or raw_path.startswith(f"{root_path}/")): + return raw_path[len(root_path) :] + return raw_path + + def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str: """The per-server protected-resource metadata URL matching the spelling the request arrived on, so a strict RFC 9728 client resolves the same route the proxy registered. @@ -188,7 +204,7 @@ def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str the route decorators insert it (see :func:`well_known_root_suffix`).""" request: Final = Request(scope) base_url: Final = get_request_base_url(request) - _path: Final = scope.get("_original_path") or scope.get("path", "") or "" + _path: Final = get_route_relative_request_path(scope) if _path.startswith(f"/{server_name}/mcp"): return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{server_name}/mcp" diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 0dc85c0318c..3c6eb06bc71 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -51,6 +51,8 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, get_passthrough_www_authenticate, + get_route_relative_request_path, + well_known_root_suffix, ) from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, @@ -3782,14 +3784,15 @@ if MCP_AVAILABLE: request = StarletteRequest(scope) base_url = get_request_base_url(request) - _path = scope.get("_original_path") or scope.get("path", "") or "" + _path = get_route_relative_request_path(scope) # Pick the well-known AS-metadata form that matches the inbound route # so strict RFC 9728 §3.2 clients can resolve it correctly. + as_metadata_root = f"{base_url}/.well-known/oauth-authorization-server{well_known_root_suffix()}" if _path.startswith(f"/mcp/{server_name}"): - _as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}" + _as_url = f"{as_metadata_root}/mcp/{server_name}" else: - _as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}" + _as_url = f"{as_metadata_root}/{server_name}" authorization_uri = f'Bearer authorization_uri="{_as_url}"' raise HTTPException( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index de5e4628f54..0840d37ffa1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -15,7 +15,7 @@ from pydantic import ( field_validator, model_validator, ) -from typing_extensions import NotRequired, Required, TypedDict +from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS @@ -3537,6 +3537,7 @@ class SpendLogsMetadata(TypedDict): max_retries: int | None # Max retries configured for this request cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.) compression_savings: CompressionSavingsMetadata | None + autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 65a271d4029..283194bad7c 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -316,6 +316,7 @@ class DBSpendUpdateWriter: model_id=payload.get("model_id"), llm_router=_get_llm_router, cost_breakdown=metadata.get("cost_breakdown"), + recorded_autorouter_savings=metadata.get("autorouter_savings"), ) transaction: Final = build_autorouter_turn_transaction( payload=payload, @@ -1877,6 +1878,7 @@ class DBSpendUpdateWriter: llm_router=_get_llm_router, usage_object=usage_obj, cost_breakdown=_metadata.get("cost_breakdown"), + recorded_autorouter_savings=_metadata.get("autorouter_savings"), ) daily_transaction: Final = BaseDailySpendTransaction( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1541b8acfdc..c1099081867 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -64,6 +64,15 @@ _TRANSPORT_ONLY_CREDENTIAL_KEYS: Final = frozenset({"provider_specific_header", # Excludes the two explicit litellm headers which are handled with higher priority. _GENERIC_SESSION_ID_HEADER_RE: Final = re.compile(r"^x-.+-session-id$", re.IGNORECASE) _EXPLICIT_SESSION_HEADERS: Final = frozenset({"x-litellm-trace-id", "x-litellm-session-id"}) +# Codex carries its conversation uuid in unprefixed headers, so the +# x--session-id convention above never matches it. Current builds send +# ``session-id``/``thread-id``; builds before the codex-api split sent +# ``session_id``/``conversation_id``. Ordered session before thread. +_CODEX_SESSION_ID_HEADERS: Final = ("session-id", "session_id", "thread-id", "conversation_id") +# Matches every first-party Codex originator: codex-tui, codex_cli_rs, codex_exec, +# codex_vscode, "Codex ...". A separator is required so an unrelated "codexfoo" client +# does not read as Codex. +_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE) # Session-id values must be non-empty strings of alphanumerics, hyphens, or underscores # (covers UUIDs and most common session-id formats). _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") @@ -583,6 +592,35 @@ def _extract_generic_session_id_from_headers( return None +def _extract_codex_session_id_from_headers( + normalized: Mapping[str, str], +) -> str | None: + """ + Read Codex's conversation uuid off one of ``_CODEX_SESSION_ID_HEADERS``. + + Codex sends no request metadata the Anthropic path could parse and no + ``x-``-prefixed session header, so without this every turn of a Codex session + falls through to a freshly generated per-call trace id and lands as its own + row in the logs instead of grouping. + + Unprefixed names like ``session-id`` are generic enough that another client + could send one meaning something unrelated, and colliding values across + callers would merge their traces, so this only applies to callers that + identify as Codex. + """ + user_agent: Final = normalized.get("user-agent") + if not isinstance(user_agent, str) or not is_codex_user_agent(user_agent): + return None + return next( + ( + value + for value in (normalized.get(header) for header in _CODEX_SESSION_ID_HEADERS) + if isinstance(value, str) and _SESSION_ID_VALUE_RE.match(value) + ), + None, + ) + + def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None: """ Extract chain id for call chaining from request headers. @@ -592,6 +630,7 @@ def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None: 2. ``x-litellm-session-id`` (explicit) 3. Any ``x--session-id`` header whose value looks like a session id (alphanumeric / UUID, at least 8 chars). E.g. ``x-claude-code-session-id``. + 4. Codex's unprefixed ``session-id`` / ``thread-id``, for Codex callers only. Header keys are matched case-insensitively so this works with raw header dicts from any transport. @@ -606,6 +645,7 @@ def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None: normalized.get("x-litellm-trace-id") or normalized.get("x-litellm-session-id") or _extract_generic_session_id_from_headers(normalized) + or _extract_codex_session_id_from_headers(normalized) ) @@ -640,10 +680,13 @@ def is_claude_code_user_agent(user_agent: str) -> bool: def is_codex_user_agent(user_agent: str) -> bool: - """Codex identifies itself as ``codex_cli_rs/ ...`` (TUI), - ``codex_exec/ ...`` (exec mode), or ``codex_vscode/ ...`` - (IDE extension); all share the ``codex_`` prefix.""" - return user_agent.startswith("codex_") + """Codex builds its user agent as ``/ ...`` and ships + several first-party originators: ``codex-tui``, ``codex_cli_rs``, + ``codex_exec`` (exec mode), ``codex_vscode`` (IDE extension) and ``Codex ...`` + (see ``is_first_party_originator`` in codex-rs). They agree only on the + ``codex`` stem, and the TUI sends a bare ``codex-tui`` with no version at all, + so match the stem plus a separator rather than any one spelling.""" + return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) def should_auto_drop_params_for_agentic_cli(user_agent: str, data: dict, proxy_config: ProxyConfig) -> bool: @@ -3001,36 +3044,36 @@ async def add_guardrails_from_policy_engine( ) +_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( + (LlmProviders.ANTHROPIC.value, LlmProviders.BEDROCK.value, LlmProviders.VERTEX_AI.value) +) +_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = LlmProviders.ANTHROPIC.value + + def add_provider_specific_headers_to_request( data: dict, headers: dict, ): from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key - anthropic_headers: Final = {} - # boolean to indicate if a header was added - added_header = False - for header in ANTHROPIC_API_HEADERS: - if header in headers: - header_value = headers[header] - anthropic_headers[header] = header_value - added_header = True + anthropic_api_headers: Final = {header: headers[header] for header in ANTHROPIC_API_HEADERS if header in headers} + anthropic_oauth_credential_headers: Final = { + header: value + for header, value in headers.items() + if header.lower() == "authorization" and is_anthropic_oauth_key(value) + } - # Check for Authorization header with Anthropic OAuth token (sk-ant-oat*) - # This needs to be handled via provider-specific headers to ensure it only - # goes to Anthropic-compatible providers, not all providers in the router - for header, value in headers.items(): - if header.lower() == "authorization" and is_anthropic_oauth_key(value): - anthropic_headers[header] = value - added_header = True - break - if added_header is True: - # Anthropic headers work across multiple providers - # Store as comma-separated list so retrieval can match any of them - data["provider_specific_header"] = ProviderSpecificHeader( - custom_llm_provider=f"{LlmProviders.ANTHROPIC.value},{LlmProviders.BEDROCK.value},{LlmProviders.VERTEX_AI.value}", - extra_headers=anthropic_headers, + scoped_headers: Final = [ + ProviderSpecificHeader(custom_llm_provider=providers, extra_headers=extra_headers) + for providers, extra_headers in ( + (_ANTHROPIC_API_HEADER_PROVIDERS, anthropic_api_headers), + (_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS, anthropic_oauth_credential_headers), ) + if extra_headers + ] + + if scoped_headers: + data["provider_specific_header"] = scoped_headers[0] if len(scoped_headers) == 1 else scoped_headers def _add_otel_traceparent_to_data(data: dict, request: Request): diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b003daa9d79..217fc61a56c 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -27,6 +27,7 @@ from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.litellm_core_utils.ptu_pricing import ( CUSTOM_PRICING_FIELDS, PTU_EMPTIED_PRICING_FIELDS, + PTU_MODEL_INFO_FIELDS, PTU_ZEROED_PRICING_FIELDS, PTU_ZEROED_TABLE_FIELDS, SEARCH_CONTEXT_SIZES, @@ -247,7 +248,6 @@ def _raise_on_strategy_router_write_violation( ) -_PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to") _PTU_PRICED_PAIR: Final = frozenset({"ptu_count", "cost_per_ptu_per_hour"}) @@ -261,7 +261,7 @@ def _explicitly_cleared_ptu_fields(model_info: ModelInfo | None) -> frozenset[st return frozenset() return frozenset( field - for field in _PTU_MODEL_INFO_FIELDS + for field in PTU_MODEL_INFO_FIELDS if field in model_info.model_fields_set and getattr(model_info, field) is None ) @@ -294,7 +294,7 @@ def _raise_if_ptu_cost_attribution_disabled(incoming_model_info: Mapping[str, ob """ if is_ptu_cost_attribution_enabled(): return - supplied: Final = tuple(field for field in _PTU_MODEL_INFO_FIELDS if incoming_model_info.get(field) is not None) + supplied: Final = tuple(field for field in PTU_MODEL_INFO_FIELDS if incoming_model_info.get(field) is not None) if not supplied: return raise HTTPException( diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 9c2e94dd861..4652719a23b 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2726,6 +2726,34 @@ ], "default_model_placeholder": "sap/gpt-4" }, + { + "provider": "SCX_AI", + "provider_display_name": "SCX.ai", + "litellm_provider": "scx-ai", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.scx.ai/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "scx-ai/GLM-5.2" + }, { "provider": "Snowflake", "provider_display_name": "Snowflake", diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 997180efdde..b0f1546e15e 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, NamedTuple import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token if TYPE_CHECKING: @@ -437,6 +438,97 @@ def extract_cache_creation_tokens(usage_object: Mapping[str, object] | None) -> return int(written) +def _proxy_llm_router() -> "Router | None": + """The running proxy's router, or ``None`` outside a proxy (public rates only).""" + try: + from litellm.proxy.proxy_server import llm_router + except Exception: # noqa: BLE001 # SDK-only usage has no proxy module to import + return None + return llm_router + + +def _numeric_savings(value: object) -> float | None: + """``value`` as a recorded savings figure, or ``None`` when it is not one.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + + +def autorouter_savings_for_request( + model: str | None, + custom_llm_provider: str | None, + routing_decision: Mapping[str, object] | None, + usage_object: Mapping[str, object] | None, + model_id: str | None = None, + llm_router: "Callable[[], Router | None] | None" = None, + cost_breakdown: Mapping[str, object] | None = None, +) -> float | None: + """Auto-router savings for one request, or ``None`` when the driver is off. + + ``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a + figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a + real figure for a routed request whose baseline resolved to the served deployment. + Never raises: pricing failures inside degrade to zero, and the driver-off cases + return ``None``, so this is safe on the logging path where a raise would fail the + request's logging. + """ + usage: Final = _usage_from_spend_log(usage_object) + if usage is None or not model: + return None + # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline + # the deciding router recorded on its decision; neither means the driver is off. + decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {} + recorded: Final = decision.get("savings_baseline_model") + recorded_id: Final = decision.get("savings_baseline_deployment_id") + configured: Final = litellm.autorouter_savings_baseline_model + baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None) + baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None + if not decision or not baseline_model: + return None + router_instance: Final = llm_router() if llm_router else None + return compute_autorouter_savings( + baseline_model=baseline_model, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + selected_info=_effective_model_info(router_instance, model_id, model or ""), + baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""), + cost_breakdown=cost_breakdown, + ) + + +def autorouter_savings_for_logging_payload( + request_metadata: Mapping[str, object], + model: str | None, + custom_llm_provider: str | None, + model_id: str | None, + usage_object: Mapping[str, object] | None, + cost_breakdown: Mapping[str, object] | None, +) -> float | None: + """The figure the logging payload records for a request, or ``None`` when none should be. + + Internal sub-calls (the auto-router classifier, shadow eval's shadow and judge legs) + are excluded here for the same reason the spend writer zeroes them: they can carry a + real routing decision, but they are not requests the caller made, so a figure stamped + on them would report savings for traffic no user sent. + """ + if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): + return None + routing_decision: Final = request_metadata.get("routing_decision") + return autorouter_savings_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + routing_decision=routing_decision if isinstance(routing_decision, Mapping) else None, + usage_object=usage_object, + model_id=model_id, + llm_router=_proxy_llm_router, + cost_breakdown=cost_breakdown, + ) + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, @@ -446,6 +538,7 @@ def compute_savings_spend( model_id: str | None = None, llm_router: "Callable[[], Router | None] | None" = None, cost_breakdown: Mapping[str, object] | None = None, + recorded_autorouter_savings: object = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -488,6 +581,11 @@ def compute_savings_spend( hypothetical token delta off flat rate keys, so they are blind to tiered pricing in the same way; that is pre-existing behaviour on two shipped drivers rather than something introduced here, and moving those numbers is its own change. + + ``recorded_autorouter_savings`` is the figure the logging path stamped on the spend + log's metadata, honoured over recomputation so the rollup, the turn table and the + per-request record cannot disagree; rows written before the field shipped carry + nothing and recompute, mirroring ``_recorded_token_cost``. """ # Deployment rates when the request came through one, public rates otherwise -- # `_effective_model_info` merges a deployment's configured prices over the built-in @@ -505,32 +603,24 @@ def compute_savings_spend( write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost) prompt_caching: Final = read_discount - write_premium - usage: Final = _usage_from_spend_log(usage_object) - if usage is None or not model: - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) - - # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline - # the deciding router recorded on its decision; neither means the driver is off. - decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {} - recorded: Final = decision.get("savings_baseline_model") - recorded_id: Final = decision.get("savings_baseline_deployment_id") - configured: Final = litellm.autorouter_savings_baseline_model - baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None) - baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None + # The figure the logging path recorded wins, before the usage gate on purpose: a row + # whose usage no longer parses still carries the number computed when it did. + recorded_savings: Final = _numeric_savings(recorded_autorouter_savings) autorouter: Final = ( - compute_autorouter_savings( - baseline_model=baseline_model, - selected_model=model, - selected_provider=custom_llm_provider, - usage=usage, - # Absent means the router never recorded a shape, which is the conservative - # reading: charge the cache write rather than claim a first turn's saving. - conversation_continuing=decision.get("conversation_continuing") is not False, - selected_info=_effective_model_info(router_instance, model_id, model or ""), - baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""), + recorded_savings + if recorded_savings is not None + else autorouter_savings_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + routing_decision=routing_decision, + usage_object=usage_object, + model_id=model_id, + llm_router=llm_router, cost_breakdown=cost_breakdown, ) - if decision and baseline_model - else 0.0 ) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) + return SavingsSpend( + compression=compression, + prompt_caching=prompt_caching, + autorouter=0.0 if autorouter is None else autorouter, + ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index cba1f9069d3..b6f695db512 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -100,6 +100,7 @@ def _get_spend_logs_metadata( litellm_overhead_time_ms: float | None = None, cost_breakdown: CostBreakdown | None = None, litellm_call_id: str | None = None, + autorouter_savings: float | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -132,6 +133,7 @@ def _get_spend_logs_metadata( max_retries=None, cost_breakdown=None, compression_savings=None, + autorouter_savings=autorouter_savings, litellm_call_id=litellm_call_id, ) verbose_proxy_logger.debug( @@ -158,6 +160,7 @@ def _get_spend_logs_metadata( clean_metadata["cold_storage_object_key"] = cold_storage_object_key clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms clean_metadata["cost_breakdown"] = cost_breakdown + clean_metadata["autorouter_savings"] = autorouter_savings clean_metadata["litellm_call_id"] = litellm_call_id return clean_metadata @@ -385,6 +388,9 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs cost_breakdown=( standard_logging_payload.get("cost_breakdown", None) if standard_logging_payload is not None else None ), + autorouter_savings=( + standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None + ), litellm_call_id=cast( str | None, kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"), diff --git a/litellm/router.py b/litellm/router.py index 7dedbe851d7..045fd32847c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -66,6 +66,8 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.ptu_pricing import ( + PTU_COST_ATTRIBUTION_ENV_VAR, + declares_ptu, is_ptu_cost_attribution_enabled, ptu_config_error, ptu_identity_error, @@ -8234,6 +8236,21 @@ class Router: ) duplicate_ids: Final = frozenset(model_id for model_id in declared_ids if declared_ids.count(model_id) > 1) + ptu_declared: Final = tuple( + str(entry.get("model_name")) + for entry in original_model_list + if isinstance(entry.get("model_info"), dict) + and entry["model_info"].get("db_model") is not True + and declares_ptu(entry["model_info"]) + ) + if ptu_declared and not is_ptu_cost_attribution_enabled(): + verbose_router_logger.warning( + "PTU fields are set on config.yaml deployment(s) %s, but PTU cost attribution is disabled, so no " + "flat cost accrues and this traffic is billed per token. Set %s=True to enable it", + ", ".join(ptu_declared), + PTU_COST_ATTRIBUTION_ENV_VAR, + ) + for model in original_model_list: _model_name = model.pop("model_name") _litellm_params = model.pop("litellm_params") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8a71d209618..ac2ab1c8363 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3193,6 +3193,7 @@ class StandardLoggingPayload(TypedDict): stream: bool | None response_cost: float cost_breakdown: CostBreakdown | None # Detailed cost breakdown + autorouter_savings: ReadOnly[float | None] # None = not an auto-routed caller request; 0.0 is a real figure response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None status: StandardLoggingPayloadStatus status_fields: StandardLoggingPayloadStatusFields @@ -3789,6 +3790,7 @@ class LlmProviders(str, Enum): LIBERTAI = "libertai" PINSTRIPES = "pinstripes" COGNITION = "cognition" + SCX_AI = "scx-ai" DARKBLOOM = "darkbloom" META = "meta" LITELLM_AGENT = "litellm_agent" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a1961136d11..3af7d9e5019 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4881,6 +4881,38 @@ "supports_tool_choice": true, "supports_vision": false }, + "azure/gpt-audio-mini": { + "deprecation_date": "2027-04-06", + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "azure/gpt-audio-mini-2025-10-06": { "deprecation_date": "2027-04-06", "input_cost_per_audio_token": 1e-05, @@ -5094,6 +5126,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/gpt-realtime-mini": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "azure/gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, @@ -19498,106 +19562,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "gemini/gemini-3.1-flash-lite-image": { - "rpm": 1000, - "tpm": 4000000, - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, - "vertex_ai/gemini-3.1-flash-lite-image": { - "input_cost_per_image": 0.00028, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 65536, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "image_generation", - "output_cost_per_image": 0.0336, - "output_cost_per_image_token": 3e-05, - "output_cost_per_token": 1.5e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_reasoning": true, - "supports_system_messages": true, - "supports_vision": true - }, "gemini-3.1-flash-image": { "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -19675,6 +19639,44 @@ }, "web_search_billing_unit": "per_query" }, + "gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -21505,6 +21507,42 @@ }, "web_search_billing_unit": "per_query" }, + "gemini/gemini-3.1-flash-lite-image": { + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "gemini", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "rpm": 1000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_vision": true, + "tpm": 4000000 + }, "gemini/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -26041,33 +26079,33 @@ "supports_minimal_reasoning_effort": true }, "gpt-5.6": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06, - "cache_creation_input_token_cost_flex": 3.125e-06, - "cache_creation_input_token_cost_priority": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, - "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_above_272k_tokens_flex": 5e-06, - "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_flex": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_flex": 2.5e-06, + "cache_creation_input_token_cost_priority": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_flex": 2e-07, + "cache_read_input_token_cost_priority": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_flex": 2e-06, + "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, - "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_flex": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_flex": 1e-05, + "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -26104,33 +26142,33 @@ "supports_xhigh_reasoning_effort": true }, "gpt-5.6-sol": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06, - "cache_creation_input_token_cost_flex": 3.125e-06, - "cache_creation_input_token_cost_priority": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, - "cache_read_input_token_cost_flex": 2.5e-07, - "cache_read_input_token_cost_priority": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, - "input_cost_per_token_above_272k_tokens_flex": 5e-06, - "input_cost_per_token_batches": 2.5e-06, - "input_cost_per_token_flex": 2.5e-06, - "input_cost_per_token_priority": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_flex": 2.5e-06, + "cache_creation_input_token_cost_priority": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_flex": 2e-07, + "cache_read_input_token_cost_priority": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_flex": 2e-06, + "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, - "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, - "output_cost_per_token_batches": 1.5e-05, - "output_cost_per_token_flex": 1.5e-05, - "output_cost_per_token_priority": 6e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_flex": 1e-05, + "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -26372,19 +26410,19 @@ "supports_parallel_function_calling": true }, "daybreak-blue-latest": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_272k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1e-05, + "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost_above_272k_tokens": 8e-07, + "input_cost_per_token": 4e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-05, - "output_cost_per_token_above_272k_tokens": 4.5e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_272k_tokens": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -31147,6 +31185,23 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "moonshot", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://platform.kimi.ai/docs/pricing/chat-k3", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, "moonshot/kimi-latest": { "cache_read_input_token_cost": 1.5e-07, "deprecation_date": "2026-01-28", @@ -36731,6 +36786,40 @@ "supports_vision": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, + "scx-ai/GLM-5.2": { + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 6.1e-07, + "litellm_provider": "scx-ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.98e-06, + "source": "https://scx.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "scx-ai/Qwen3.8-Max": { + "cache_read_input_token_cost": 2.1e-07, + "input_cost_per_token": 1.65e-06, + "litellm_provider": "scx-ai", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.99e-06, + "source": "https://scx.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 200000, @@ -41015,6 +41104,44 @@ "supports_reasoning": false, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" }, + "vertex_ai/gemini-3.1-flash-lite-image": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_image": 0.00028, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "image_generation", + "output_cost_per_image": 0.0336, + "output_cost_per_image_token": 3e-05, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_video_input": true, + "supports_vision": true + }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_audio_token": 5e-07, @@ -48554,6 +48681,156 @@ "supports_tool_choice": true, "supports_vision": true }, + "us.openai.gpt-5.6-sol": { + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "global.openai.gpt-5.6-sol": { + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "output_cost_per_token": 3e-05, + "output_cost_per_token_above_272k_tokens": 4.5e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "us.openai.gpt-5.6-terra": { + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, + "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, + "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "global.openai.gpt-5.6-terra": { + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_272k_tokens": 1.8e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "us.openai.gpt-5.6-luna": { + "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, + "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, + "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "global.openai.gpt-5.6-luna": { + "input_cost_per_token": 2e-07, + "input_cost_per_token_above_272k_tokens": 4e-07, + "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4e-08, + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_above_272k_tokens": 1.8e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7c1ca34c23c..1d8d374c2c4 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2261,6 +2261,23 @@ "interactions": true } }, + "scx-ai": { + "display_name": "SCX.ai (`scx-ai`)", + "url": "https://docs.litellm.ai/docs/providers/scx_ai", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "snowflake": { "display_name": "Snowflake (`snowflake`)", "url": "https://docs.litellm.ai/docs/providers/snowflake", diff --git a/pyproject.toml b/pyproject.toml index 6e3c181ae1d..fca5c7da1e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,8 +67,8 @@ proxy = [ "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.28.1,<2.0", - "litellm-proxy-extras==0.4.88", - "litellm-enterprise==0.1.58", + "litellm-proxy-extras==0.4.89", + "litellm-enterprise==0.1.59", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", @@ -341,9 +341,13 @@ filterwarnings = [ paths_to_mutate = [ "litellm/proxy/management_endpoints/", ] +# Only the unit tier that maps to paths_to_mutate. mutmut times and +# coverage-maps this whole set once before mutating, so a tier that needs a +# seeded database (tests/proxy_behavior/) kills the run before it starts, and +# a mutation score is only meaningful against the tests that claim to cover +# the mutated code anyway. tests_dir = [ "tests/test_litellm/proxy/management_endpoints/", - "tests/proxy_behavior/management/", ] also_copy = [ "litellm/", @@ -360,10 +364,16 @@ mutate_only_covered_lines = true # - rerunning a "failed" test on a mutant would mask which mutants are killed # vs. survive, so reruns are wrong for mutation testing regardless. # - xdist is unnecessary inside mutmut (mutmut handles its own parallelism). +# test_saml_sso.py cannot run inside mutmut's mutants/ sandbox: the copied tree +# re-imports cryptography's hash classes under a second identity, so x509 .sign() +# rejects the SHA256 instance the fixture builds with "Algorithm must be a +# registered hash algorithm". Nothing to do with mutation coverage, and one +# erroring test is enough to end the stats phase before any mutant runs. pytest_add_cli_args = [ "-p", "no:retry", "-p", "no:rerunfailures", "-p", "no:xdist", + "--ignore=tests/test_litellm/proxy/management_endpoints/test_saml_sso.py", ] [tool.coverage.run] diff --git a/ruff-tests.toml b/ruff-tests.toml index ff29bcff313..e52e1a96d00 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -32,6 +32,14 @@ # PT017 an `assert` on the caught error inside `except`. Nothing runs the handler when # the call stops raising, so the test goes green on the exact regression it was # written to catch. `pytest.raises` fails when the call succeeds +# RUF043 a `match=` pattern carrying regex metacharacters in a plain string. `match=` is +# `re.search`, so a `.` copied out of an error message is a wildcard and the block +# accepts messages the author never meant to accept. Mark a real regex raw, wrap a +# literal message in `re.escape`, and the pattern says which one it is +# F823 a module-level name read inside a function that also binds it lower down. The +# later binding makes the name local for the whole body, so the read raises +# UnboundLocalError, and in an autouse fixture that takes every test in the +# directory down with it # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -53,4 +61,6 @@ lint.select = [ "PT017", "PLR0133", "PLW0127", + "RUF043", + "F823", ] diff --git a/scripts/mutation_report.py b/scripts/mutation_report.py index a606e3f71cf..e0d4d569484 100644 --- a/scripts/mutation_report.py +++ b/scripts/mutation_report.py @@ -22,6 +22,7 @@ import tomllib from collections import defaultdict from difflib import SequenceMatcher from pathlib import Path +from typing import Final, NamedTuple from textwrap import dedent ROOT = Path(__file__).resolve().parent.parent @@ -33,16 +34,24 @@ def load_mutmut_config() -> dict: return tomllib.load(f)["tool"]["mutmut"] -def get_survivors() -> list[str]: +class MutmutResults(NamedTuple): + survivors: tuple[str, ...] + reported: int + + +def get_survivors() -> MutmutResults: proc = subprocess.run( [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False ) - survivors = [] - for line in proc.stdout.splitlines(): - m = re.match(r"\s*(\S+):\s*survived\s*$", line) - if m: - survivors.append(m.group(1)) - return survivors + verdicts = tuple( + m.groups() + for line in proc.stdout.splitlines() + if (m := re.match(r"\s*(\S+):\s*(\S.*?)\s*$", line)) + ) + return MutmutResults( + survivors=tuple(name for name, verdict in verdicts if verdict == "survived"), + reported=len(verdicts), + ) def get_mutmut_show(mutant_name: str) -> str: @@ -222,7 +231,52 @@ def render_meta_style_mutant( return "\n".join(out) -def render(config: dict, survivors: list[str], stats: dict | None) -> str: +RESOLVED_KEYS: Final = frozenset({"killed", "survived", "total"}) + + +def unresolved_counts(stats: dict) -> dict[str, int]: + """Every non-zero count that is neither a kill nor a survivor means a mutant did not + reach the tests. Reading it as "anything else" rather than as a list of known statuses + keeps a status this reporter has never met from passing as a clean sweep.""" + return {k: v for k, v in sorted(stats.items()) if k not in RESOLVED_KEYS and isinstance(v, int) and v > 0} + + +def clean_sweep_is_provable(stats: dict | None) -> bool: + """`mutmut results` omits killed mutants, so its silence is equally consistent with a + perfect run and with a run that never started. Only the stats file can tell them apart, + and only when it agrees that nothing survived and every mutant reached the tests.""" + if not stats or stats.get("killed", 0) <= 0 or stats.get("survived", 0) != 0: + return False + return not unresolved_counts(stats) + + +def no_survivors_verdict(results: MutmutResults, stats: dict | None) -> str: + if clean_sweep_is_provable(stats): + return "**No surviving mutants, and the run killed some, so the test suite caught every mutation.**" + if stats and stats.get("survived", 0) > 0: + return ( + f"**mutmut-cicd-stats.json counts {stats['survived']} surviving mutant(s) that " + "`mutmut results` did not list, so the two disagree and neither can be trusted. " + "This is not a passing score.**" + ) + if stats and unresolved_counts(stats): + unresolved = ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in unresolved_counts(stats).items()) + return ( + f"**No survivors, but {unresolved}, so those mutants never reached the tests " + "and the suite was not shown to catch them. This is not a passing score.**" + ) + if stats: + return "**Not one mutant was killed. This is not a passing score.**" + return ( + f"**mutmut-cicd-stats.json is missing and `mutmut results` printed {results.reported} " + "verdict(s), none of them a survivor. Since that command never lists killed mutants, a " + "clean sweep and a run that mutated nothing look identical from here. This is not a " + "passing score.**" + ) + + +def render(config: dict, results: MutmutResults, stats: dict | None) -> str: + survivors = list(results.survivors) by_function: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) for survivor in survivors: module_path, function_name, mutant_num = parse_mutant_name(survivor) @@ -235,17 +289,8 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str: out.append("## Summary") out.append("") if stats: - total = stats.get("total", 0) or sum( - stats.get(k, 0) - for k in ( - "killed", - "survived", - "no_tests", - "skipped", - "suspicious", - "timeout", - "segfault", - ) + total = stats.get("total", 0) or ( + stats.get("killed", 0) + stats.get("survived", 0) + sum(unresolved_counts(stats).values()) ) killed = stats.get("killed", 0) survived = stats.get("survived", 0) @@ -254,17 +299,15 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str: out.append(f"- Killed: **{killed}**") out.append(f"- Survived: **{survived}**") out.append(f"- Mutation score: **{score:.1f}%**") - for k in ("no_tests", "skipped", "suspicious", "timeout", "segfault"): - v = stats.get(k, 0) - if v: - out.append(f"- {k.replace('_', ' ').title()}: {v}") + for k, v in unresolved_counts(stats).items(): + out.append(f"- {k.replace('_', ' ').title()}: {v}") else: out.append(f"- Survivors found: **{len(survivors)}**") out.append("- (mutmut-cicd-stats.json not available — full counts unavailable)") out.append("") if not survivors: - out.append("**No surviving mutants — the test suite caught every mutation.**") + out.append(no_survivors_verdict(results, stats)) out.append("") return "\n".join(out) @@ -407,15 +450,22 @@ def main() -> int: except json.JSONDecodeError as exc: print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr) - survivors = get_survivors() - report = render(config, survivors, stats) + results = get_survivors() + report = render(config, results, stats) out_path = ROOT / "mutation-report.md" out_path.write_text(report) print( - f"Wrote {out_path} ({len(survivors)} survivor" - f"{'s' if len(survivors) != 1 else ''}, {len(report)} chars)" + f"Wrote {out_path} ({len(results.survivors)} survivor" + f"{'s' if len(results.survivors) != 1 else ''}, {len(report)} chars)" ) + if not results.survivors and not clean_sweep_is_provable(stats): + print( + "error: nothing was shown to have been killed, so the report cannot say " + "anything about the suite", + file=sys.stderr, + ) + return 1 return 0 diff --git a/terraform/provider/.goreleaser.yml b/terraform/provider/.goreleaser.yml index f41a29406b8..ba898ed9b2c 100644 --- a/terraform/provider/.goreleaser.yml +++ b/terraform/provider/.goreleaser.yml @@ -72,6 +72,7 @@ signs: - "--detach-sign" - "${artifact}" release: + prerelease: auto extra_files: - glob: 'terraform-registry-manifest.json' name_template: '{{ .ProjectName }}_{{ .Version }}_manifest.json' diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 7c744f04064..ff2f3f817f9 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -2,11 +2,22 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +Up to `0.4.0` the provider had its own version line, cut from the headings in +this file. It now ships at the **LiteLLM version**, on every LiteLLM release +channel, built from the same commit as the proxy (see `RELEASING.md`). The +headings below no longer drive a release; they record what changed and which +LiteLLM line first carried it. A change that breaks existing configurations +or state must be called out loudly here, because the version number can no +longer signal it. ## [Unreleased] +### Changed + +- **Versioning**: the provider is now published at the LiteLLM version, from the same commit as the proxy, on every LiteLLM release (dev, rc, stable). The `0.x` line ends at `0.4.0`; a `~> 0.4` constraint will not receive further releases, so re-pin to the LiteLLM version your proxy runs (for example `~> 1.99.0`). Existing `0.x` versions remain in the registry and keep verifying + ## [0.4.0] - 2026-08-06 ### Fixed diff --git a/terraform/provider/README.md b/terraform/provider/README.md index 3b59edd97c6..fe67d6aa430 100644 --- a/terraform/provider/README.md +++ b/terraform/provider/README.md @@ -6,6 +6,18 @@ This Terraform provider allows you to manage LiteLLM resources through Infrastru This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`) +## Versioning + +The provider version **is the LiteLLM version**. Every LiteLLM release (dev, rc and stable) publishes the provider at the same version as the proxy, built from the same commit, so `1.99.0` of the provider is the one that shipped with `1.99.0` of the proxy and was audited against that proxy's API. Pin the provider to the line your proxy runs: + +```hcl +version = "~> 1.99.0" +``` + +Pre-release versions (`1.99.0-rc.1`, `1.99.0-dev.1`) are published too; Terraform only selects one when it is pinned exactly. + +Versions `0.1.0` through `0.4.0` predate this scheme and sit on their own line. They stay in the registry, but **a `~> 0.4` constraint will never pick up another release**: re-pin to the LiteLLM version to keep receiving updates. + ## Features - Manage LiteLLM model configurations @@ -32,7 +44,7 @@ terraform { required_providers { litellm = { source = "BerriAI/litellm" - version = "~> 0.1.1" #HERE UPDATE VERSION ACCORDINGLY + version = "~> 1.99.0" # the LiteLLM version your proxy runs } } } @@ -218,6 +230,6 @@ This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENS - Always use environment variables or secure secret management solutions to handle sensitive information like API keys and AWS credentials. - Refer to the comprehensive documentation in the `docs/` directory for detailed usage examples and configuration options. -- Make sure to keep your provider version updated for the latest features and bug fixes. +- Keep the provider version in step with the LiteLLM version your proxy runs; see [Versioning](#versioning). - The provider now supports AWS cross-account access with `aws_session_name` and `aws_role_name` parameters in the model resource. - All example configurations have been consolidated into the documentation for better organization and maintenance. diff --git a/terraform/provider/RELEASING.md b/terraform/provider/RELEASING.md index 7b359047e2f..59f4c5f066c 100644 --- a/terraform/provider/RELEASING.md +++ b/terraform/provider/RELEASING.md @@ -4,7 +4,16 @@ This document describes the release process for the LiteLLM Terraform Provider. ## Overview -Releases are automated via GitHub Actions when a version tag is pushed. The workflow builds the provider for multiple platforms, signs the artifacts with GPG, and publishes them to GitHub Releases. +The provider is released **in lockstep with LiteLLM**: every LiteLLM release (dev, rc and stable) publishes the provider at the LiteLLM version, built from the same commit as the proxy. There is no separate provider release to cut. + +The flow, end to end: + +1. `BerriAI/project-releaser`'s release pipeline resolves the commit to release (`main` HEAD for dev; `main` HEAD or an operator-supplied SHA for rc/stable) and passes the release approval gate +2. Its componentized terraform job rsyncs `terraform/provider/` from that commit into `BerriAI/terraform-provider-litellm`, commits, and pushes the tag `v` (for example `v1.99.0`, `v1.99.0-rc.1`, `v1.99.0-dev.1`), alongside the `terraform-aws-litellm` / `terraform-google-litellm` module mirrors which get the same tag +3. The tag push triggers the mirror's own `Release` workflow (goreleaser): multi-platform build, GPG-signed checksums, GitHub release. It runs unattended; project-releaser does not wait for it +4. The public Terraform Registry ingests the GitHub release as provider version `` + +`terraform/provider/` only exists from LiteLLM ~1.95, so a stable patch cut from an older line skips the provider and publishes only the modules. ## Prerequisites @@ -68,113 +77,26 @@ Before publishing to the Terraform Registry: **Note**: The public key fingerprint must match the key used to sign the provider releases. -## Release Steps +## What a change needs -### 1. Prepare the Release +1. **Land it in `BerriAI/litellm`.** Open a PR against `litellm_internal_staging` with the source change and a `CHANGELOG.md` entry under `[Unreleased]`. CI runs `gofmt`, `go vet`, build, tests and the endpoint-drift audit. A change that breaks existing configurations or state must say so in the changelog: the version number cannot signal it any more +2. **Wait for the next LiteLLM release.** The nightly dev release carries it within a day; it reaches a stable version on the next stable cut +3. **Verify** (optional): the version appears at https://registry.terraform.io/providers/BerriAI/litellm and https://github.com/BerriAI/terraform-provider-litellm/releases. If the tag is on the mirror but there is no release, the goreleaser run failed: https://github.com/BerriAI/terraform-provider-litellm/actions -Before creating a release: +Locally, before opening the PR: -1. **Update CHANGELOG.md** - - Move items from `[Unreleased]` section to a new version section - - Follow [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format - - Use [Semantic Versioning](https://semver.org/spec/v2.0.0.html) for version numbers - - Include all notable changes since the last release +```bash +make test +make build +``` - Example: - ```markdown - ## [0.1.2] - 2026-02-20 +## Out-of-band publish or recovery - ### Added - - New feature description +Dispatch `Build and Publish Componentized Images + Chart` in `BerriAI/project-releaser` by hand with only `publish_terraform` enabled and the `git_ref` / `tag` of the release to (re)publish. The run waits on project-releaser's release approval, then mirrors and tags exactly as the pipeline does. - ### Fixed - - Bug fix description +The mirror is push-only: do not commit or tag `BerriAI/terraform-provider-litellm` directly. The publish refuses to overwrite an existing tag; a version that failed in goreleaser is recovered by re-running the mirror's `Release` workflow for that tag, not by re-tagging. - ### Changed - - Changed behavior description - ``` - -2. **Verify tests pass** - ```bash - make test - ``` - -3. **Verify the build works locally** - ```bash - make build - ``` - -4. **Land the changes in BerriAI/litellm** - - Open a PR to `BerriAI/litellm` updating `terraform/provider/CHANGELOG.md` (and any source changes) and merge it - -### 2. Mirror and Tag via project-releaser - -The provider source lives at `terraform/provider/` in `BerriAI/litellm`; `BerriAI/terraform-provider-litellm` is a thin release mirror. Do not commit or tag the mirror directly - -Normally there is nothing to do here. `BerriAI/project-releaser`'s release pipeline runs the same check on every release except `adhoc`, nightly included: it reads the topmost released heading in `terraform/provider/CHANGELOG.md`, probes the mirror for `v`, and dispatches `Publish Terraform provider` only when the changelog has moved ahead of what the mirror carries. Cutting the version heading in step 1 is therefore what releases the provider, and the next release picks it up, so the wait is a day rather than a week - -Dispatch by hand only for an out-of-band release, or to recover a run that failed: - -1. Go to `BerriAI/project-releaser` > **Actions** > `Publish Terraform provider` -2. Click **Run workflow**: - - `git_ref`: full 40-char commit SHA from `BerriAI/litellm` to release from - - `provider_version`: the new version without the `v` prefix (e.g. `0.3.0`) - - `dry_run`: optional; validates without pushing - -Automatic or manual, the run waits on the `production-release` approval in `project-releaser`, then rsyncs `terraform/provider/` into the mirror repo, commits, and pushes tag `v`. That approval is the only one in the flow. The tag push triggers the mirror's `Release` workflow (goreleaser), which runs unattended - -**Important**: -- Tags must follow the format: `v..` (e.g., `v0.1.2`, `v1.0.0`) -- The workflow refuses to overwrite an existing tag; publish a new version instead - -### 3. Monitor the Release Workflow - -1. Go to: https://github.com/BerriAI/terraform-provider-litellm/actions -2. Find the "Release" workflow run for your tag -3. Monitor the progress and check for any errors - -The workflow will: -- Check out the code -- Set up Go -- Import the GPG key -- Run `go mod tidy` -- Build binaries for multiple platforms (Linux, macOS, Windows, FreeBSD) -- Create archives and checksums -- Sign the checksums with GPG -- Create a GitHub release -- Upload all artifacts - -### 4. Verify the Release - -After the workflow completes successfully: - -1. **Check the GitHub Release** - - Go to: https://github.com/BerriAI/terraform-provider-litellm/releases - - Verify the release was created with the correct version - - Confirm all artifacts are present: - - Binary archives for each platform - - SHA256SUMS file - - SHA256SUMS.sig (GPG signature) - - terraform-registry-manifest.json - -2. **Verify the signature** (optional) - ```bash - # Download the checksums and signature - wget https://github.com/BerriAI/terraform-provider-litellm/releases/download/v0.1.2/terraform-provider-litellm_0.1.2_SHA256SUMS - wget https://github.com/BerriAI/terraform-provider-litellm/releases/download/v0.1.2/terraform-provider-litellm_0.1.2_SHA256SUMS.sig - - # Verify the signature - gpg --verify terraform-provider-litellm_0.1.2_SHA256SUMS.sig terraform-provider-litellm_0.1.2_SHA256SUMS - ``` - -### 5. Publish to Terraform Registry (Optional) - -If this provider is published to the Terraform Registry: - -1. The registry should automatically detect the new release via the GitHub webhook -2. If not, you may need to manually trigger a sync on the Terraform Registry dashboard -3. Verify the new version appears at: https://registry.terraform.io/providers/BerriAI/litellm/latest +The mirror's `.github/` directory (the `Release` workflow) is the one thing the rsync preserves, so a change to the goreleaser *workflow* is a direct PR on the mirror; a change to `.goreleaser.yml` itself lands here like any other source change. ## Troubleshooting @@ -207,21 +129,15 @@ If this provider is published to the Terraform Registry: ### Tag Already Exists -**Error**: The publish workflow refuses to push because the tag already exists on the mirror +**Error**: The publish job refuses to push because the tag already exists on the mirror -**Solution**: Tags are immutable by design. Re-run the workflow with a new patch version instead of deleting or moving an existing tag +**Solution**: Tags are immutable by design and the version is the LiteLLM version, so this means the provider was already mirrored for this release. If the registry is missing the version, re-run the mirror's `Release` workflow for the existing tag rather than re-tagging ## Version Numbering -This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html): +The provider version is the LiteLLM version, verbatim: `X.Y.Z` for a stable release, `X.Y.Z-rc.N` for a release candidate and `X.Y.Z-dev.N` for a nightly. It says which proxy the provider shipped with and was audited against; it does not follow SemVer's break-signalling, so breaking changes are announced in `CHANGELOG.md` and the registry docs instead. -- **MAJOR** version (1.0.0): Incompatible API changes -- **MINOR** version (0.1.0): New functionality in a backward-compatible manner -- **PATCH** version (0.0.1): Backward-compatible bug fixes - -For pre-1.0 releases: -- Breaking changes may occur in minor versions -- Patch versions should only contain bug fixes +Versions `0.1.0` to `0.4.0` predate this and remain in the registry on their own line. A `~> 0.4` constraint never receives another release. ## Security Considerations @@ -237,5 +153,4 @@ For pre-1.0 releases: - [Terraform Provider Publishing](https://www.terraform.io/docs/registry/providers/publishing.html) - [HashiCorp GPG Signing Requirements](https://www.terraform.io/docs/registry/providers/publishing.html#signing-releases) - [GitHub Actions Secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) -- [Semantic Versioning](https://semver.org/) - [Keep a Changelog](https://keepachangelog.com/) diff --git a/test-quality-budget.json b/test-quality-budget.json index 33e307779c5..0dea4e8fe93 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -1,18 +1,18 @@ { "TQ001": { - "limit": 750 + "limit": 744 }, "TQ002": { "limit": 742 }, "TQ003": { - "limit": 1078 + "limit": 62 }, "TQ004": { - "limit": 757 + "limit": 469 }, "TQ005": { - "limit": 2810 + "limit": 2405 }, "TQ006": { "limit": 34 diff --git a/tests/agent_tests/local_only_agent_tests/test_a2a.py b/tests/agent_tests/local_only_agent_tests/test_a2a.py index 16ff545db14..e2e73808b95 100644 --- a/tests/agent_tests/local_only_agent_tests/test_a2a.py +++ b/tests/agent_tests/local_only_agent_tests/test_a2a.py @@ -6,8 +6,6 @@ Run with: """ import asyncio -import os -import sys import json from typing import Optional from uuid import uuid4 @@ -18,9 +16,6 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import StandardLoggingPayload -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from a2a.types import MessageSendParams, SendMessageRequest diff --git a/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py b/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py index 4369bb800af..ff7e9da0368 100644 --- a/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py +++ b/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py @@ -10,13 +10,10 @@ Prerequisites: - LangGraph server running on localhost:2024 """ -import os -import sys from uuid import uuid4 import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from a2a.types import MessageSendParams, SendMessageRequest, SendStreamingMessageRequest diff --git a/tests/audio_tests/conftest.py b/tests/audio_tests/conftest.py index c4ff576e5bd..21e7c868641 100644 --- a/tests/audio_tests/conftest.py +++ b/tests/audio_tests/conftest.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, diff --git a/tests/audio_tests/test_audio_speech.py b/tests/audio_tests/test_audio_speech.py index fb9e679699a..f5a0cef6049 100644 --- a/tests/audio_tests/test_audio_speech.py +++ b/tests/audio_tests/test_audio_speech.py @@ -4,7 +4,6 @@ import asyncio import os import random -import sys import time import traceback from litellm._uuid import uuid @@ -13,9 +12,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/audio_tests/test_whisper.py b/tests/audio_tests/test_whisper.py index 333d806fe41..ba0ec02a02f 100644 --- a/tests/audio_tests/test_whisper.py +++ b/tests/audio_tests/test_whisper.py @@ -4,7 +4,6 @@ import asyncio import logging import os -import sys import time import traceback from typing import Optional @@ -41,9 +40,6 @@ def _audio_file2(): load_dotenv() -sys.path.insert( - 0, os.path.abspath("../") -) # Adds the parent directory to the system path from litellm import Router diff --git a/tests/batches_tests/conftest.py b/tests/batches_tests/conftest.py index e1899a22b6c..b46726c0c85 100644 --- a/tests/batches_tests/conftest.py +++ b/tests/batches_tests/conftest.py @@ -1,12 +1,7 @@ import asyncio -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402,F401 diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py index ae02c1be12c..b44b8435cd9 100644 --- a/tests/batches_tests/test_batch_rate_limits.py +++ b/tests/batches_tests/test_batch_rate_limits.py @@ -5,14 +5,10 @@ Integration Tests for Batch Rate Limits import asyncio import json import os -import sys import pytest from fastapi import HTTPException -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import DualCache diff --git a/tests/batches_tests/test_batches_logging_unit_tests.py b/tests/batches_tests/test_batches_logging_unit_tests.py index 62b6f5b08e4..5211b3ecb29 100644 --- a/tests/batches_tests/test_batches_logging_unit_tests.py +++ b/tests/batches_tests/test_batches_logging_unit_tests.py @@ -1,15 +1,10 @@ import asyncio import json -import os -import sys import traceback from unittest.mock import AsyncMock, MagicMock, patch from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import logging import time diff --git a/tests/batches_tests/test_bedrock_files_and_batches.py b/tests/batches_tests/test_bedrock_files_and_batches.py index b9045cc43d6..336fd7dd953 100644 --- a/tests/batches_tests/test_bedrock_files_and_batches.py +++ b/tests/batches_tests/test_bedrock_files_and_batches.py @@ -3,15 +3,11 @@ import asyncio import json as json_module import os -import sys import traceback import tempfile from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest diff --git a/tests/batches_tests/test_fine_tuning_api.py b/tests/batches_tests/test_fine_tuning_api.py index bd6672a52e9..41b47c1ee68 100644 --- a/tests/batches_tests/test_fine_tuning_api.py +++ b/tests/batches_tests/test_fine_tuning_api.py @@ -1,12 +1,7 @@ -import os -import sys import traceback import json import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from openai import APITimeoutError as Timeout import litellm diff --git a/tests/batches_tests/test_openai_batches_and_files.py b/tests/batches_tests/test_openai_batches_and_files.py index 0a49b3d77d1..ebd7fde7971 100644 --- a/tests/batches_tests/test_openai_batches_and_files.py +++ b/tests/batches_tests/test_openai_batches_and_files.py @@ -3,14 +3,10 @@ import asyncio import json import os -import sys import tempfile from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import logging import time @@ -103,6 +99,25 @@ def load_vertex_ai_credentials(): print("created gcs path service account=", os.environ["GCS_PATH_SERVICE_ACCOUNT"]) +async def cancel_batch_unless_already_terminal(batch_id: str, provider: str) -> None: + try: + cancel_batch_response = await litellm.acancel_batch(batch_id=batch_id, custom_llm_provider=provider) + except openai.ConflictError as e: + if "Cannot cancel a batch with status 'completed'" in str(e): + print(f"Batch already completed, cannot cancel: {e}") + return + if "Cannot cancel a batch with status 'failed'" not in str(e): + raise + failed_batch = await litellm.aretrieve_batch(batch_id=batch_id, custom_llm_provider=provider) + print(f"Batch failed before cancel, errors={failed_batch.errors}") + failure_codes = {err.code for err in (failed_batch.errors.data if failed_batch.errors else None) or []} + assert failure_codes == {"token_limit_exceeded"}, ( + f"batch failed for a reason other than the org's enqueued token limit: {failed_batch.errors}" + ) + return + print("cancel_batch_response=", cancel_batch_response) + + @pytest.mark.parametrize("provider", ["openai"]) # , "azure" @pytest.mark.asyncio @skip_if_no_openai_network @@ -176,24 +191,7 @@ async def test_create_batch(provider, tmp_path): result_file_path = tmp_path / "batch_job_results_furniture.jsonl" result_file_path.write_bytes(result) - # Cancel Batch - handle race condition where batch may already be completed - try: - cancel_batch_response = await litellm.acancel_batch( - batch_id=create_batch_response.id, - custom_llm_provider=provider, - ) - print("cancel_batch_response=", cancel_batch_response) - except openai.ConflictError as e: - # Only allow to pass if it's specifically the "batch already completed" error - if "Cannot cancel a batch with status 'completed'" in str(e): - print(f"Batch already completed, cannot cancel: {e}") - else: - # Re-raise other ConflictError types - raise - except Exception as e: - # Re-raise any other unexpected errors - print(f"Unexpected error during batch cancellation: {e}") - raise + await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider) pass @@ -395,24 +393,7 @@ async def test_async_create_batch(provider, tmp_path): result_file_path = tmp_path / "batch_job_results_furniture.jsonl" result_file_path.write_bytes(file_content.content) - # Cancel Batch - handle race condition where batch may already be completed - try: - cancel_batch_response = await litellm.acancel_batch( - batch_id=create_batch_response.id, - custom_llm_provider=provider, - ) - print("cancel_batch_response=", cancel_batch_response) - except openai.ConflictError as e: - # Only allow to pass if it's specifically the "batch already completed" error - if "Cannot cancel a batch with status 'completed'" in str(e): - print(f"Batch already completed, cannot cancel: {e}") - else: - # Re-raise other ConflictError types - raise - except Exception as e: - # Re-raise any other unexpected errors - print(f"Unexpected error during batch cancellation: {e}") - raise + await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider) mock_file_response = { diff --git a/tests/code_coverage_tests/bedrock_pricing.py b/tests/code_coverage_tests/bedrock_pricing.py index b2c9e78b06c..5984dd8b3a4 100644 --- a/tests/code_coverage_tests/bedrock_pricing.py +++ b/tests/code_coverage_tests/bedrock_pricing.py @@ -1,7 +1,5 @@ import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import litellm import requests from bs4 import BeautifulSoup diff --git a/tests/code_coverage_tests/check_spanattributes_value_usage.py b/tests/code_coverage_tests/check_spanattributes_value_usage.py index b180c572e73..6d1daa45fc7 100644 --- a/tests/code_coverage_tests/check_spanattributes_value_usage.py +++ b/tests/code_coverage_tests/check_spanattributes_value_usage.py @@ -27,10 +27,8 @@ import ast import os import re from typing import List, Tuple -import sys # Add parent directory to path so we can import litellm -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/code_coverage_tests/enforce_llms_folder_style.py b/tests/code_coverage_tests/enforce_llms_folder_style.py index 04a95b45196..a284cf9e1a9 100644 --- a/tests/code_coverage_tests/enforce_llms_folder_style.py +++ b/tests/code_coverage_tests/enforce_llms_folder_style.py @@ -1,8 +1,6 @@ import ast import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/code_coverage_tests/test_router_strategy_async.py b/tests/code_coverage_tests/test_router_strategy_async.py index 05bdca10f45..80bfcad4453 100644 --- a/tests/code_coverage_tests/test_router_strategy_async.py +++ b/tests/code_coverage_tests/test_router_strategy_async.py @@ -4,14 +4,9 @@ Test that all cache calls in async functions in router_strategy/ are async """ import os -import sys from typing import Dict, List, Tuple import ast -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os class AsyncCacheCallVisitor(ast.NodeVisitor): diff --git a/tests/documentation_tests/test_api_docs.py b/tests/documentation_tests/test_api_docs.py index 2faac371c39..d8536f13b9c 100644 --- a/tests/documentation_tests/test_api_docs.py +++ b/tests/documentation_tests/test_api_docs.py @@ -4,11 +4,7 @@ import os from dataclasses import dataclass import argparse import re -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/documentation_tests/test_exception_types.py b/tests/documentation_tests/test_exception_types.py index 87e128605c4..f554c4b38d4 100644 --- a/tests/documentation_tests/test_exception_types.py +++ b/tests/documentation_tests/test_exception_types.py @@ -11,9 +11,6 @@ import re # Backup the original sys.path original_sys_path = sys.path.copy() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm public_exceptions = litellm.LITELLM_EXCEPTION_TYPES diff --git a/tests/documentation_tests/test_router_settings.py b/tests/documentation_tests/test_router_settings.py index a1b6f1dac1d..75032f80dfa 100644 --- a/tests/documentation_tests/test_router_settings.py +++ b/tests/documentation_tests/test_router_settings.py @@ -2,11 +2,7 @@ import os import re import inspect from typing import Type -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/documentation_tests/test_standard_logging_payload.py b/tests/documentation_tests/test_standard_logging_payload.py index cdb51411833..22f7b71033f 100644 --- a/tests/documentation_tests/test_standard_logging_payload.py +++ b/tests/documentation_tests/test_standard_logging_payload.py @@ -1,12 +1,7 @@ -import os import re -import sys from typing import get_type_hints -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.types.utils import StandardLoggingPayload diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 840a40a54cd..15bd2c19ca9 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -77,13 +77,26 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers -A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. `fixture_bundle.py` owns the format. Record serves the proxy the same filtered stored response replay will serve later, so the two modes are byte-identical from the proxy's side of the socket +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. `fixture_bundle.py` owns the format. Record serves the proxy the same filtered stored response replay will serve later, so the two modes are byte-identical from the proxy's side of the socket + +Multipart identity is the fiddly corner, and the rules exist because each one had a collision behind it. A part counts as an upload when it carries a filename or declares its own content type, and everything else is an ordinary field. Field names get a `name[n]` suffix on repeats, with a literal `[` doubled first, so a form that repeats `purpose` never keys the same as one that literally sends `purpose[1]`. A field whose name reads as a credential is stored as ``, which stays key-preserving because the key is recomputed from the stored request rather than saved alongside it, so the live request carrying the real value still matches its redacted fixture. A field value that is not UTF-8 is stored as a base64 sha256 digest, base64 and not hex because the canonicalizer rewrites any 64-character hex run to `` and would fold every binary value onto one key. The uploaded parts contribute a JSON list rather than a `field:filename` string, so a separator inside a filename cannot impersonate a field boundary, and their byte length is stored for a reader's benefit but deliberately left out of the key, since the canonicalizer absorbs timestamp and id drift inside a file that changes its length Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, edge path, and a content hash, so identity survives re-records and machine changes while any real content drift comes back as an HTTP 599 naming the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a retry loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live providers A replayed response carries the recorded provider response id, and `LiteLLM_SpendLogs.request_id` (the table's primary key) is that id, so a replay against a database that still holds the record run's rows silently dedupes its spend inserts and any spend assertion goes red with zero matching rows and nothing in the proxy log. Run both modes with `E2E_RESET_SPEND_LOGS=1` (plus `DATABASE_URL` in the runner env) so each session truncates the table after itself, or replay against a fresh database, which is the CI shape -Current limits: streaming chunk fidelity is LIT-5742 (a streamed response records as one buffered body), CI wiring is LIT-5748, Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), multipart uploads have per-run random boundaries (the digest changes every run, so they always miss), and deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base) +The same id reuse reaches the managed-object tables. A replayed `/v1/files` or `/v1/batches` response carries the recorded provider object id, and `LiteLLM_ManagedObjectTable.model_object_id` is unique, so a unified batch create replayed against a database that still holds the record run's row fails on a Prisma unique-constraint violation, which surfaces as a 500, makes the router retry, and exhausts the recording. Replay the batches suite against a fresh database, or truncate `LiteLLM_ManagedObjectTable` and `LiteLLM_ManagedFileTable` before the run + +Edge-wired today: `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` (the reference), `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the Anthropic deployments in `llm_translation/test_messages_e2e.py` except the streaming test, and the OpenAI batch deployment behind `batches/` (`capabilities.openai_batch_params`). The mount base is not the same for both providers: OpenAI deployments register `f"{base}/v1"`, Anthropic deployments register `base` on its own, because litellm's Anthropic handler appends `/v1/messages` to `api_base` itself where the OpenAI handler appends only `/chat/completions`. Recording one suite locally is two runs against a proxy you already have up: + +```bash +E2E_FIXTURE_MODE=record E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 uv run pytest tests/e2e/llm_translation/test_chat_completions_contract_e2e.py +E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 uv run pytest tests/e2e/llm_translation/test_chat_completions_contract_e2e.py +``` + +Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days, and publishing one for CI is LIT-5748 + +Current limits: streaming chunk fidelity is LIT-5742 (a streamed response records as one buffered body), CI wiring is LIT-5748, Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode ## Typing diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 9096050a45a..29778b06d7a 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -57,13 +57,15 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop ```bash -E2E_FIXTURE_MODE=record uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v -E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v +E2E_FIXTURE_MODE=record E2E_FIXTURE_DIR=/tmp/e2e-fixtures uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v +E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v ``` +Bundles stay local. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and expires seven days after it was recorded, so record the suite you want before you replay it and never commit the result; publishing bundles for CI is LIT-5748 + One sharp edge: a replayed response reuses the recorded provider response id, and that id is the primary key of `LiteLLM_SpendLogs`, so replaying against a database that still holds the record run's rows silently dedupes the spend writes and a spend assertion fails with zero rows. Run both commands above with `E2E_RESET_SPEND_LOGS=1` (and `DATABASE_URL` set in the pytest env) so each session truncates the spend log table after itself, or point replay at a fresh database -Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (streaming, Bedrock, multipart) +Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. The suites wired to the edge today are `quota_management/spend_tracking/test_provider_edge_spend_e2e.py`, `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the non-streaming Anthropic tests in `llm_translation/test_messages_e2e.py`, and the OpenAI batch deployment behind `batches/`. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (streaming, Bedrock) Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index ca48204962a..f02d4eb4fe4 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -1,9 +1,11 @@ # Batches Test Coverage Matrix Live e2e coverage of the Batches API over a real proxy, real provider keys, and -real cost. Synchronous tier only: a batch's completion window is 24h, so these -tests never wait for `completed`. They assert the proxy accepts, routes, retrieves, -cancels, and lists a batch; everything created is deleted on teardown. +real cost. Mostly synchronous tier: a batch's completion window is 24h, so the +lifecycle matrix never waits for `completed`. It asserts the proxy accepts, routes, +retrieves, cancels, and lists a batch; everything created is deleted on teardown. +The exception is `TestBatchTerminalState`, which covers the completed state and +cost write-back via a cross-run marker baton (design below). ## Provider x operation @@ -12,19 +14,26 @@ row per supported (provider, scenario) pair, so there are no skipped cells in th parametrized run. The batches suite never skips: missing provider creds or upstream failures are hard test failures (see `tests/e2e/CLAUDE.md`). -| Provider | create | retrieve | cancel | list | file backing | -|-----------|--------|----------|--------|------|--------------| -| OpenAI | yes | yes | yes | yes | OpenAI Files | -| Azure | yes | yes | yes | yes | Azure Files | -| Vertex AI | yes | yes | yes | yes | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) | -| Bedrock | yes (unified only) | yes | no (limited upstream) | no | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | +| Provider | create | retrieve | cancel | list | content download | file backing | +|-----------|--------|----------|--------|------|------------------|--------------| +| OpenAI | yes | yes | yes | yes | yes (lifecycle + terminal output) | OpenAI Files | +| Azure | yes | yes | yes | yes | yes (byte-verbatim) | Azure Files | +| Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) | +| Bedrock | yes (unified only) | yes | no (limited upstream) | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | Bedrock cancel is unreliable upstream and list is unsupported, so both are gated off -(`can_cancel=False`, `can_list=False`) when that provider is enabled in the matrix. +(`can_cancel=False`, `can_list=False`) when that provider is enabled in the matrix; +flipping those gates is tracked in LIT-4774 and deliberately not part of this suite. Bedrock file upload requires a model on the request (`encoded` / `unified` scenarios only); `model_param` and `provider_fallback` are omitted because `POST /bedrock/v1/files` has no model-less passthrough path. +`GET /v1/files/{id}/content` is exercised for the unified upload path per backend in +`test_unified_file_content_downloads`. Azure stores the JSONL verbatim, so its download +is asserted byte-equal to the upload. Vertex (GCS) and Bedrock (S3) transform lines at +upload time, so those assert a 200 with non-empty parseable JSON lines instead. Gemini +(non-Vertex) raises `NotImplementedError` for file content and has no cell here. + ## Routing scenarios (per `litellm/proxy/batches_endpoints/endpoints.py`) Each create-capable provider runs all four. The test asserts the returned file id @@ -71,11 +80,59 @@ File delete asserts `object=="file"` and `deleted==True`. | `batch_client.py` | typed file upload/download + batch create/retrieve/cancel/list/delete over the shared ProxyClient; runtime batch model registration via /model/new; denial helpers | | `capabilities.py` | the provider x scenario matrix + per-provider /model/new params + id-shape classifiers + per-provider raw-id assertion | | `conftest.py` | session-scoped batch deployment registration and teardown | -| `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial | +| `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial, per-backend content download, failure paths, second-hop routing, terminal state + cost | + +## Failure paths + +`TestBatchFailurePaths` pins the customer-facing error contracts. A malformed input +file is a 400 at upload naming the bad content. A JSONL line whose url contradicts +the batch endpoint passes create (providers validate asynchronously) and drives the +batch to `failed` with structured `errors.data` (code/line/message), a null +`output_file_id`, and a $0 spend row keyed `{batch_id}_batch_cost` (LIT-4852: a +failed batch books $0 instead of crashing cost tracking). Cancelling that failed +batch is a 409 naming the terminal status. A file id encoded for one deployment wins +over a conflicting `model` param on create: the batch routes and re-encodes by the +file's embedded model (foreign-id precedence). + +## Second hop (two chained gateways) + +`TestBatchSecondHop` registers a `litellm_proxy/` deployment pointing at +the proxy's own base URL with a freshly minted virtual key, so unified upload and +create traverse gateway -> gateway -> OpenAI (LIT-5347, PR #36240). The pin: +`target_model_names` is rewritten to the inner deployment on the second hop and the +nested managed ids round-trip retrieve. This self-chaining only needs the proxy to +reach its own `PROXY_BASE_URL`, which holds both locally and on the e2e stage. + +## Terminal state + cost write-back (cross-run marker baton) + +The 24h completion window rules out submit-and-wait inside one run, so +`TestBatchTerminalState` amortizes across runs. Each run submits a 1-line marker +batch (stable metadata key/value plus a per-run field) and deliberately never +cancels or deletes it or its input file: the marker is the baton the next run picks +up (OpenAI files expire on their own after ~30 days). Polling is list-only, up to 5 +minutes, because retrieving a non-terminal batch books a $0 spend row whose +request_id then blocks the later real-cost row (`skip_duplicates`); the single +retrieve happens only once a completed marker exists. The assertion target is the +newest completed marker from ANY run: run-scoped deployment names mean the list +re-encodes prior-run batches under new encoded ids, so their spend keys are fresh +and a prior-run marker is billable by this run. On the 6h stage cadence the full +assertions are therefore deterministic from run 2 onward. On a cold start (no +completed marker within the poll budget) the test passes on the submission +assertions alone: a documented vacuous pass, not a skip. Markers aged past the 24h +window (25h-73h band, within the newest 100-item list page) must be terminal. + +The cost assertion is the LIT-5730 headline: retrieving a completed model-encoded +batch must write a positive spend row with call_type `aretrieve_batch` and token +usage. Before the fix in `litellm/batches/batch_utils.py`, the retrieve endpoint +re-encoded the response's `output_file_id` in place before the queued logging +worker ran, the worker sent that encoded id to OpenAI, got a 404, and the spend row +never landed. ## Out of scope (intentionally) -Driving a batch to `completed`, cost tracking on completion, and the DB write-back -are not covered here; the 24h window makes them unfit for a synchronous gate. That -logic belongs in a DI-stubbed proxy integration test under `tests/test_litellm/proxy/` -where the provider client is injected to return `completed` deterministically. +Unified (managed) batch cost is owned by the hourly `CheckBatchCost` poller, and a +terminal DB status short-circuits retrieve for those ids, so the terminal-state cell +uses the encoded path; poller timing does not fit an e2e gate and belongs in a +DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Bedrock +cancel/list stay gated pending LIT-4774. Gemini (non-Vertex) file content raises +`NotImplementedError` upstream and is not a coverage cell. diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index 968a357e8af..31e49f22450 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -51,6 +51,17 @@ class FileList(BaseModel): has_more: bool | None = None +class BatchErrorItem(BaseModel): + code: str | None = None + line: int | None = None + message: str | None = None + + +class BatchErrorList(BaseModel): + object: str | None = None + data: list[BatchErrorItem] = [] + + class BatchObject(BaseModel): id: str object: str | None = None @@ -58,6 +69,9 @@ class BatchObject(BaseModel): endpoint: str | None = None input_file_id: str | None = None output_file_id: str | None = None + error_file_id: str | None = None + errors: BatchErrorList | None = None + metadata: dict[str, str] | None = None completion_window: str | None = None created_at: int | None = None model: str | None = None @@ -79,12 +93,18 @@ class BatchCreateBody(BaseModel): endpoint: str = "/v1/chat/completions" completion_window: str = "24h" model: str | None = None + metadata: dict[str, str] | None = None class ModelQuery(BaseModel): model: str | None = None +class BatchListQuery(BaseModel): + model: str | None = None + limit: int | None = None + + def is_model_access_denied(resp: StreamingResponse) -> bool: """True if the proxy rejected the call because the key may not access the model.""" return resp.status_code == 403 and "key_model_access_denied" in resp.body @@ -175,12 +195,17 @@ class BatchClient: ) def list_batches( - self, *, key: str, provider: str | None = None + self, + *, + key: str, + provider: str | None = None, + model: str | None = None, + limit: int | None = None, ) -> Result[BatchList]: return self.proxy.transport.get( _batches_path(provider), headers=self.proxy.transport.bearer(key), - params=NoBody(), + params=BatchListQuery(model=model, limit=limit), response_type=BatchList, ) diff --git a/tests/e2e/batches/capabilities.py b/tests/e2e/batches/capabilities.py index 3988fb5e7e1..ee44a50d215 100644 --- a/tests/e2e/batches/capabilities.py +++ b/tests/e2e/batches/capabilities.py @@ -5,9 +5,9 @@ from __future__ import annotations import base64 import os from dataclasses import dataclass -from typing import Literal +from typing import Final, Literal -from e2e_config import unique_marker +from e2e_config import provider_edge_base, unique_marker from models import LiteLLMParamsBody _BATCH_RUN = unique_marker() @@ -17,6 +17,21 @@ def batch_model_name(base: str) -> str: return f"{base}-{_BATCH_RUN}" +OPENAI_BATCH_BACKEND: Final = "gpt-4o-mini" + + +def openai_batch_params() -> LiteLLMParamsBody: + """The OpenAI batch deployment, wired through the record/replay edge when a fixture + mode is active and straight at OpenAI otherwise (LIT-5974). Azure, Vertex, and + Bedrock stay live: none of them has an edge mount.""" + base = provider_edge_base("openai") + return LiteLLMParamsBody( + model=f"openai/{OPENAI_BATCH_BACKEND}", + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + ) + + def _env_ref(*names: str) -> str: for name in names: value = os.environ.get(name) @@ -47,10 +62,7 @@ class Provider: def litellm_params(self) -> LiteLLMParamsBody: match self.name: case "openai": - return LiteLLMParamsBody( - model="openai/gpt-4o-mini", - api_key="os.environ/OPENAI_API_KEY", - ) + return openai_batch_params() case "azure": return LiteLLMParamsBody( model="azure/gpt-5.4-mini-batch", @@ -107,7 +119,11 @@ class Capability: PROVIDERS: tuple[Provider, ...] = ( Provider( - "openai", batch_model_name("openai-batch"), "gpt-4o-mini", can_cancel=True, can_list=True + "openai", + batch_model_name("openai-batch"), + OPENAI_BATCH_BACKEND, + can_cancel=True, + can_list=True, ), Provider( "azure", @@ -210,6 +226,16 @@ def is_model_encoded_id(id_str: str) -> bool: return False +def decoded_model_from_id(id_str: str) -> str | None: + """Deployment name embedded in a model-encoded file/batch id, or None.""" + for prefix in ("file-", "batch_"): + if id_str.startswith(prefix): + decoded = _b64_decode(id_str[len(prefix) :]) + if decoded.startswith("litellm:") and ";model," in decoded: + return decoded.split(";model,", 1)[1].split(";")[0] + return None + + def matches_id_shape(shape: IdShape, id_str: str) -> bool: if shape == "managed": return is_managed_id(id_str) diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 12b848dd063..7af064b1fdd 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1,11 +1,12 @@ """Live e2e for the Batches API across every provider LiteLLM supports. -Synchronous tier only: a batch's completion window is 24h, so these never wait for -"completed". Each case uploads a tiny JSONL, creates the batch through one of the -four routing scenarios, asserts it was accepted (non-terminal status) and routed to -the right provider, then retrieves / cancels / lists where the provider supports it. -Everything created is deleted on teardown. Completion + cost tracking are out of -scope here (see COVERAGE.md). +Mostly synchronous tier: a batch's completion window is 24h, so the lifecycle +matrix never waits for "completed". Each case uploads a tiny JSONL, creates the +batch through one of the four routing scenarios, asserts it was accepted +(non-terminal status) and routed to the right provider, then retrieves / cancels / +lists where the provider supports it. Everything created is deleted on teardown. +The exception is TestBatchTerminalState, which carries completed-state + cost +write-back coverage via a cross-run marker baton (design in COVERAGE.md). Routing signal: for provider_fallback the raw batch id discriminates the provider; for the encoded/unified/model_param scenarios the proxy re-encodes the id, so the @@ -23,8 +24,9 @@ from datetime import datetime, timedelta, timezone from typing import Callable import pytest +from pydantic import BaseModel -from e2e_config import unique_marker +from e2e_config import PROXY_BASE_URL, unique_marker from batch_client import ( UPLOAD_FILENAME, @@ -40,12 +42,17 @@ from capabilities import ( BATCH_ID_SHAPE, CAPABILITIES, FILE_ID_SHAPE, + OPENAI_BATCH_BACKEND, OPENAI_BATCH_MODEL, + PROVIDERS, Capability, + Provider, batch_model_name, coverage_cells_for_lifecycle, + decoded_model_from_id, is_managed_id, matches_id_shape, + openai_batch_params, raw_id_matches_provider, ) from e2e_http import ( @@ -474,11 +481,22 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( ) -OPENAI_FILE_CONTENT_BACKEND = "gpt-4o-mini" +FILE_CONTENT_CELLS = { + "azure": "llm.files.azure_openai.content.nonstream.works", + "vertex_ai": "llm.files.vertex.content.nonstream.works", + "bedrock": "llm.files.bedrock.content.nonstream.works", +} +BYTE_FIDELITY_CONTENT_PROVIDERS = frozenset({"azure"}) class TestBatchFileContent: - """GET /v1/files/{id}/content returns the uploaded batch JSONL bytes.""" + """GET /v1/files/{id}/content returns the uploaded batch JSONL bytes. + + Azure stores the upload verbatim, so its download is asserted byte-equal. + Vertex (GCS) and Bedrock (S3) transform each JSONL line into the provider's + request format at upload time, so their downloads assert 200 plus non-empty + parseable JSON lines instead of byte equality. + """ @pytest.mark.covers( "llm.files.openai.content.nonstream.works", @@ -488,17 +506,11 @@ class TestBatchFileContent: self, client: BatchClient, resources: ResourceManager ) -> None: proxy_name = f"e2e-file-content-{unique_marker()}" - model_id = client.create_model( - proxy_name, - LiteLLMParamsBody( - model=f"openai/{OPENAI_FILE_CONTENT_BACKEND}", - api_key="os.environ/OPENAI_API_KEY", - ), - ) + model_id = client.create_model(proxy_name, openai_batch_params()) resources.defer(lambda: client.delete_model(model_id)) key = resources.key() - payload = render_jsonl(OPENAI_FILE_CONTENT_BACKEND) + payload = render_jsonl(OPENAI_BATCH_BACKEND) file = unwrap( client.upload_file( content=payload, @@ -522,6 +534,62 @@ class TestBatchFileContent: "downloaded file content must match the uploaded JSONL bytes" ) + @pytest.mark.parametrize( + "provider", + [ + pytest.param( + p, + id=p.name, + marks=pytest.mark.covers( + FILE_CONTENT_CELLS[p.name], exercised_on=["files"] + ), + ) + for p in PROVIDERS + if p.name in FILE_CONTENT_CELLS + ], + ) + def test_unified_file_content_downloads( + self, + provider: Provider, + client: BatchClient, + resources: ResourceManager, + batch_deployments: None, + ) -> None: + key = resources.key() + payload = render_jsonl(provider.raw_model) + file = unwrap( + client.upload_file( + content=payload, + form=FileUploadForm(purpose="batch", target_model_names=provider.model), + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert_file_object(file, provider=provider.name) + assert is_managed_id(file.id), ( + f"{provider.name}: unified upload must return a managed file id, got {file.id!r}" + ) + + downloaded = client.proxy.transport.download( + f"/v1/files/{file.id}/content", + headers=client.proxy.transport.bearer(key), + ) + assert downloaded.status_code == 200, ( + f"{provider.name}: file content must be 200, " + f"got {downloaded.status_code}: {downloaded.body[:300]}" + ) + body = downloaded.body.strip() + assert body, f"{provider.name}: file content download returned an empty body" + if provider.name in BYTE_FIDELITY_CONTENT_PROVIDERS: + assert body == payload.decode().strip(), ( + f"{provider.name}: downloaded content must match the uploaded JSONL bytes" + ) + else: + for line in body.splitlines(): + assert json.loads(line), ( + f"{provider.name}: content line is not JSON: {line[:200]}" + ) + class TestOpenAIFiles: """GET /v1/files (list) and GET /v1/files/{id} (retrieve) over the OpenAI route. @@ -1045,3 +1113,384 @@ class TestHostedVllmBatch: f"hosted_vllm batch has non-transitional status {batch.status!r}" ) assert_batch_object(batch) + + +BATCH_TERMINAL_STATUSES = frozenset({"completed", "failed", "expired", "cancelled"}) +FAILED_BATCH_POLL_SECONDS = 120.0 +FAILED_BATCH_POLL_INTERVAL_SECONDS = 5.0 + +AZURE_BATCH_RAW_MODEL = next(p.raw_model for p in PROVIDERS if p.name == "azure") + + +def _mismatched_endpoint_jsonl(model: str) -> bytes: + line = { + "custom_id": "req-1", + "method": "POST", + "url": "/v1/embeddings", + "body": {"model": model, "input": "ping"}, + } + return (json.dumps(line) + "\n").encode() + + +def _poll_until_terminal(client: BatchClient, batch_id: str, key: str) -> BatchObject: + deadline = time.monotonic() + FAILED_BATCH_POLL_SECONDS + fetched = retrieve_batch(client, batch_id, key=key, provider=None) + while fetched.status not in BATCH_TERMINAL_STATUSES and time.monotonic() < deadline: + time.sleep(FAILED_BATCH_POLL_INTERVAL_SECONDS) + fetched = retrieve_batch(client, batch_id, key=key, provider=None) + return fetched + + +class TestBatchFailurePaths: + """Customer-facing failure contracts for /v1/batches. + + A malformed input file is rejected at upload with a 400 naming the bad + content. A JSONL line whose url contradicts the batch endpoint is accepted + at create (providers validate asynchronously) and drives the batch to + "failed" with structured per-line errors, a null output_file_id, and a + zero-cost spend row (LIT-4852: a failed batch must book $0, not crash cost + tracking). Cancelling that already-failed batch returns a 409 naming the + terminal status. A file id encoded for one deployment wins over a + conflicting model param on create: the batch routes (and re-encodes) by the + file's embedded model, pinning that precedence. + """ + + @pytest.mark.covers( + "llm.batches.openai.malformed_jsonl.nonstream.works", + exercised_on=["files"], + ) + def test_malformed_jsonl_upload_rejected( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + result = client.upload_file( + content=b"this is not json\n", + form=FileUploadForm(purpose="batch"), + model=OPENAI_BATCH_MODEL, + key=resources.key(), + ) + match result: + case UnknownApiError(status_code=400, body=body): + assert "json" in body.lower(), ( + f"400 must name the malformed JSONL so users can fix the file, got: {body[:300]}" + ) + case _: + pytest.fail(f"malformed JSONL upload must be rejected with a 400, got: {result}") + + @pytest.mark.covers( + "llm.batches.openai.jsonl_endpoint_mismatch.nonstream.works", + "llm.batches.openai.cancel_terminal.nonstream.works", + exercised_on=["batches", "files"], + ) + def test_endpoint_mismatch_fails_batch_and_cancel_conflicts( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + key = resources.key() + file = unwrap( + client.upload_file( + content=_mismatched_endpoint_jsonl("gpt-4o-mini"), + form=FileUploadForm(purpose="batch"), + model=OPENAI_BATCH_MODEL, + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + + created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) + require_successful_call(created) + batch = BatchObject.model_validate_json(created.body) + + fetched = _poll_until_terminal(client, batch.id, key) + assert fetched.status == "failed", ( + f"endpoint-mismatched batch must fail, got {fetched.status!r}" + ) + assert fetched.output_file_id is None, ( + f"failed batch must have no output file, got {fetched.output_file_id!r}" + ) + assert fetched.errors is not None and fetched.errors.data, ( + "failed batch must surface structured errors so users can fix the JSONL" + ) + first_error = fetched.errors.data[0] + assert first_error.message, "batch error item has no message" + assert first_error.code, "batch error item has no code" + + rows = client.proxy.poll_logs_for_request_id(f"{fetched.id}_batch_cost") + assert rows, ( + f"failed batch {fetched.id} wrote no spend row; retrieve must book $0 (LIT-4852)" + ) + assert all((row.spend or 0) == 0 for row in rows), ( + f"failed batch must cost $0, got {[(r.request_id, r.spend) for r in rows]}" + ) + assert rows[0].call_type == "aretrieve_batch", ( + f"batch cost row call_type={rows[0].call_type!r}" + ) + + conflict = client.cancel_batch(batch.id, key=key) + match conflict: + case UnknownApiError(status_code=409, body=body): + assert "failed" in body.lower(), ( + f"409 must name the terminal status blocking the cancel, got: {body[:300]}" + ) + case _: + pytest.fail(f"cancel of a failed batch must return a 409 conflict, got: {conflict}") + + @pytest.mark.covers( + "llm.batches.openai.foreign_file_id.nonstream.works", + exercised_on=["batches", "files"], + ) + def test_foreign_encoded_file_id_routes_by_file_model( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + key = resources.key() + file = unwrap( + client.upload_file( + content=render_jsonl(AZURE_BATCH_RAW_MODEL), + form=FileUploadForm(purpose="batch"), + model=AZURE_BATCH_MODEL, + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert decoded_model_from_id(file.id) == AZURE_BATCH_MODEL, ( + f"upload did not encode the azure deployment into the file id: {file.id!r}" + ) + + created = client.create_batch( + body=BatchCreateBody(input_file_id=file.id, model=OPENAI_BATCH_MODEL), key=key + ) + require_successful_call(created) + batch = BatchObject.model_validate_json(created.body) + resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + + assert decoded_model_from_id(batch.id) == AZURE_BATCH_MODEL, ( + "create with a foreign encoded file id must route by the file's embedded model, " + f"but the batch id encodes {decoded_model_from_id(batch.id)!r} " + f"(model param was {OPENAI_BATCH_MODEL!r})" + ) + fetched = retrieve_batch(client, batch.id, key=key, provider=None) + assert fetched.id == batch.id + assert fetched.status, "retrieved foreign-file batch has no status" + + +class TestBatchSecondHop: + """Two-proxy batch routing: a litellm_proxy deployment chained to the gateway + itself (LIT-5347, PR #36240). + + The hop deployment's litellm_params point litellm_proxy/ at this + gateway's own base URL with a freshly minted virtual key, so the unified + upload and batch create traverse gateway -> gateway -> OpenAI. The regression + this pins: target_model_names must be rewritten to the inner deployment on + the second hop and the nested managed ids must round-trip retrieve. + """ + + @pytest.mark.covers( + "llm.batches.openai.second_hop.nonstream.works", + exercised_on=["batches", "files"], + ) + def test_unified_create_and_retrieve_via_chained_gateway( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + key = resources.key() + hop_name = batch_model_name("openai-batch-hop") + model_id = client.create_model( + hop_name, + LiteLLMParamsBody( + model=f"litellm_proxy/{OPENAI_BATCH_MODEL}", + api_base=PROXY_BASE_URL, + api_key=key, + ), + ) + resources.defer(lambda: client.delete_model(model_id)) + + file = unwrap( + client.upload_file( + content=render_jsonl("gpt-4o-mini"), + form=FileUploadForm(purpose="batch", target_model_names=hop_name), + key=key, + ) + ) + resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) + assert is_managed_id(file.id), ( + f"second-hop unified upload must return a managed file id, got {file.id!r}" + ) + + created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) + require_successful_call(created) + batch = BatchObject.model_validate_json(created.body) + resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key))) + + assert is_managed_id(batch.id), ( + f"second-hop create must return a managed batch id, got {batch.id!r}" + ) + assert batch.status in CREATED_BATCH_STATUSES, ( + f"second-hop batch has non-transitional status {batch.status!r}" + ) + assert_batch_object(batch) + + fetched = retrieve_batch(client, batch.id, key=key, provider=None) + assert fetched.id == batch.id + assert fetched.status, "second-hop retrieve returned no status" + + +class BatchOutputBody(BaseModel): + choices: list[object] = [] + + +class BatchOutputResponse(BaseModel): + status_code: int | None = None + body: BatchOutputBody | None = None + + +class BatchOutputLine(BaseModel): + response: BatchOutputResponse + + +TERMINAL_MARKER_KEY = "litellm_e2e_suite" +TERMINAL_MARKER_VALUE = "batches-terminal-baton" +TERMINAL_POLL_SECONDS = 300.0 +TERMINAL_POLL_INTERVAL_SECONDS = 10.0 +TERMINAL_LIST_LIMIT = 100 +TERMINAL_BAND_MIN_AGE_SECONDS = 25 * 3600 +TERMINAL_BAND_MAX_AGE_SECONDS = 73 * 3600 + + +def _marker_batches(client: BatchClient, key: str) -> list[BatchObject]: + listed = unwrap( + client.list_batches(key=key, model=OPENAI_BATCH_MODEL, limit=TERMINAL_LIST_LIMIT) + ) + return [ + b + for b in listed.data + if (b.metadata or {}).get(TERMINAL_MARKER_KEY) == TERMINAL_MARKER_VALUE + ] + + +def _await_completed_marker( + client: BatchClient, key: str +) -> tuple[BatchObject | None, list[BatchObject]]: + deadline = time.monotonic() + TERMINAL_POLL_SECONDS + while True: + markers = _marker_batches(client, key) + completed = max( + (b for b in markers if b.status == "completed"), + key=lambda b: b.created_at or 0, + default=None, + ) + if completed is not None or time.monotonic() >= deadline: + return completed, markers + time.sleep(TERMINAL_POLL_INTERVAL_SECONDS) + + +def _assert_aged_markers_terminal(markers: list[BatchObject]) -> None: + now = time.time() + stuck = [ + b + for b in markers + if b.created_at is not None + and TERMINAL_BAND_MIN_AGE_SECONDS <= now - b.created_at <= TERMINAL_BAND_MAX_AGE_SECONDS + and b.status not in BATCH_TERMINAL_STATUSES + ] + assert not stuck, ( + "marker batches past their 24h completion window must be terminal; stuck: " + f"{[(b.id, b.status, b.created_at) for b in stuck]}" + ) + + +class TestBatchTerminalState: + """Terminal state + cost write-back via a cross-run marker baton. + + Each run submits a 1-line marker batch (stable metadata key/value plus a + per-run field) and never cancels or deletes it: the marker is the baton the + next run picks up. Polling is list-only for up to 5 minutes because a + retrieve of a non-terminal batch books a $0 spend row whose request_id then + blocks the real-cost row (skip_duplicates); the single retrieve happens only + once a completed marker exists. The assertion target is the newest completed + marker from ANY run, so on the 6h stage cadence the full assertions are + deterministic from run 2 onward. On a cold start (no marker has ever + completed within the poll budget) the test passes on the submission + assertions alone: that is a documented vacuous pass, not a skip, and this + run's marker becomes the next run's target. Markers aged past OpenAI's 24h + completion window (25h-73h band, within the newest list page) must be + terminal. The cost assertion is the LIT-5730 headline: retrieving a + completed model-encoded batch must write a positive spend row keyed + {batch_id}_batch_cost; before the fix the logging worker fetched the + re-encoded output_file_id, 404d, and the row never landed. + """ + + @pytest.mark.covers( + "llm.batches.openai.terminal_state.nonstream.works", + "llm.batches.openai.terminal_state.nonstream.cost_logged", + exercised_on=["batches", "files"], + ) + def test_completed_batch_downloads_output_and_books_cost( + self, client: BatchClient, resources: ResourceManager, batch_deployments: None + ) -> None: + key = resources.key() + file = unwrap( + client.upload_file( + content=render_jsonl("gpt-4o-mini"), + form=FileUploadForm(purpose="batch"), + model=OPENAI_BATCH_MODEL, + key=key, + ) + ) + created = client.create_batch( + body=BatchCreateBody( + input_file_id=file.id, + metadata={ + TERMINAL_MARKER_KEY: TERMINAL_MARKER_VALUE, + "run": unique_marker(), + }, + ), + key=key, + ) + require_successful_call(created) + submitted = BatchObject.model_validate_json(created.body) + assert submitted.status in CREATED_BATCH_STATUSES, ( + f"marker batch has non-transitional status {submitted.status!r}" + ) + assert (submitted.metadata or {}).get(TERMINAL_MARKER_KEY) == TERMINAL_MARKER_VALUE, ( + f"create dropped the marker metadata: {submitted.metadata!r}" + ) + + completed, markers = _await_completed_marker(client, key) + _assert_aged_markers_terminal(markers) + if completed is None: + return + + fetched = retrieve_batch(client, completed.id, key=key, provider=None) + assert fetched.status == "completed", ( + f"listed-completed marker retrieved as {fetched.status!r}" + ) + assert fetched.output_file_id, "completed batch has no output_file_id" + + downloaded = client.proxy.transport.download( + f"/v1/files/{fetched.output_file_id}/content", + headers=client.proxy.transport.bearer(key), + ) + assert downloaded.status_code == 200, ( + f"output content must be 200, got {downloaded.status_code}: {downloaded.body[:300]}" + ) + first_line = BatchOutputLine.model_validate_json(downloaded.body.strip().splitlines()[0]) + assert first_line.response.status_code == 200, ( + f"batch output line reports failure: {downloaded.body[:400]}" + ) + assert first_line.response.body is not None and first_line.response.body.choices, ( + "batch output line has no choices" + ) + + rows = client.proxy.poll_logs_for_request_id( + f"{fetched.id}_batch_cost", + predicate=lambda found: any((row.spend or 0) > 0 for row in found), + ) + priced = [row for row in rows if (row.spend or 0) > 0] + assert priced, ( + f"completed batch {fetched.id} wrote no positive-cost spend row under " + f"request_id {fetched.id}_batch_cost; cost write-back is broken (LIT-5730)" + ) + cost_row = priced[0] + assert cost_row.call_type == "aretrieve_batch", ( + f"batch cost row call_type={cost_row.call_type!r}" + ) + assert (cost_row.total_tokens or 0) > 0, ( + f"batch cost row has no token usage: {cost_row.total_tokens!r}" + ) diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 44ed5765e38..1d4e1e028ca 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -64,6 +64,7 @@ - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"} - {id: llm.responses.openai.passthrough.stream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "A streamed POST /openai_passthrough/v1/responses is costed and keyed by the provider response id; it used to log a zero-cost row under a random id (GitHub issue #36523)"} +- {id: llm.responses.openai.passthrough_websocket.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai/v1/responses is accepted, so a responses.connect client reaches OpenAI through the same prefix its HTTP traffic uses; the prefix carried no websocket route and refused the upgrade with a 403 (GitHub issue #36088)"} - {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"} - {id: llm.responses.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Responses API"} - {id: llm.responses.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Anthropic translation (smoke)"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 8ae7dd01b5a..e6f08123b7c 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -26,6 +26,13 @@ - {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"} - {id: llm.batches.openai.key_model_access_denied.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Key model restriction 403 on upload/create"} - {id: llm.batches.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.18 / LIT-4778", rationale: "Missing input_file_id and invalid batch id rejected"} +- {id: llm.batches.openai.terminal_state.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "A batch actually reaches completed and its output file downloads through GET /v1/files/{id}/content with per-line provider responses"} +- {id: llm.batches.openai.terminal_state.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_batches_e2e.py / LIT-5730", fail_before_fix: proven, rationale: "Retrieving a completed model-encoded batch writes a positive spend row keyed {batch_id}_batch_cost (pins LIT-4852/LIT-5666; before the fix the logging worker 404d fetching the re-encoded output_file_id and the row was never written)"} +- {id: llm.batches.openai.malformed_jsonl.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "Uploading a non-JSON batch file is rejected with a 400 naming the bad line"} +- {id: llm.batches.openai.jsonl_endpoint_mismatch.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "JSONL line url that contradicts the batch endpoint drives the batch to failed with structured errors, retrieve stays clean, and the terminal retrieve books a zero-cost spend row (LIT-4852)"} +- {id: llm.batches.openai.cancel_terminal.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "Cancelling an already-terminal batch returns a 409 conflict naming the terminal status"} +- {id: llm.batches.openai.foreign_file_id.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "Create with one deployment's encoded file id and a conflicting model param routes by the file's embedded model; the returned batch id pins that precedence"} +- {id: llm.batches.openai.second_hop.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5347", rationale: "A litellm_proxy deployment chained to the gateway itself preserves target_model_names through nested unified ids; upload, create, and retrieve work over the two-hop chain (PR #36240)"} - {id: llm.files.openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai_files_endpoints/files_endpoints.py:46", rationale: "File upload returns OpenAIFileObject"} - {id: llm.files.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.16 / LIT-4778", rationale: "File upload without purpose rejected"} - {id: llm.files.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File retrieve by id"} @@ -40,10 +47,14 @@ - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"} - {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"} - {id: llm.files.openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files/{id}/content returns uploaded batch JSONL bytes"} +- {id: llm.files.azure_openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on an Azure unified file returns the uploaded JSONL bytes verbatim"} +- {id: llm.files.vertex.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on a Vertex unified file streams the GCS object back (provider-transformed JSONL, so asserts non-empty JSON lines rather than byte equality)"} +- {id: llm.files.bedrock.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on a Bedrock unified file streams the S3 object back (provider-transformed JSONL, so asserts non-empty JSON lines rather than byte equality)"} - {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_realtime_bedrock_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"} - {id: llm.google_native.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "LIT-4076 / proxy/google_endpoints/endpoints.py", fail_before_fix: proven, rationale: "google-native generateContent must stamp x-litellm-response-cost so SDK traffic reconciles against spend"} - {id: llm.google_native.gemini.basic.stream.works, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: stream, assertions: [works], source: "PR #28213 / proxy/proxy_server.py async_data_generator", fail_before_fix: proven, rationale: "streamGenerateContent must relay single-prefixed SSE frames with no [DONE] sentinel; doubled data: prefixes and the OpenAI terminator both break the Vertex Java SDK"} - {id: llm.realtime.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: realtime, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.19 / LIT-4778", rationale: "HTTP /v1/realtime/client_secrets returns an ephemeral credential"} +- {id: llm.realtime.openai.passthrough.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai_passthrough/v1/realtime is accepted and relayed to OpenAI; only HTTP routes were registered under the prefix, so realtime clients were refused with a 403 before a socket existed (GitHub issue #36088)"} - {id: llm.vector_stores.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store create/list/retrieve/delete lifecycle"} - {id: llm.vector_stores.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store search and invalid id errors"} - {id: llm.bedrock_native.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse happy path"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 8bf39f6021f..0266c75e1a7 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -150,6 +150,15 @@ ANOMALY_SPEND_SETTLE_SECONDS = float( ) +def ws_base_url() -> str: + """PROXY_BASE_URL with its scheme swapped for the websocket one, so a suite + opening a socket points at the same proxy every HTTP suite uses.""" + for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")): + if PROXY_BASE_URL.startswith(scheme): + return ws_scheme + PROXY_BASE_URL[len(scheme) :] + return PROXY_BASE_URL + + def datadog_mcp_url(*, toolsets: str = "core") -> str: """Regional Datadog remote MCP endpoint for this process's DD_SITE. diff --git a/tests/e2e/fixture_bundle.py b/tests/e2e/fixture_bundle.py index 6feb40fc8bc..aa0ba100b6c 100644 --- a/tests/e2e/fixture_bundle.py +++ b/tests/e2e/fixture_bundle.py @@ -5,7 +5,9 @@ version + format version) plus one subdirectory per test, holding one JSON file per provider-bound interaction in call order. Bundles older than ``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a green replay run can never certify against fixtures that have drifted more than -a week from the live providers. +a week from the live providers. Bump ``BUNDLE_FORMAT_VERSION`` whenever a change +moves recorded keys: a bundle recorded under the old rules then fails naming +both versions instead of quietly missing on every call. This module owns the format only. The provider-edge server that produces and consumes it lives in provider_edge.py (LIT-5745) and the canonical match keys @@ -28,7 +30,7 @@ from typing import Final from pydantic import BaseModel, JsonValue -BUNDLE_FORMAT_VERSION: Final = 2 +BUNDLE_FORMAT_VERSION: Final = 3 MAX_BUNDLE_AGE: Final = timedelta(days=7) MANIFEST_FILENAME: Final = "manifest.json" @@ -47,7 +49,14 @@ class RecordedRequest(BaseModel): over ``method``, ``path`` (the edge path including the provider mount, query string excluded), and the canonicalized headers, params, body, form, and file identity. Non-JSON bodies store a canonicalized content digest - instead of the bytes.""" + instead of the bytes. + + ``file_name`` is a JSON list of the uploaded parts' ``[field, filename, + content-type]`` triples rather than a flat label, so a separator inside a + filename cannot impersonate a field boundary. ``file_bytes`` is recorded for + a reader's benefit and stays out of the key: the canonicalizer absorbs + timestamp and id drift inside an uploaded file, and that drift moves the + byte count.""" method: str path: str diff --git a/tests/e2e/fixture_canonical.py b/tests/e2e/fixture_canonical.py index 427f06bf8fb..c043951a108 100644 --- a/tests/e2e/fixture_canonical.py +++ b/tests/e2e/fixture_canonical.py @@ -129,7 +129,6 @@ def canonicalize(request: RecordedRequest) -> CanonicalRequest: else { "name": None if request.file_name is None else canonical_string(request.file_name), "sha256": request.file_sha256, - "bytes": request.file_bytes, } ) content: Final[dict[str, JsonValue]] = { diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py index 5df61247db2..fa33737467e 100644 --- a/tests/e2e/llm_translation/endpoints_client.py +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -87,6 +87,7 @@ class RichMessagesRequest(BaseModel): max_tokens: int = 64 system: list[TextBlock] messages: list[RichMessage] + cache: dict[str, bool] = {"no-cache": True} class CompletionsRequest(BaseModel): diff --git a/tests/e2e/llm_translation/passthrough_client.py b/tests/e2e/llm_translation/passthrough_client.py index e0dfae679a9..20a8592db20 100644 --- a/tests/e2e/llm_translation/passthrough_client.py +++ b/tests/e2e/llm_translation/passthrough_client.py @@ -11,9 +11,13 @@ native request models are co-located here because only this suite uses them. from __future__ import annotations from dataclasses import dataclass +from urllib.parse import urlencode from pydantic import BaseModel, Field +from websockets.exceptions import InvalidStatus +from websockets.sync.client import connect +from e2e_config import ws_base_url from proxy_client import ProxyClient from e2e_http import FileUploadForm, Headers, NoBody, Result, StreamingResponse from models import ChatMessage @@ -175,6 +179,26 @@ class OpenAIEmbeddingBody(BaseModel): input: str +class WebsocketEnvelope(BaseModel): + """The one field every provider event carries, so the first frame off a + passthrough socket identifies itself without the suite parsing raw dicts.""" + + type: str + + +class WebsocketHandshake(BaseModel): + """What the proxy did with a websocket upgrade on a passthrough prefix. + + `rejected_status` is the HTTP status of a refused upgrade: a prefix carrying no + websocket route answers 403, before any socket exists. `first_event_type` is the + type of the first frame an accepted socket delivered, which is None when the + provider waits for the client to speak first. + """ + + rejected_status: int | None = None + first_event_type: str | None = None + + class PassthroughBatchList(BaseModel): """OpenAI's own batch page, relayed verbatim. `object` is required so a body that is not an OpenAI list fails validation instead of passing vacuously.""" @@ -339,5 +363,39 @@ class PassthroughClient: ), ) + # ---- OpenAI websocket passthrough ---------------------------------- + # + # The same prefixes over an upgrade instead of a POST, for the provider APIs + # that only speak websocket (realtime, responses.connect). + + def openai_passthrough_websocket( + self, + key: str, + path: str, + *, + model: str | None = None, + open_timeout: float = 30.0, + first_event_timeout: float = 30.0, + ) -> WebsocketHandshake: + query = f"?{urlencode({'model': model})}" if model is not None else "" + try: + connection = connect( + f"{ws_base_url()}{path}{query}", + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=open_timeout, + ) + except InvalidStatus as rejected: + return WebsocketHandshake(rejected_status=rejected.response.status_code) + with connection: + try: + frame = connection.recv(timeout=first_event_timeout) + except TimeoutError: + return WebsocketHandshake() + text = frame.decode("utf-8") if isinstance(frame, bytes) else frame + return WebsocketHandshake( + first_event_type=WebsocketEnvelope.model_validate_json(text).type + ) + + def build_client(proxy: ProxyClient) -> PassthroughClient: return PassthroughClient(proxy=proxy) diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index e6c5c19cbd1..632a9cf7e57 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -21,20 +21,13 @@ from pydantic import BaseModel, ConfigDict from websockets.sync.client import connect from websockets.sync.connection import Connection -from e2e_config import PROXY_BASE_URL, unique_marker +from e2e_config import unique_marker, ws_base_url from proxy_client import ProxyClient from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) -def ws_base_url() -> str: - for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")): - if PROXY_BASE_URL.startswith(scheme): - return ws_scheme + PROXY_BASE_URL[len(scheme) :] - return PROXY_BASE_URL - - def realtime_ws_url(model: str) -> str: return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}" diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index 78955974cd5..2e9cfcfe648 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -27,10 +27,10 @@ from pathlib import Path import pytest +from e2e_config import ws_base_url from realtime_client import ( PROVIDERS, RealtimeProvider, - ws_base_url, realtime_model, ) diff --git a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index 16628fd257a..f84ce197f88 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -25,10 +25,10 @@ import asyncio import pytest +from e2e_config import ws_base_url from realtime_client import ( PROVIDERS, RealtimeProvider, - ws_base_url, realtime_model, ) diff --git a/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py b/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py index 2eb7aeb643d..114beaae2fb 100644 --- a/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_contract_e2e.py @@ -6,7 +6,7 @@ Exercises the gateway against a live OpenAI deployment using customer request sh from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import provider_edge_base, unique_marker from e2e_http import StreamingResponse, assert_client_error, require_successful_call, unwrap from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody @@ -38,10 +38,15 @@ class ChatErrorEnvelope(BaseModel): def _register_chat_model(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: + base = provider_edge_base("openai") model = f"e2e-chat-sec-{unique_marker()}" model_id = proxy.create_model( model, - LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody( + model=OPENAI_BACKEND, + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + ), ) resources.defer(lambda: proxy.delete_model(model_id)) return model, resources.key() diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 35a53f055d8..265cc202ff4 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -9,7 +9,7 @@ covered by tests/e2e/quota_management/spend_tracking/. from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import provider_edge_base, unique_marker from e2e_http import ( assert_client_error, require_successful_call, @@ -27,6 +27,18 @@ class _OptionalEmbeddingsBody(BaseModel): input: str | list[str] | None = None +def _openai_embeddings_params() -> LiteLLMParamsBody: + """The OpenAI embeddings deployment, wired through the record/replay edge when a + fixture mode is active and straight at OpenAI otherwise (LIT-5974). Bedrock and + Vertex stay live: SigV4 signs the Host header, and neither has an edge mount.""" + base = provider_edge_base("openai") + return LiteLLMParamsBody( + model="openai/text-embedding-3-small", + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + ) + + class TestEmbeddingsEndpoint: @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") def test_embeddings_returns_vector( @@ -35,9 +47,7 @@ class TestEmbeddingsEndpoint: model = f"e2e-embeddings-{unique_marker()}" model_id = endpoints_client.create_model( model, - LiteLLMParamsBody( - model="openai/text-embedding-3-small", api_key="os.environ/OPENAI_API_KEY" - ), + _openai_embeddings_params(), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() @@ -106,9 +116,7 @@ class TestEmbeddingsEndpoint: model = f"e2e-embeddings-array-{unique_marker()}" model_id = endpoints_client.create_model( model, - LiteLLMParamsBody( - model="openai/text-embedding-3-small", api_key="os.environ/OPENAI_API_KEY" - ), + _openai_embeddings_params(), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() @@ -140,9 +148,7 @@ class TestEmbeddingsEndpoint: model = f"e2e-embeddings-missin-{unique_marker()}" model_id = endpoints_client.create_model( model, - LiteLLMParamsBody( - model="openai/text-embedding-3-small", api_key="os.environ/OPENAI_API_KEY" - ), + _openai_embeddings_params(), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index e0317e0389d..7f81a5e3946 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -9,7 +9,7 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import provider_edge_base, unique_marker from e2e_http import assert_client_error, require_successful_call, unwrap from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager @@ -50,16 +50,27 @@ def _approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) +def _anthropic_params() -> LiteLLMParamsBody: + """The Anthropic deployment, wired through the record/replay edge when a fixture + mode is active (LIT-5974). The mount base carries no ``/v1``: litellm's Anthropic + handler appends ``/v1/messages`` to ``api_base`` itself, where the OpenAI handler + appends only ``/chat/completions``.""" + base = provider_edge_base("anthropic") + return LiteLLMParamsBody( + model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY", api_base=base + ) + + class TestAnthropicMessages: def _register( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, + endpoints_client: EndpointsClient, + resources: ResourceManager, + params: LiteLLMParamsBody | None = None, ) -> tuple[str, str]: model = f"e2e-messages-{unique_marker()}" model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" - ), + model, _anthropic_params() if params is None else params ) resources.defer(lambda: endpoints_client.delete_model(model_id)) return model, resources.key() @@ -81,12 +92,7 @@ class TestAnthropicMessages: self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: model = f"e2e-messages-cost-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" - ), - ) + model_id = endpoints_client.create_model(model, _anthropic_params()) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() @@ -131,7 +137,13 @@ class TestAnthropicMessages: def test_messages_streams_completion( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: - model, key = self._register(endpoints_client, resources) + """Stays on a live Anthropic deployment in every mode: the edge buffers a + streamed response into one body, so chunk fidelity waits on LIT-5742.""" + model, key = self._register( + endpoints_client, + resources, + LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"), + ) result = endpoints_client.proxy.messages_stream( key, diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 17b0dbe1ae5..7e6a8b25155 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -28,6 +28,7 @@ from passthrough_client import ( ) EMBEDDING_MODEL = "text-embedding-3-small" +REALTIME_MODEL = "gpt-realtime-2" pytestmark = pytest.mark.e2e @@ -339,3 +340,52 @@ class TestOpenAIPassthroughSpend: f"the embeddings row logged no prompt tokens, so whatever cost it carries " f"was not computed from the real usage: {row}" ) + + +class TestOpenAIPassthroughWebsocket: + """The OpenAI passthrough prefixes must answer a websocket upgrade, not only a POST. + + The customer points realtime and responses.connect clients at the same prefixes + their HTTP traffic already uses. Only HTTP routes were registered under those + prefixes, so every upgrade was refused before a socket existed and those clients + could not reach the gateway at all. A refused upgrade is an HTTP response, not a + close frame, which is why these assert on the handshake rather than a close code. + """ + + @pytest.mark.covers("llm.realtime.openai.passthrough.stream.works") + def test_realtime_upgrade_reaches_openai_through_the_passthrough_prefix( + self, client: PassthroughClient, scoped_key: str + ) -> None: + """Pins GitHub issue #36088: /openai_passthrough/v1/realtime accepts the + upgrade and relays OpenAI's own session, instead of rejecting it with a 403.""" + handshake = client.openai_passthrough_websocket( + scoped_key, "/openai_passthrough/v1/realtime", model=REALTIME_MODEL + ) + + assert handshake.rejected_status is None, ( + f"/openai_passthrough/v1/realtime refused the websocket upgrade with HTTP " + f"{handshake.rejected_status}, so a realtime client cannot connect through " + "the gateway at all" + ) + assert handshake.first_event_type == "session.created", ( + "the accepted socket never carried OpenAI's opening session event, so the " + f"upgrade was not relayed upstream; the first frame was " + f"{handshake.first_event_type}" + ) + + @pytest.mark.covers("llm.responses.openai.passthrough_websocket.stream.works") + def test_responses_upgrade_is_accepted_on_the_openai_prefix( + self, client: PassthroughClient, scoped_key: str + ) -> None: + """Pins GitHub issue #36088 on the second prefix: /openai/v1/responses upgrades + as well. A responses.connect socket waits for the client to speak first, so the + accepted handshake is the whole signal here.""" + handshake = client.openai_passthrough_websocket( + scoped_key, "/openai/v1/responses", first_event_timeout=2.0 + ) + + assert handshake.rejected_status is None, ( + f"/openai/v1/responses refused the websocket upgrade with HTTP " + f"{handshake.rejected_status}; the prefix relays this route over HTTP but " + "drops a responses.connect client before the socket opens" + ) diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index ab0791e6b74..25a1e8043ed 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -20,10 +20,9 @@ headers must never touch disk. An unmatched replay call returns HTTP proxy relays as a provider error the failing test surfaces. v1 limits: only the mounts in ``EDGE_MOUNTS`` (SigV4 providers like Bedrock -sign the Host header, so a forwarding edge breaks their signatures), JSON and -opaque single-part bodies (multipart boundaries are random per request), -streaming fidelity is LIT-5742, and CI wiring is LIT-5748. Suites that do not -wire the edge keep hitting providers live in every mode. +sign the Host header, so a forwarding edge breaks their signatures), streaming +fidelity is LIT-5742, and CI wiring is LIT-5748. Suites that do not wire the +edge keep hitting providers live in every mode. """ from __future__ import annotations @@ -32,6 +31,7 @@ import base64 import difflib import functools import hashlib +import re import threading from collections import deque from collections.abc import Mapping @@ -59,7 +59,13 @@ from fixture_bundle import ( prepare_bundle, slug_for_test, ) -from fixture_canonical import CanonicalRequest, canonical_string, canonicalize +from fixture_canonical import ( + SECRET_PLACEHOLDER, + CanonicalRequest, + canonical_string, + canonicalize, + is_secret_field, +) from fixture_mode import ( FIXTURE_MODES, InvalidFixtureMode, @@ -103,26 +109,244 @@ _RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) -def _edge_request(method: str, path: str, query: str, body: bytes | None) -> RecordedRequest: - """The identity replay matches on: the edge path (mount included), the query - as params, and the body as parsed JSON, or as a canonicalized content digest - when it is not JSON so opaque uploads still match across runs.""" - params: Final = dict(parse_qsl(query, keep_blank_values=True)) - if not body: - return RecordedRequest(method=method.lower(), path=path, headers={}, params=params) - decoded: Final = body.decode("utf-8", errors="replace") +_BOUNDARY_PATTERN: Final = re.compile( + r'(?:^|;)\s*boundary\s*=\s*(?:"([^"]*)"|([^;,\s]+))', re.IGNORECASE +) +_DISPOSITION_NAME_PATTERN: Final = re.compile(r'(?:^|;)\s*name="([^"]*)"', re.IGNORECASE) +_DISPOSITION_FILENAME_PATTERN: Final = re.compile( + r'(?:^|;)\s*filename="([^"]*)"', re.IGNORECASE +) +_UNPARSED_MULTIPART: Final = "" +_BOUNDARY_PLACEHOLDER: Final = b"--" +_BINARY_FIELD_PREFIX: Final = " str: + wanted: Final = name.lower() + return next((value for key, value in headers.items() if key.lower() == wanted), "") + + +def _multipart_boundary(content_type: str) -> str | None: + """The declared boundary, or None when the envelope is not multipart or names no + usable boundary. ``boundary`` is matched only as a parameter in its own right, so a + longer name ending in it (``myboundary=``) is not mistaken for one, and an empty + boundary is refused rather than splitting the body on a bare ``--``.""" + if "multipart/form-data" not in content_type.lower(): + return None + match: Final = _BOUNDARY_PATTERN.search(content_type) + if match is None: + return None + quoted, bare = match.group(1), match.group(2) + return (quoted if quoted is not None else bare) or None + + +def _part_headers(head: bytes) -> dict[str, str]: + return { + name.strip().lower(): value.strip() + for line in head.decode("utf-8", errors="replace").split("\r\n") + for name, separator, value in [line.partition(":")] + if separator + } + + +def _parse_multipart_part(segment: bytes) -> _MultipartPart | None: + head, separator, content = segment.partition(b"\r\n\r\n") + if not separator: + return None + headers: Final = _part_headers(head) + disposition: Final = headers.get("content-disposition", "") + name_match: Final = _DISPOSITION_NAME_PATTERN.search(disposition) + if name_match is None: + return None + filename_match: Final = _DISPOSITION_FILENAME_PATTERN.search(disposition) + return _MultipartPart( + field_name=name_match.group(1), + filename=None if filename_match is None else filename_match.group(1), + content=content, + content_type=headers.get("content-type", ""), + ) + + +def _multipart_parts(body: bytes, boundary: str) -> tuple[_MultipartPart, ...] | None: + """The wire body split back into its parts, or None when it does not parse as the + declared envelope so the caller can fall back to the opaque content digest.""" + segments: Final = body.split(b"--" + boundary.encode()) + if len(segments) < 3 or not segments[-1].startswith(b"--"): + return None + parsed: Final = tuple( + _parse_multipart_part(segment.removeprefix(b"\r\n").removesuffix(b"\r\n")) + for segment in segments[1:-1] + ) + if any(part is None for part in parsed): + return None + return tuple(part for part in parsed if part is not None) + + +def _content_digest(content: bytes) -> str: + """Text is canonicalized before hashing so a per-run marker inside an uploaded JSONL + does not move the key; anything that is not UTF-8 is hashed byte for byte, since a + lossy decode collapses every binary payload of one length onto one digest.""" try: - parsed: Final[JsonValue] = _JSON.validate_json(decoded) - except ValueError: - return RecordedRequest( - method=method.lower(), - path=path, - headers={}, - params=params, - file_sha256=hashlib.sha256(canonical_string(decoded).encode()).hexdigest(), - file_bytes=len(body), + text: Final = content.decode("utf-8") + except UnicodeDecodeError: + return hashlib.sha256(content).hexdigest() + return hashlib.sha256(canonical_string(text).encode()).hexdigest() + + +def _is_file_part(part: _MultipartPart) -> bool: + """Whether a part is an upload rather than an ordinary field. A filename says so + outright, and so does a declared content type: clients attach one per part only for + a file, and a client that omits the filename (httpx drops the parameter when it is + empty) would otherwise have the file's bytes stored inline as a field value and key + identically to a plain field of the same name.""" + return part.filename is not None or bool(part.content_type) + + +def _field_value(part: _MultipartPart) -> str: + """What a field part contributes to the stored form. A secret-named field never has + its value written out, since the bundle is a file on disk and the key redacts that + field to the same placeholder either way, so replay still matches. A value that is + not UTF-8 is carried as a digest rather than decoded lossily, because a replacing + decode collapses every binary value of one length onto one string. That digest is + base64 rather than hex, since the canonicalizer rewrites any long hex run to a + ```` placeholder and would collapse the values right back together.""" + if is_secret_field(part.field_name): + return SECRET_PLACEHOLDER + try: + return part.content.decode("utf-8") + except UnicodeDecodeError: + digest: Final = base64.b64encode(hashlib.sha256(part.content).digest()).decode() + return f"{_BINARY_FIELD_PREFIX}{digest}>" + + +def _form_fields(fields: tuple[_MultipartPart, ...]) -> dict[str, str]: + """The ordinary field parts, flattened into the mapping the bundle format stores. A + name sent more than once takes an occurrence suffix instead of overwriting the + earlier value, so nothing an upload said is dropped from its key. The suffix is + escaped so a field literally named ``x[1]`` cannot collide with a second ``x``.""" + form: dict[str, str] = {} + for part in fields: + name = part.field_name.replace("[", "[[") + occurrence = 1 + while name in form: + name = f"{part.field_name.replace('[', '[[')}[{occurrence}]" + occurrence += 1 + form[name] = _field_value(part) + return form + + +def _file_identity(files: tuple[_MultipartPart, ...]) -> tuple[str | None, str | None, int | None]: + """Name, content digest, and total length for the uploaded file parts. + + The name is a structured list of every part's field name, filename, and declared + content type rather than a joined string, so a filename containing the separator + cannot be confused for a different split, and two parts that differ only in the type + they declare stay apart. It goes through the canonicalizer as one string, which is + why per-run markers inside a filename do not move the key in the multi-file case any + more than they do in the single-file one. + + The digest covers content only. A lone file keeps its own canonicalized digest; + several fold into one ordered digest, so parts arriving in a different order key + differently. Total length is recorded for a reader but deliberately kept out of the + key: it is the raw byte count, and keying on it would undo exactly the drift the + canonicalized digest exists to absorb.""" + if not files: + return None, None, None + names: Final = _JSON.dump_json( + [[part.field_name, part.filename, part.content_type] for part in files] + ).decode() + total: Final = sum(len(part.content) for part in files) + if len(files) == 1: + return names, _content_digest(files[0].content), total + folded: Final = _JSON.dump_json([_content_digest(part.content) for part in files]) + return names, hashlib.sha256(folded).hexdigest(), total + + +def _multipart_request( + method: str, path: str, params: dict[str, str], parts: tuple[_MultipartPart, ...] +) -> RecordedRequest: + """A multipart upload keyed by what it says rather than by its wire bytes: every + ordinary field, plus the identity of the uploaded file. The random per-request + boundary is envelope, never content, so it never reaches the digest.""" + form: Final = _form_fields(tuple(part for part in parts if not _is_file_part(part))) + file_name, file_sha256, file_bytes = _file_identity( + tuple(part for part in parts if _is_file_part(part)) + ) + return RecordedRequest( + method=method, + path=path, + headers={}, + params=params, + form=form, + file_name=file_name, + file_sha256=file_sha256, + file_bytes=file_bytes, + ) + + +def _opaque_request( + method: str, + path: str, + params: dict[str, str], + body: bytes, + digested: bytes, + file_name: str | None = None, +) -> RecordedRequest: + """A body kept out of the bundle and matched on its digest alone. ``digested`` is + what the digest runs over, which is the body itself unless something in it has to be + normalized away first.""" + return RecordedRequest( + method=method, + path=path, + headers={}, + params=params, + file_name=file_name, + file_sha256=_content_digest(digested), + file_bytes=len(body), + ) + + +def edge_request( + method: str, path: str, query: str, body: bytes | None, content_type: str = "" +) -> RecordedRequest: + """The identity replay matches on: the edge path (mount included), the query as + params, and the body as parsed JSON, as parsed multipart fields and file identity + when the content type declares an envelope, or as a content digest otherwise so + opaque uploads still match across runs. A multipart body that does not parse still + has its boundary normalized away, because that boundary is fresh every request and + would otherwise guarantee a miss.""" + params: Final = dict(parse_qsl(query, keep_blank_values=True)) + lowered_method: Final = method.lower() + if not body: + return RecordedRequest(method=lowered_method, path=path, headers={}, params=params) + boundary: Final = _multipart_boundary(content_type) + if boundary is not None: + parts = _multipart_parts(body, boundary) + if parts is not None: + return _multipart_request(lowered_method, path, params, parts) + return _opaque_request( + lowered_method, + path, + params, + body, + body.replace(b"--" + boundary.encode(), _BOUNDARY_PLACEHOLDER), + _UNPARSED_MULTIPART, ) - return RecordedRequest(method=method.lower(), path=path, headers={}, params=params, body=parsed) + try: + parsed: Final[JsonValue] = _JSON.validate_json(body) + except ValueError: + return _opaque_request(lowered_method, path, params, body, body) + return RecordedRequest( + method=lowered_method, path=path, headers={}, params=params, body=parsed + ) def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: @@ -351,7 +575,9 @@ def handle_edge_request( return _text_reply( 404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}" ) - request: Final = _edge_request(method, split.path, split.query, body) + request: Final = edge_request( + method, split.path, split.query, body, _header_value(headers, "content-type") + ) match backend: case RecordEdge(): return _handle_record( diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 4dab0aaa3fa..cd70ac45da6 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -56,7 +56,7 @@ def chat_override( json=ReliabilityChatBody( model=model, messages=[ChatMessage(role="user", content=content)], - max_tokens=16, + max_tokens=64, stream=stream, router_settings_override=override, ), diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py index 5b7d21c6ef7..fe2d924ae2c 100644 --- a/tests/e2e/router/test_reliability_fallbacks_e2e.py +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -49,7 +49,7 @@ class TestReliabilityFallbacks: resources.defer(lambda: client.proxy.delete_model(model_id)) resp = chat_override( - client.proxy, scoped_key, primary, "say hi", + client.proxy, scoped_key, primary, f"say hi {unique_marker()}", override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), ) _assert_served_by_fallback(resp) @@ -63,7 +63,7 @@ class TestReliabilityFallbacks: resources.defer(lambda: client.proxy.delete_model(model_id)) resp = chat_override( - client.proxy, scoped_key, primary, "say hi", + client.proxy, scoped_key, primary, f"say hi {unique_marker()}", override=RouterSettingsOverride(fallbacks=[{primary: ["gpt-5.5"]}]), ) _assert_served_by_fallback(resp) diff --git a/tests/e2e/test_provider_edge.py b/tests/e2e/test_provider_edge.py index 492eee57aaf..14a9fd53393 100644 --- a/tests/e2e/test_provider_edge.py +++ b/tests/e2e/test_provider_edge.py @@ -23,11 +23,13 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from typing import Final import pytest from pydantic import TypeAdapter from e2e_http import RawResponse, forward +from fixture_canonical import canonicalize from fixture_bundle import ( BundleRecorder, Interaction, @@ -46,6 +48,7 @@ from provider_edge import ( RecordEdge, ReplayEdge, ReplaySource, + edge_request, handle_edge_request, provider_edge_api_base, replay_leftover_error, @@ -53,8 +56,10 @@ from provider_edge import ( ) CHAT_PATH = "/openai/v1/chat/completions" +UPLOAD_PATH = "/openai/v1/files" REPLAY_MOUNTS = {"openai": "https://replay.invalid"} JSON_OBJECT = TypeAdapter(dict[str, object]) +BATCH_JSONL = b'{"custom_id":"one"}\n{"custom_id":"two"}\n' def json_object(body: bytes) -> dict[str, object]: @@ -164,6 +169,46 @@ def chat_body(prompt: str) -> bytes: return json.dumps({"model": "gpt", "messages": [{"role": "user", "content": prompt}]}).encode() +def multipart_body( + boundary: str, + fields: tuple[tuple[str, str], ...] = (), + files: tuple[tuple[str, str, bytes], ...] = (), +) -> bytes: + """One multipart/form-data body on the wire, exactly as ``requests`` writes it, with + the boundary under the caller's control instead of randomly generated.""" + parts = [ + f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n'.encode() + + value.encode() + for name, value in fields + ] + [ + ( + f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"; ' + f'filename="{filename}"\r\nContent-Type: application/octet-stream\r\n\r\n' + ).encode() + + content + for name, filename, content in files + ] + return b"\r\n".join(parts) + f"\r\n--{boundary}--\r\n".encode() + + +def upload_headers(boundary: str) -> dict[str, str]: + return { + "content-type": f"multipart/form-data; boundary={boundary}", + "authorization": "Bearer sk-upload-secret", + } + + +def record_upload(root: Path, body: bytes, boundary: str) -> None: + with fake_provider() as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + call_edge(edge, "POST", UPLOAD_PATH, body=body, headers=upload_headers(boundary)) + + +def replay_upload(root: Path, body: bytes, boundary: str) -> RawResponse: + with running_edge(ReplayEdge(source=replay_source(root)), REPLAY_MOUNTS) as edge: + return call_edge(edge, "POST", UPLOAD_PATH, body=body, headers=upload_headers(boundary)) + + class TestRecordMode: def test_forwards_to_the_provider_and_writes_one_interaction_file(self, tmp_path: Path) -> None: root = tmp_path / "bundle" @@ -328,6 +373,347 @@ class TestReplayMode: assert replayed.status_code == 200 +class TestMultipartIdentity: + """LIT-5974: a multipart upload is keyed by its parsed fields and file identity. + ``requests`` picks a fresh random boundary per request, so hashing the wire body + made every upload miss on replay; parsing the envelope keys the upload on what it + actually says, which is stable across runs and still separates real drift.""" + + def test_a_fresh_boundary_replays_the_same_upload(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorded = multipart_body( + "d0a1b2c3d4e5f60718293a4b5c6d7e8f", + fields=(("purpose", "batch"),), + files=(("file", "batch.jsonl", BATCH_JSONL),), + ) + record_upload(root, recorded, "d0a1b2c3d4e5f60718293a4b5c6d7e8f") + + rerun = multipart_body( + "ffffeeeeddddccccbbbbaaaa99998888", + fields=(("purpose", "batch"),), + files=(("file", "batch.jsonl", BATCH_JSONL),), + ) + assert rerun != recorded + replayed = replay_upload(root, rerun, "ffffeeeeddddccccbbbbaaaa99998888") + assert replayed.status_code == 200, replayed.body[:400] + + def test_the_stored_request_carries_fields_and_file_identity_but_no_secrets( + self, tmp_path: Path + ) -> None: + root = tmp_path / "bundle" + boundary = "0123456789abcdef0123456789abcdef" + record_upload( + root, + multipart_body( + boundary, + fields=(("purpose", "batch"),), + files=(("file", "batch.jsonl", BATCH_JSONL),), + ), + boundary, + ) + + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + interaction = Interaction.model_validate_json(raw) + assert interaction.request.form == {"purpose": "batch"} + assert interaction.request.file_name == json.dumps( + [["file", "batch.jsonl", "application/octet-stream"]], separators=(",", ":") + ) + assert interaction.request.file_bytes == len(BATCH_JSONL) + stored = interaction.request.model_dump_json() + assert boundary not in stored + assert "sk-upload-secret" not in stored + assert "custom_id" not in stored + + @pytest.mark.parametrize( + ("fields", "files"), + [ + pytest.param( + (("purpose", "batch"),), + (("file", "batch.jsonl", b'{"custom_id":"three"}\n'),), + id="file-content", + ), + pytest.param( + (("purpose", "batch"),), + (("file", "other.jsonl", BATCH_JSONL),), + id="file-name", + ), + pytest.param( + (("purpose", "fine-tune"),), + (("file", "batch.jsonl", BATCH_JSONL),), + id="form-field", + ), + pytest.param( + (("purpose", "batch"), ("purpose", "batch")), + (("file", "batch.jsonl", BATCH_JSONL),), + id="repeated-form-field", + ), + pytest.param( + (("purpose", "batch"),), + ( + ("file", "batch.jsonl", BATCH_JSONL), + ("mask", "mask.jsonl", BATCH_JSONL), + ), + id="extra-file-part", + ), + ], + ) + def test_a_structurally_different_upload_misses( + self, + tmp_path: Path, + fields: tuple[tuple[str, str], ...], + files: tuple[tuple[str, str, bytes], ...], + ) -> None: + root = tmp_path / "bundle" + record_upload( + root, + multipart_body( + "aaaaaaaabbbbbbbbccccccccdddddddd", + fields=(("purpose", "batch"),), + files=(("file", "batch.jsonl", BATCH_JSONL),), + ), + "aaaaaaaabbbbbbbbccccccccdddddddd", + ) + + drifted = replay_upload( + root, + multipart_body("11112222333344445555666677778888", fields=fields, files=files), + "11112222333344445555666677778888", + ) + assert drifted.status_code == REPLAY_MISS_STATUS + + def test_several_file_parts_separate_when_their_contents_swap(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + image, mask = b"image-bytes", b"mask-bytes" + record_upload( + root, + multipart_body( + "1a1a1a1a2b2b2b2b3c3c3c3c4d4d4d4d", + fields=(("prompt", "a cat"),), + files=(("image", "a.png", image), ("mask", "b.png", mask)), + ), + "1a1a1a1a2b2b2b2b3c3c3c3c4d4d4d4d", + ) + + swapped = replay_upload( + root, + multipart_body( + "5e5e5e5e6f6f6f6f7070707081818181", + fields=(("prompt", "a cat"),), + files=(("image", "a.png", mask), ("mask", "b.png", image)), + ), + "5e5e5e5e6f6f6f6f7070707081818181", + ) + assert swapped.status_code == REPLAY_MISS_STATUS + + same = replay_upload( + root, + multipart_body( + "9292929203030303a4a4a4a4b5b5b5b5", + fields=(("prompt", "a cat"),), + files=(("image", "a.png", image), ("mask", "b.png", mask)), + ), + "9292929203030303a4a4a4a4b5b5b5b5", + ) + assert same.status_code == 200, same.body[:400] + + def test_a_body_that_does_not_match_its_declared_boundary_stays_opaque( + self, tmp_path: Path + ) -> None: + root = tmp_path / "bundle" + opaque = b"custom_id one\ncustom_id two\n" + absent = "boundary-that-is-absent-from-the-body" + record_upload(root, opaque, absent) + + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + interaction = Interaction.model_validate_json(raw) + assert interaction.request.form is None + assert interaction.request.file_name == "" + assert interaction.request.file_bytes == len(opaque) + assert "custom_id" not in interaction.request.model_dump_json() + assert replay_upload(root, opaque, absent).status_code == 200 + + +def raw_multipart(boundary: str, *parts: tuple[str, bytes]) -> bytes: + """A body assembled from literal part headers, so a test can send the shapes a + well-formed helper cannot: a file part with no filename, a declared per-part content + type, a repeated or bracketed field name, or a non-UTF-8 value.""" + return ( + b"".join( + f"--{boundary}\r\n{head}\r\n\r\n".encode() + content + b"\r\n" + for head, content in parts + ) + + f"--{boundary}--\r\n".encode() + ) + + +def upload_key(body: bytes, boundary: str) -> str: + content_type: Final = f"multipart/form-data; boundary={boundary}" + return canonicalize(edge_request("POST", UPLOAD_PATH, "", body, content_type)).key + + +DISPOSITION = 'Content-Disposition: form-data; name="{name}"' +FILE_DISPOSITION = DISPOSITION + '; filename="{filename}"' + + +class TestMultipartIdentityEdges: + """The identity a multipart upload keys on, pinned against the ways two materially + different uploads could otherwise collapse onto one key. A collision here is the + dangerous failure: replay would answer one request with another's response.""" + + def test_a_declared_part_content_type_separates_otherwise_identical_uploads(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + as_json = raw_multipart( + boundary, + (FILE_DISPOSITION.format(name="file", filename="a") + "\r\nContent-Type: application/json", b"xy"), + ) + as_csv = raw_multipart( + boundary, + (FILE_DISPOSITION.format(name="file", filename="a") + "\r\nContent-Type: text/csv", b"xy"), + ) + + assert upload_key(as_json, boundary) != upload_key(as_csv, boundary) + + def test_a_file_part_without_a_filename_is_not_mistaken_for_a_plain_field(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + upload = raw_multipart( + boundary, + (DISPOSITION.format(name="file") + "\r\nContent-Type: application/octet-stream", b"CONTENT"), + ) + plain_field = raw_multipart(boundary, (DISPOSITION.format(name="file"), b"CONTENT")) + + request = edge_request( + "POST", UPLOAD_PATH, "", upload, f"multipart/form-data; boundary={boundary}" + ) + + assert upload_key(upload, boundary) != upload_key(plain_field, boundary) + assert request.form == {} + assert b"CONTENT".decode() not in request.model_dump_json() + + def test_a_filename_carrying_a_per_run_marker_keys_the_same_next_run(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + + def upload(marker: str) -> str: + body = raw_multipart( + boundary, + (FILE_DISPOSITION.format(name="one", filename=f"{marker}.jsonl"), b"first"), + (FILE_DISPOSITION.format(name="two", filename="steady.jsonl"), b"second"), + ) + return upload_key(body, boundary) + + assert upload("a1b2c3d4e5f6") == upload("0f9e8d7c6b5a") + + def test_a_separator_inside_a_filename_cannot_forge_a_different_split(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + colon_in_filename = raw_multipart( + boundary, (FILE_DISPOSITION.format(name="file", filename="a:b.jsonl"), b"same") + ) + colon_in_field = raw_multipart( + boundary, (FILE_DISPOSITION.format(name="file:a", filename="b.jsonl"), b"same") + ) + + assert upload_key(colon_in_filename, boundary) != upload_key(colon_in_field, boundary) + + def test_a_repeated_field_cannot_collide_with_a_literal_indexed_name(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + repeated = raw_multipart( + boundary, + (DISPOSITION.format(name="purpose"), b"x"), + (DISPOSITION.format(name="purpose"), b"y"), + ) + literal_index = raw_multipart( + boundary, + (DISPOSITION.format(name="purpose"), b"x"), + (DISPOSITION.format(name="purpose[1]"), b"y"), + ) + + assert upload_key(repeated, boundary) != upload_key(literal_index, boundary) + + def test_two_binary_field_values_of_one_length_stay_apart(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + first = raw_multipart(boundary, (DISPOSITION.format(name="blob"), b"\xff\xfe\xfd")) + second = raw_multipart(boundary, (DISPOSITION.format(name="blob"), b"\xf0\xf1\xf2")) + + assert upload_key(first, boundary) != upload_key(second, boundary) + + def test_a_secret_named_field_never_reaches_the_stored_request(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + body = raw_multipart( + boundary, + (DISPOSITION.format(name="openai_api_key"), b"sk-live-DEADBEEF-0123456789abcd"), + (DISPOSITION.format(name="purpose"), b"batch"), + ) + + request = edge_request( + "POST", UPLOAD_PATH, "", body, f"multipart/form-data; boundary={boundary}" + ) + + assert "sk-live-DEADBEEF-0123456789abcd" not in request.model_dump_json() + assert request.form == {"openai_api_key": "", "purpose": "batch"} + + def test_a_redacted_field_still_matches_the_live_request_that_carried_the_secret( + self, + ) -> None: + boundary = "0123456789abcdef0123456789abcdef" + + def upload(secret: str) -> str: + body = raw_multipart( + boundary, + (DISPOSITION.format(name="openai_api_key"), secret.encode()), + (DISPOSITION.format(name="purpose"), b"batch"), + ) + return upload_key(body, boundary) + + assert upload("sk-live-DEADBEEF-0123456789abcd") == upload("") + + def test_a_length_change_the_canonicalizer_absorbs_does_not_move_the_key(self) -> None: + boundary = "0123456789abcdef0123456789abcdef" + + def upload(created: str) -> str: + body = raw_multipart( + boundary, + ( + FILE_DISPOSITION.format(name="file", filename="batch.jsonl"), + b'{"created_at":"' + created.encode() + b'"}', + ), + ) + return upload_key(body, boundary) + + assert upload("2026-08-21T02:08:19Z") == upload("2026-08-21T02:08:19.123456Z") + + @pytest.mark.parametrize( + "content_type", + [ + pytest.param("multipart/form-data; myboundary=zzz; boundary={boundary}", id="lookalike-parameter"), + pytest.param("multipart/form-data; BOUNDARY={boundary}", id="uppercase-parameter"), + ], + ) + def test_the_boundary_parameter_is_read_the_way_the_client_meant_it( + self, content_type: str + ) -> None: + boundary = "0123456789abcdef0123456789abcdef" + body = raw_multipart( + boundary, (FILE_DISPOSITION.format(name="file", filename="batch.jsonl"), BATCH_JSONL) + ) + + request = edge_request( + "POST", UPLOAD_PATH, "", body, content_type.format(boundary=boundary) + ) + + assert request.form == {} + assert request.file_name is not None + assert "batch.jsonl" in request.file_name + + def test_an_empty_declared_boundary_falls_back_instead_of_splitting_on_dashes(self) -> None: + body = b'--\r\nContent-Disposition: form-data; name="a"\r\n\r\nvalue\r\n----\r\n' + + request = edge_request( + "POST", UPLOAD_PATH, "", body, 'multipart/form-data; boundary=""' + ) + + assert request.form is None + assert request.file_sha256 is not None + + class TestReplayLeftover: def test_partially_consumed_recording_names_the_leftover(self, tmp_path: Path) -> None: root = tmp_path / "bundle" diff --git a/tests/enterprise/conftest.py b/tests/enterprise/conftest.py index f23a5664f83..4c95f967bc4 100644 --- a/tests/enterprise/conftest.py +++ b/tests/enterprise/conftest.py @@ -3,13 +3,9 @@ import asyncio import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm @@ -31,9 +27,6 @@ def setup_and_teardown(): This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ curr_dir = os.getcwd() # Get the current working directory - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path from litellm import Router @@ -41,8 +34,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index b6c9cd0294b..05886e4b7f6 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import logging diff --git a/tests/enterprise/litellm_enterprise/integrations/test_custom_guardrail.py b/tests/enterprise/litellm_enterprise/integrations/test_custom_guardrail.py index 8a29e5c1ced..c6e48061698 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_custom_guardrail.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_custom_guardrail.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks, Mode diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py index 6c4a008c823..7315f2b9881 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py @@ -3,15 +3,10 @@ Mock prometheus unit tests, these don't rely on LLM API calls """ import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import patch diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py index f5c39fb86ae..28fd03daf37 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py @@ -9,16 +9,12 @@ except Exception: PrometheusLogger = None import asyncio -import sys from dotenv import load_dotenv load_dotenv() import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from unittest.mock import MagicMock import pytest diff --git a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py b/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py index f90ac9abb7d..265abbe95cf 100644 --- a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py +++ b/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py @@ -1,10 +1,6 @@ import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import pytest from fastapi import HTTPException diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py index e5074c44210..4f44a4adeed 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py @@ -2,13 +2,10 @@ Test the /guardrails/apply_guardrail endpoint """ -import os -import sys from unittest.mock import AsyncMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from fastapi import HTTPException diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index 6b6b5d768dd..463076229e9 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -2,13 +2,10 @@ Test the Bedrock guardrail apply_guardrail functionality """ -import os -import sys from unittest.mock import AsyncMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index ed6735a7126..34a0d1c9f7a 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -1,5 +1,4 @@ import os -import sys import traceback from litellm._uuid import uuid from unittest import mock @@ -10,7 +9,6 @@ from fastapi import Request load_dotenv() import time -sys.path.insert(0, os.path.abspath("../..")) import logging import pytest diff --git a/tests/guardrails_tests/conftest.py b/tests/guardrails_tests/conftest.py index f2f65645c3d..6eeb0924341 100644 --- a/tests/guardrails_tests/conftest.py +++ b/tests/guardrails_tests/conftest.py @@ -7,13 +7,9 @@ import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from tests._vcr_conftest_common import ( # noqa: E402,F401 @@ -122,7 +118,6 @@ def setup_and_teardown(): Module-scoped setup. Reloads litellm only in single-process mode (skipped under xdist to avoid cross-worker interference). """ - sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 8b22cc0eb73..43d088268eb 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -1,9 +1,6 @@ -import sys -import os import io, asyncio import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrail, diff --git a/tests/guardrails_tests/test_custom_guardrail.py b/tests/guardrails_tests/test_custom_guardrail.py index 9d7efeecdca..3c88ed53cd3 100644 --- a/tests/guardrails_tests/test_custom_guardrail.py +++ b/tests/guardrails_tests/test_custom_guardrail.py @@ -3,11 +3,8 @@ Test custom guardrail + unit tests for guardrails """ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/guardrails_tests/test_deepkeep_guardrails.py b/tests/guardrails_tests/test_deepkeep_guardrails.py index d06610f3f4c..74bdea2e0b9 100644 --- a/tests/guardrails_tests/test_deepkeep_guardrails.py +++ b/tests/guardrails_tests/test_deepkeep_guardrails.py @@ -1,5 +1,4 @@ import os -import sys from unittest.mock import patch, AsyncMock from httpx import Response, Request @@ -13,9 +12,6 @@ from litellm.proxy.guardrails.guardrail_hooks.deepkeep.deepkeep import ( ) from litellm.exceptions import GuardrailRaisedException -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 diff --git a/tests/guardrails_tests/test_dynamoai_guardrails.py b/tests/guardrails_tests/test_dynamoai_guardrails.py index 6f0ea00165b..4f56f7cd444 100644 --- a/tests/guardrails_tests/test_dynamoai_guardrails.py +++ b/tests/guardrails_tests/test_dynamoai_guardrails.py @@ -2,11 +2,8 @@ Test DynamoAI Guardrails integration """ -import sys -import os import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.guardrails.guardrail_hooks.dynamoai import DynamoAIGuardrails from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/guardrails_tests/test_eu_ai_act_article5.py b/tests/guardrails_tests/test_eu_ai_act_article5.py index f7384667481..d17e56c7450 100644 --- a/tests/guardrails_tests/test_eu_ai_act_article5.py +++ b/tests/guardrails_tests/test_eu_ai_act_article5.py @@ -8,11 +8,9 @@ Tests 40 different sentences to validate the conditional matching logic: - identifier or block word alone should ALLOW """ -import sys import os import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, @@ -162,7 +160,6 @@ def content_filter_guardrail(): """Initialize content filter guardrail with EU AI Act Article 5 template.""" # Get absolute path to the policy template - import os content_filter_dir = os.path.join( os.path.dirname(__file__), diff --git a/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py b/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py index 221ca5aa6e6..cfc59030076 100644 --- a/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py +++ b/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py @@ -7,11 +7,9 @@ Tests the exact 3 scenarios requested: 3. Request 3: Safe query in French that should pass (allowed) """ -import sys import os import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, diff --git a/tests/guardrails_tests/test_guardrail_load_balancing.py b/tests/guardrails_tests/test_guardrail_load_balancing.py index 4f71f83c433..2e71d2c99a3 100644 --- a/tests/guardrails_tests/test_guardrail_load_balancing.py +++ b/tests/guardrails_tests/test_guardrail_load_balancing.py @@ -2,11 +2,8 @@ Test guardrail load balancing through the Router and ProxyLogging. """ -import os -import sys from unittest.mock import MagicMock, patch, AsyncMock -sys.path.insert(0, os.path.abspath("../..")) import litellm import pytest diff --git a/tests/guardrails_tests/test_guardrails_config.py b/tests/guardrails_tests/test_guardrails_config.py index aaacb607261..5160954b0eb 100644 --- a/tests/guardrails_tests/test_guardrails_config.py +++ b/tests/guardrails_tests/test_guardrails_config.py @@ -2,8 +2,6 @@ ## Unit Tests for guardrails config import asyncio import inspect -import os -import sys import time import traceback from litellm._uuid import uuid @@ -15,7 +13,6 @@ from pydantic import BaseModel import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging -sys.path.insert(0, os.path.abspath("../..")) from typing import Any, List, Literal, Optional, Tuple, Union from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/guardrails_tests/test_javelin_guardrails.py b/tests/guardrails_tests/test_javelin_guardrails.py index 62655a3c077..a2e7747d657 100644 --- a/tests/guardrails_tests/test_javelin_guardrails.py +++ b/tests/guardrails_tests/test_javelin_guardrails.py @@ -1,10 +1,7 @@ -import sys -import os import pytest from unittest.mock import AsyncMock, patch from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.guardrails.guardrail_hooks.javelin import JavelinGuardrail import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/guardrails_tests/test_lakera_v2.py b/tests/guardrails_tests/test_lakera_v2.py index 74e19350192..a71759862b2 100644 --- a/tests/guardrails_tests/test_lakera_v2.py +++ b/tests/guardrails_tests/test_lakera_v2.py @@ -1,12 +1,9 @@ -import sys -import os import io, asyncio import pytest import time from litellm import mock_completion from unittest.mock import MagicMock, AsyncMock, patch -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import LakeraAIGuardrail from litellm.types.guardrails import PiiEntityType, PiiAction diff --git a/tests/guardrails_tests/test_lasso_guardrails.py b/tests/guardrails_tests/test_lasso_guardrails.py index 75b571e236b..fd585623744 100644 --- a/tests/guardrails_tests/test_lasso_guardrails.py +++ b/tests/guardrails_tests/test_lasso_guardrails.py @@ -1,5 +1,4 @@ import os -import sys from fastapi.exceptions import HTTPException from unittest.mock import patch from httpx import Response, Request @@ -14,9 +13,6 @@ from litellm.proxy.guardrails.guardrail_hooks.lasso.lasso import ( LassoGuardrailAPIError, ) -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 diff --git a/tests/guardrails_tests/test_presidio_pii.py b/tests/guardrails_tests/test_presidio_pii.py index edc63bd9419..b3b2a790ba8 100644 --- a/tests/guardrails_tests/test_presidio_pii.py +++ b/tests/guardrails_tests/test_presidio_pii.py @@ -1,10 +1,8 @@ -import sys import os import pytest from litellm import mock_completion from unittest.mock import patch -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, diff --git a/tests/guardrails_tests/test_semantic_guard.py b/tests/guardrails_tests/test_semantic_guard.py index c9f4a902895..92c55507568 100644 --- a/tests/guardrails_tests/test_semantic_guard.py +++ b/tests/guardrails_tests/test_semantic_guard.py @@ -3,9 +3,7 @@ Tests for the Semantic Guard guardrail — embedding-based prompt injection dete """ import os -import sys -sys.path.insert(0, os.path.abspath("../..")) from unittest.mock import MagicMock diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py index e587d666a79..385fee93ab4 100644 --- a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -10,11 +10,9 @@ for Singapore financial institutions: 5. sg_mas_model_security — Adversarial attacks on financial AI """ -import sys import os import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py index 42c3a15f9f6..1e8b8a48b85 100644 --- a/tests/guardrails_tests/test_sg_pdpa_guardrails.py +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -15,11 +15,9 @@ Each sub-guardrail validates: - identifier or block word alone → ALLOW (no match) """ -import sys import os import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, diff --git a/tests/guardrails_tests/test_tracing_guardrails.py b/tests/guardrails_tests/test_tracing_guardrails.py index 46f4f3e6e9b..bd8b7bad33f 100644 --- a/tests/guardrails_tests/test_tracing_guardrails.py +++ b/tests/guardrails_tests/test_tracing_guardrails.py @@ -1,4 +1,3 @@ -import sys import os import io, asyncio import json @@ -7,7 +6,6 @@ import time from litellm import mock_completion from unittest.mock import MagicMock, AsyncMock, patch -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, diff --git a/tests/image_gen_tests/base_image_generation_test.py b/tests/image_gen_tests/base_image_generation_test.py index ab46bd36feb..c50b09d329c 100644 --- a/tests/image_gen_tests/base_image_generation_test.py +++ b/tests/image_gen_tests/base_image_generation_test.py @@ -2,14 +2,9 @@ import asyncio import httpx import json import pytest -import sys from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, Mock, patch -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/image_gen_tests/conftest.py b/tests/image_gen_tests/conftest.py index 9f808c11161..7e9a5c0d629 100644 --- a/tests/image_gen_tests/conftest.py +++ b/tests/image_gen_tests/conftest.py @@ -1,12 +1,7 @@ import asyncio -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402,F401 diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index c4d0f5fc773..1be3ca0745d 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -1,14 +1,9 @@ import logging -import os -import sys import traceback from dotenv import load_dotenv from openai.types.image import Image -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( AmazonNovaCanvasConfig, @@ -18,13 +13,9 @@ logging.basicConfig(level=logging.DEBUG) load_dotenv() import asyncio -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest from litellm.llms.bedrock.image_generation.cost_calculator import cost_calculator from litellm.types.utils import ImageResponse, ImageObject -import os import litellm from litellm.llms.bedrock.image_generation.amazon_stability3_transformation import ( diff --git a/tests/image_gen_tests/test_fal_ai_image_generation.py b/tests/image_gen_tests/test_fal_ai_image_generation.py index 23032e44ded..d33f2c4262e 100644 --- a/tests/image_gen_tests/test_fal_ai_image_generation.py +++ b/tests/image_gen_tests/test_fal_ai_image_generation.py @@ -1,11 +1,8 @@ import asyncio -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import aimage_generation diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index ca8ec3bbe32..0c2f57066e8 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -1,6 +1,5 @@ import logging import os -import sys import traceback import asyncio from typing import Optional @@ -11,9 +10,6 @@ from unittest.mock import patch, AsyncMock import json from abc import ABC, abstractmethod -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.utils import ImageResponse diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 9047557c493..02cee2e8a00 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -3,14 +3,10 @@ import logging import os -import sys import traceback from unittest.mock import AsyncMock, MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from dotenv import load_dotenv from openai.types.image import Image @@ -19,7 +15,6 @@ from litellm.caching import InMemoryCache logging.basicConfig(level=logging.DEBUG) load_dotenv() import asyncio -import os import pytest import litellm diff --git a/tests/image_gen_tests/test_image_variation.py b/tests/image_gen_tests/test_image_variation.py index 301835057a7..b566385bb8a 100644 --- a/tests/image_gen_tests/test_image_variation.py +++ b/tests/image_gen_tests/test_image_variation.py @@ -2,14 +2,9 @@ ## This tests the litellm support for the openai /generations endpoint import logging -import os -import sys import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from dotenv import load_dotenv from openai.types.image import Image @@ -18,7 +13,6 @@ from litellm.caching import InMemoryCache logging.basicConfig(level=logging.DEBUG) load_dotenv() import asyncio -import os import pytest import litellm diff --git a/tests/image_gen_tests/test_xinference.py b/tests/image_gen_tests/test_xinference.py index 6dd56daf193..3dc4fee85da 100644 --- a/tests/image_gen_tests/test_xinference.py +++ b/tests/image_gen_tests/test_xinference.py @@ -1,14 +1,9 @@ import logging -import os -import sys import traceback import pytest import json from unittest.mock import Mock, patch, AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.types.utils import ImageObject diff --git a/tests/integration/test_oci_integration.py b/tests/integration/test_oci_integration.py index 94b8930bce8..231a3bd8445 100644 --- a/tests/integration/test_oci_integration.py +++ b/tests/integration/test_oci_integration.py @@ -20,12 +20,10 @@ Run only these tests: import math import os -import sys from typing import NamedTuple, Optional import pytest -sys.path.insert(0, os.path.abspath("../..")) # --------------------------------------------------------------------------- # Fixtures / helpers diff --git a/tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py deleted file mode 100644 index 2a8768df722..00000000000 --- a/tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ /dev/null @@ -1,1268 +0,0 @@ -"""Tests for MCP OAuth discoverable endpoints""" - -import pytest -from fastapi import HTTPException -from unittest.mock import AsyncMock, MagicMock, patch - -TRUSTED_PROXY_IP = "10.0.0.5" -TRUSTED_PROXY_RANGES = ["10.0.0.0/8"] - - -def set_request_from_trusted_proxy(mock_request): - mock_request.client = MagicMock() - mock_request.client.host = TRUSTED_PROXY_IP - - -@pytest.fixture -def trusted_proxy_origin_headers(): - with ( - patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.is_request_from_trusted_proxy", - return_value=True, - ), - patch( - "litellm.proxy._experimental.mcp_server.oauth_utils.IPAddressUtils.is_request_from_trusted_proxy", - return_value=True, - ), - ): - yield - - -@pytest.mark.asyncio -async def test_authorize_endpoint_includes_response_type(): - """Test that authorize endpoint includes response_type=code parameter (fixes #15684)""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - authorize, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_oauth_server", - name="test_oauth", - server_name="test_oauth", - alias="test_oauth", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://litellm.example.com/" - mock_request.headers = {} - - # Mock the encryption functions to avoid needing a signing key - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" - ) as mock_encrypt: - mock_encrypt.return_value = "mocked_encrypted_state" - - # Call authorize endpoint - response = await authorize( - request=mock_request, - client_id="test_client_id", - mcp_server_name="test_oauth", - redirect_uri="http://127.0.0.1:60108/callback", - state="test_state", - ) - - # Verify response is a redirect - assert response.status_code == 307 # FastAPI RedirectResponse default - - # Verify response_type is in the redirect URL - assert "response_type=code" in response.headers["location"] - assert "https://provider.com/oauth/authorize" in response.headers["location"] - assert "client_id=test_client_id" in response.headers["location"] - assert "scope=read+write" in response.headers["location"] - - -@pytest.mark.asyncio -async def test_authorize_endpoint_forwards_pkce_parameters(): - """Test that authorize endpoint forwards PKCE parameters (code_challenge and code_challenge_method)""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - authorize, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server (simulating Google OAuth) - oauth2_server = MCPServer( - server_id="google_mcp", - name="google_mcp", - server_name="google_mcp", - alias="google_mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="669428968603-test.apps.googleusercontent.com", - client_secret="GOCSPX-test_secret", - authorization_url="https://accounts.google.com/o/oauth2/v2/auth", - token_url="https://oauth2.googleapis.com/token", - scopes=["https://www.googleapis.com/auth/drive", "openid", "email"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://litellm-proxy.example.com/" - mock_request.headers = {} - - # Mock the encryption function - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" - ) as mock_encrypt: - mock_encrypt.return_value = "mocked_encrypted_state_with_pkce" - - # Call authorize endpoint with PKCE parameters - response = await authorize( - request=mock_request, - client_id="669428968603-test.apps.googleusercontent.com", - mcp_server_name="google_mcp", - redirect_uri="http://localhost:60108/callback", - state="test_client_state", - code_challenge="x6YH_qgwbvOzbsHDuL1sW9gYkR9-gObUiIB5RkPwxDk", - code_challenge_method="S256", - ) - - # Verify response is a redirect - assert response.status_code == 307 - - # Verify PKCE parameters are included in the redirect URL - location = response.headers["location"] - assert "https://accounts.google.com/o/oauth2/v2/auth" in location - assert "code_challenge=x6YH_qgwbvOzbsHDuL1sW9gYkR9-gObUiIB5RkPwxDk" in location - assert "code_challenge_method=S256" in location - assert "client_id=669428968603-test.apps.googleusercontent.com" in location - assert "response_type=code" in location - - -@pytest.mark.asyncio -async def test_token_endpoint_forwards_code_verifier(): - """Test that token endpoint forwards code_verifier for PKCE flow""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - token_endpoint, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="google_mcp", - name="google_mcp", - server_name="google_mcp", - alias="google_mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="669428968603-test.apps.googleusercontent.com", - client_secret="GOCSPX-test_secret", - authorization_url="https://accounts.google.com/o/oauth2/v2/auth", - token_url="https://oauth2.googleapis.com/token", - scopes=["https://www.googleapis.com/auth/drive", "openid", "email"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://litellm-proxy.example.com/" - mock_request.headers = {} - - # Mock httpx client response - mock_response = MagicMock() - mock_response.json.return_value = { - "access_token": "ya29.test_access_token", - "token_type": "Bearer", - "expires_in": 3599, - "scope": "openid email https://www.googleapis.com/auth/drive", - } - mock_response.raise_for_status = MagicMock() - - # Mock the async httpx client with AsyncMock for async methods - from unittest.mock import AsyncMock - - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" - ) as mock_get_client: - mock_async_client = MagicMock() - # Use AsyncMock for the async post method - mock_async_client.post = AsyncMock(return_value=mock_response) - mock_get_client.return_value = mock_async_client - - # Call token endpoint with code_verifier - response = await token_endpoint( - request=mock_request, - grant_type="authorization_code", - code="4/test_authorization_code", - redirect_uri="http://localhost:60108/callback", - client_id="669428968603-test.apps.googleusercontent.com", - mcp_server_name="google_mcp", - client_secret="GOCSPX-test_secret", - code_verifier="test_code_verifier_from_client", - ) - - # Verify that the token endpoint was called with code_verifier - mock_async_client.post.assert_called_once() - call_args = mock_async_client.post.call_args - - # Check the data parameter includes code_verifier - assert call_args[1]["data"]["code_verifier"] == "test_code_verifier_from_client" - assert call_args[1]["data"]["code"] == "4/test_authorization_code" - assert ( - call_args[1]["data"]["client_id"] - == "669428968603-test.apps.googleusercontent.com" - ) - assert call_args[1]["data"]["client_secret"] == "GOCSPX-test_secret" - assert call_args[1]["data"]["grant_type"] == "authorization_code" - - # Verify response - response_data = response.body - import json - - token_data = json.loads(response_data) - assert token_data["access_token"] == "ya29.test_access_token" - assert token_data["token_type"] == "Bearer" - - -@pytest.mark.asyncio -async def test_register_client_without_mcp_server_name_returns_dummy(): - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - register_client, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - global_mcp_server_manager.registry.clear() - - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://proxy.litellm.example/" - mock_request.headers = {} - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._read_request_body", - new=AsyncMock(return_value={}), - ): - result = await register_client(request=mock_request) - - assert result == { - "client_id": "dummy_client", - "client_secret": "dummy", - "redirect_uris": ["https://proxy.litellm.example/callback"], - } - - -@pytest.mark.asyncio -async def test_register_client_returns_existing_server_credentials(): - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - register_client, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - global_mcp_server_manager.registry.clear() - oauth2_server = MCPServer( - server_id="stored_server", - name="stored_server", - server_name="stored_server", - alias="stored_server", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="existing-client", - client_secret="existing-secret", - authorization_url="https://provider.example/oauth/authorize", - token_url="https://provider.example/oauth/token", - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://proxy.litellm.example/" - mock_request.headers = {} - - try: - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._read_request_body", - new=AsyncMock(return_value={}), - ): - result = await register_client( - request=mock_request, mcp_server_name=oauth2_server.server_name - ) - finally: - global_mcp_server_manager.registry.clear() - - assert result == { - "client_id": "stored_server", - "client_secret": "dummy", - "redirect_uris": ["https://proxy.litellm.example/callback"], - } - - -@pytest.mark.asyncio -async def test_register_client_remote_registration_success(): - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - register_client, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - global_mcp_server_manager.registry.clear() - oauth2_server = MCPServer( - server_id="remote_server", - name="remote_server", - server_name="remote_server", - alias="remote_server", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id=None, - client_secret=None, - authorization_url="https://provider.example/oauth/authorize", - token_url="https://provider.example/oauth/token", - registration_url="https://provider.example/oauth/register", - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://proxy.litellm.example/" - mock_request.headers = {} - - request_payload = { - "client_name": "Litellm Proxy", - "grant_types": ["authorization_code", "refresh_token"], - "response_types": ["code"], - "token_endpoint_auth_method": "client_secret_post", - } - - mock_response = MagicMock() - mock_response.json.return_value = { - "client_id": "generated-client", - "client_secret": "generated-secret", - } - mock_response.raise_for_status = MagicMock() - mock_async_client = MagicMock() - mock_async_client.post = AsyncMock(return_value=mock_response) - - try: - with ( - patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._read_request_body", - new=AsyncMock(return_value=request_payload), - ), - patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", - return_value=mock_async_client, - ), - ): - response = await register_client( - request=mock_request, mcp_server_name=oauth2_server.server_name - ) - finally: - global_mcp_server_manager.registry.clear() - - import json - - assert response.status_code == 200 - payload = json.loads(response.body.decode("utf-8")) - assert payload == mock_response.json.return_value - - mock_async_client.post.assert_called_once() - call_args = mock_async_client.post.call_args - assert call_args.args[0] == oauth2_server.registration_url - assert call_args.kwargs["headers"] == { - "Content-Type": "application/json", - "Accept": "application/json", - } - assert call_args.kwargs["json"]["redirect_uris"] == [ - "https://proxy.litellm.example/callback" - ] - assert call_args.kwargs["json"]["grant_types"] == request_payload["grant_types"] - assert ( - call_args.kwargs["json"]["token_endpoint_auth_method"] - == request_payload["token_endpoint_auth_method"] - ) - - -@pytest.mark.asyncio -async def test_authorize_endpoint_respects_x_forwarded_proto( - trusted_proxy_origin_headers, -): - """Test that authorize endpoint uses X-Forwarded-Proto header to construct correct redirect_uri""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - authorize, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_oauth_server", - name="test_oauth", - server_name="test_oauth", - alias="test_oauth", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request with http base_url but X-Forwarded-Proto: https - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://litellm.example.com/" # HTTP - mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy - set_request_from_trusted_proxy(mock_request) - - # Mock the encryption functions - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" - ) as mock_encrypt: - mock_encrypt.return_value = "mocked_encrypted_state" - - # Call authorize endpoint - response = await authorize( - request=mock_request, - client_id="test_client_id", - mcp_server_name="test_oauth", - redirect_uri="http://127.0.0.1:60108/callback", - state="test_state", - ) - - # Verify redirect URL uses HTTPS in the redirect_uri parameter - location = response.headers["location"] - - # The redirect_uri parameter sent to the OAuth provider should use HTTPS - assert ( - "redirect_uri=https%3A%2F%2Flitellm.example.com%2Fcallback" in location - or "redirect_uri=https://litellm.example.com/callback" in location - ) - - -@pytest.mark.asyncio -async def test_token_endpoint_respects_x_forwarded_proto( - trusted_proxy_origin_headers, -): - """Test that token endpoint uses X-Forwarded-Proto header for redirect_uri""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - token_endpoint, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="google_mcp", - name="google_mcp", - server_name="google_mcp", - alias="google_mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_secret", - authorization_url="https://accounts.google.com/o/oauth2/v2/auth", - token_url="https://oauth2.googleapis.com/token", - scopes=["openid", "email"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request with http base_url but X-Forwarded-Proto: https - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://litellm-proxy.example.com/" # HTTP - mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy - set_request_from_trusted_proxy(mock_request) - - # Mock httpx client response - mock_response = MagicMock() - mock_response.json.return_value = { - "access_token": "test_token", - "token_type": "Bearer", - "expires_in": 3599, - } - mock_response.raise_for_status = MagicMock() - - # Mock the async httpx client - mock_async_client = MagicMock() - mock_async_client.post = AsyncMock(return_value=mock_response) - - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" - ) as mock_get_client: - mock_get_client.return_value = mock_async_client - - # Call token endpoint - await token_endpoint( - request=mock_request, - grant_type="authorization_code", - code="test_code", - redirect_uri="http://localhost:60108/callback", - client_id="test_client_id", - mcp_server_name="google_mcp", - client_secret="test_secret", - ) - - # Verify that the redirect_uri sent to the provider uses HTTPS - call_args = mock_async_client.post.call_args - assert ( - call_args[1]["data"]["redirect_uri"] - == "https://litellm-proxy.example.com/callback" - ) - - -@pytest.mark.asyncio -async def test_oauth_protected_resource_standard_pattern(): - """Test that oauth_protected_resource_mcp_standard returns standard MCP URL pattern (/mcp/{server_name})""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - oauth_protected_resource_mcp_standard, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_server", - name="test_server", - server_name="test_server", - alias="test_server", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://litellm.example.com/" - mock_request.headers = {} - - # Call the standard pattern endpoint - response = await oauth_protected_resource_mcp_standard( - request=mock_request, - mcp_server_name="test_server", - ) - - # Verify response uses standard MCP pattern: /mcp/{server_name} - assert response["resource"] == "https://litellm.example.com/mcp/test_server" - assert ( - response["authorization_servers"][0] - == "https://litellm.example.com/test_server" - ) - assert response["scopes_supported"] == oauth2_server.scopes - - -@pytest.mark.asyncio -async def test_oauth_protected_resource_legacy_pattern(): - """Test that oauth_protected_resource_mcp returns legacy URL pattern (/{server_name}/mcp)""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - oauth_protected_resource_mcp, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_server", - name="test_server", - server_name="test_server", - alias="test_server", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://litellm.example.com/" - mock_request.headers = {} - - # Call the legacy pattern endpoint - response = await oauth_protected_resource_mcp( - request=mock_request, - mcp_server_name="test_server", - ) - - # Verify response uses legacy pattern: /{server_name}/mcp - assert response["resource"] == "https://litellm.example.com/test_server/mcp" - assert ( - response["authorization_servers"][0] - == "https://litellm.example.com/test_server" - ) - assert response["scopes_supported"] == oauth2_server.scopes - - -@pytest.mark.asyncio -async def test_oauth_protected_resource_respects_x_forwarded_proto( - trusted_proxy_origin_headers, -): - """Test that oauth_protected_resource_mcp uses X-Forwarded-Proto for URLs""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - oauth_protected_resource_mcp, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_oauth_server", - name="test_oauth", - server_name="test_oauth", - alias="test_oauth", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request with http base_url but X-Forwarded-Proto: https - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://litellm.example.com/" # HTTP - mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy - set_request_from_trusted_proxy(mock_request) - - # Call the endpoint - response = await oauth_protected_resource_mcp( - request=mock_request, - mcp_server_name="test_oauth", - ) - - # Verify response uses HTTPS URLs - assert response["authorization_servers"][0].startswith( - "https://litellm.example.com/" - ) - assert response["scopes_supported"] == oauth2_server.scopes - - -@pytest.mark.asyncio -async def test_oauth_authorization_server_respects_x_forwarded_proto( - trusted_proxy_origin_headers, -): - """Test that oauth_authorization_server_mcp uses X-Forwarded-Proto for URLs""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - oauth_authorization_server_mcp, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_oauth_server", - name="test_oauth", - server_name="test_oauth", - alias="test_oauth", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request with http base_url but X-Forwarded-Proto: https - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://litellm.example.com/" # HTTP - mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy - set_request_from_trusted_proxy(mock_request) - - # Call the endpoint - response = await oauth_authorization_server_mcp( - request=mock_request, - mcp_server_name="test_oauth", - ) - - # Verify response uses HTTPS URLs - assert response["authorization_endpoint"].startswith("https://litellm.example.com/") - assert response["token_endpoint"].startswith("https://litellm.example.com/") - assert response["registration_endpoint"].startswith("https://litellm.example.com/") - assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] - assert response["scopes_supported"] == oauth2_server.scopes - - -@pytest.mark.asyncio -async def test_register_client_respects_x_forwarded_proto( - trusted_proxy_origin_headers, -): - """Test that register_client uses X-Forwarded-Proto for redirect_uris""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - register_client, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - global_mcp_server_manager.registry.clear() - - # Mock request with http base_url but X-Forwarded-Proto: https - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://proxy.litellm.example/" # HTTP - mock_request.headers = {"X-Forwarded-Proto": "https"} # Behind HTTPS proxy - set_request_from_trusted_proxy(mock_request) - - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints._read_request_body", - new=AsyncMock(return_value={}), - ): - result = await register_client(request=mock_request) - - # Verify the redirect_uris use HTTPS - assert result == { - "client_id": "dummy_client", - "client_secret": "dummy", - "redirect_uris": ["https://proxy.litellm.example/callback"], - } - - -@pytest.mark.asyncio -async def test_authorize_endpoint_respects_x_forwarded_host( - trusted_proxy_origin_headers, -): - """Test that authorize endpoint uses X-Forwarded-Host and X-Forwarded-Proto to construct correct redirect_uri""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - authorize, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="test_oauth_server", - name="test_oauth", - server_name="test_oauth", - alias="test_oauth", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_client_secret", - authorization_url="https://provider.com/oauth/authorize", - token_url="https://provider.com/oauth/token", - scopes=["read", "write"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request simulating nginx proxy: - # Internal: http://localhost:8888/github/mcp - # External: https://proxy.example.com/github/mcp - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://localhost:8888/github/mcp" - mock_request.headers = { - "X-Forwarded-Proto": "https", - "X-Forwarded-Host": "proxy.example.com", - } - set_request_from_trusted_proxy(mock_request) - - # Mock the encryption functions - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper" - ) as mock_encrypt: - mock_encrypt.return_value = "mocked_encrypted_state" - - # Call authorize endpoint - response = await authorize( - request=mock_request, - client_id="test_client_id", - mcp_server_name="test_oauth", - redirect_uri="http://127.0.0.1:60108/callback", - state="test_state", - ) - - # Verify redirect URL uses the forwarded host and scheme - location = response.headers["location"] - - # The redirect_uri parameter should use the external URL - assert ( - "redirect_uri=https%3A%2F%2Fproxy.example.com%2Fgithub%2Fmcp%2Fcallback" - in location - or "redirect_uri=https://proxy.example.com/github/mcp/callback" in location - ) - - -@pytest.mark.asyncio -async def test_token_endpoint_respects_x_forwarded_host( - trusted_proxy_origin_headers, -): - """Test that token endpoint uses X-Forwarded-Host and X-Forwarded-Proto for redirect_uri""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - token_endpoint, - ) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) - from litellm.types.mcp import MCPAuth - from litellm.types.mcp_server.mcp_server_manager import MCPServer - from litellm.proxy._types import MCPTransport - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Clear registry - global_mcp_server_manager.registry.clear() - - # Create mock OAuth2 server - oauth2_server = MCPServer( - server_id="google_mcp", - name="google_mcp", - server_name="google_mcp", - alias="google_mcp", - transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, - client_id="test_client_id", - client_secret="test_secret", - authorization_url="https://accounts.google.com/o/oauth2/v2/auth", - token_url="https://oauth2.googleapis.com/token", - scopes=["openid", "email"], - ) - global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server - - # Mock request simulating nginx proxy without port in host - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://localhost:8888/github/mcp" - mock_request.headers = { - "X-Forwarded-Proto": "https", - "X-Forwarded-Host": "proxy.example.com", - } - set_request_from_trusted_proxy(mock_request) - - # Mock httpx client response - mock_response = MagicMock() - mock_response.json.return_value = { - "access_token": "test_token", - "token_type": "Bearer", - "expires_in": 3599, - } - mock_response.raise_for_status = MagicMock() - - # Mock the async httpx client - mock_async_client = MagicMock() - mock_async_client.post = AsyncMock(return_value=mock_response) - - with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" - ) as mock_get_client: - mock_get_client.return_value = mock_async_client - - # Call token endpoint - await token_endpoint( - request=mock_request, - grant_type="authorization_code", - code="test_code", - redirect_uri="http://localhost:60108/callback", - client_id="test_client_id", - mcp_server_name="google_mcp", - client_secret="test_secret", - ) - - # Verify that the redirect_uri sent to the provider uses the external URL - call_args = mock_async_client.post.call_args - assert ( - call_args[1]["data"]["redirect_uri"] - == "https://proxy.example.com/github/mcp/callback" - ) - - -@pytest.mark.parametrize( - "base_url,x_forwarded_proto,x_forwarded_host,x_forwarded_port,expected_url", - [ - # Case 1: No forwarded headers - use original URL as-is (no trailing slash) - ( - "http://localhost:4000/", - None, - None, - None, - "http://localhost:4000", - ), - # Case 2: Only X-Forwarded-Proto - change scheme only - ( - "http://localhost:4000/", - "https", - None, - None, - "https://localhost:4000", - ), - # Case 3: X-Forwarded-Proto + X-Forwarded-Host - change scheme and host - ( - "http://localhost:4000/", - "https", - "proxy.example.com", - None, - "https://proxy.example.com", - ), - # Case 4: X-Forwarded-Host with port included in host header - ( - "http://localhost:4000/", - "https", - "proxy.example.com:8080", - None, - "https://proxy.example.com:8080", - ), - # Case 5: X-Forwarded-Host + X-Forwarded-Port as separate headers - ( - "http://localhost:4000/", - "https", - "proxy.example.com", - "8443", - "https://proxy.example.com:8443", - ), - # Case 6: Only X-Forwarded-Host without proto - use original scheme - ( - "http://localhost:4000/", - None, - "proxy.example.com", - None, - "http://proxy.example.com", - ), - # Case 7: Only X-Forwarded-Port without host - preserves original port if present - # (This is safer behavior - X-Forwarded-Port alone is unusual) - ( - "http://localhost:4000/", - None, - None, - "8443", - "http://localhost:4000", # Original port preserved when already present - ), - # Case 8: Complex internal URL with path (path is preserved) - ( - "http://localhost:8888/github/mcp", - "https", - "proxy.example.com", - None, - "https://proxy.example.com/github/mcp", - ), - # Case 9: IPv6 address in X-Forwarded-Host (should not treat :: as port separator) - ( - "http://localhost:4000/", - "https", - "[2001:db8::1]", - None, - "https://[2001:db8::1]", - ), - # Case 10: IPv6 address with port - ( - "http://localhost:4000/", - "https", - "[2001:db8::1]:8080", - None, - "https://[2001:db8::1]:8080", - ), - # Case 11: X-Forwarded-Host already has port, X-Forwarded-Port also provided (host wins) - ( - "http://localhost:4000/", - "https", - "proxy.example.com:9000", - "8443", - "https://proxy.example.com:9000", - ), - # Case 12: Standard proxy setup (most common case) - ( - "http://127.0.0.1:8888/", - "https", - "chatproxy.company.com", - None, - "https://chatproxy.company.com", - ), - # Case 13: Internal URL already has port, X-Forwarded-Port does NOT override - # (safer behavior - preserves original port when X-Forwarded-Host not provided) - ( - "http://localhost:4000/", - None, - None, - "443", - "http://localhost:4000", # Original port preserved - ), - # Case 14: Original URL with existing port in netloc, X-Forwarded-Host replaces it - ( - "http://internal.local:8888/", - "https", - "external.com", - None, - "https://external.com", - ), - ], -) -def test_get_request_base_url_comprehensive( - base_url, - x_forwarded_proto, - x_forwarded_host, - x_forwarded_port, - expected_url, - trusted_proxy_origin_headers, -): - """Comprehensive test for get_request_base_url with various header combinations""" - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - get_request_base_url, - ) - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - # Create mock request - mock_request = MagicMock(spec=Request) - mock_request.base_url = base_url - set_request_from_trusted_proxy(mock_request) - - # Build headers dict - headers = {} - if x_forwarded_proto: - headers["X-Forwarded-Proto"] = x_forwarded_proto - if x_forwarded_host: - headers["X-Forwarded-Host"] = x_forwarded_host - if x_forwarded_port: - headers["X-Forwarded-Port"] = x_forwarded_port - - # Mock headers.get() to return our test values - def mock_get(header_name, default=None): - return headers.get(header_name, default) - - mock_request.headers.get = mock_get - - # Test the function - result = get_request_base_url(mock_request) - - # Verify result - assert result == expected_url, ( - f"Expected '{expected_url}' but got '{result}'\n" - f"Input: base_url={base_url}, " - f"X-Forwarded-Proto={x_forwarded_proto}, " - f"X-Forwarded-Host={x_forwarded_host}, " - f"X-Forwarded-Port={x_forwarded_port}" - ) - - -def test_get_request_base_url_ignores_forwarded_headers_from_untrusted_client(): - try: - from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( - get_request_base_url, - ) - from fastapi import Request - except ImportError: - pytest.skip("MCP discoverable endpoints not available") - - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://gateway.example.com/mcp" - mock_request.headers = { - "X-Forwarded-Proto": "https", - "X-Forwarded-Host": "attacker.example.com", - "X-Forwarded-Port": "443", - } - mock_request.client = MagicMock() - mock_request.client.host = "203.0.113.10" - - with patch( - "litellm.proxy.proxy_server.general_settings", - { - "use_x_forwarded_for": True, - "mcp_trusted_proxy_ranges": TRUSTED_PROXY_RANGES, - }, - create=True, - ): - assert get_request_base_url(mock_request) == "https://gateway.example.com/mcp" - - -def test_validate_trusted_redirect_uri_rejects_spoofed_forwarded_host(): - try: - from litellm.proxy._experimental.mcp_server.oauth_utils import ( - validate_trusted_redirect_uri, - ) - from fastapi import Request - except ImportError: - pytest.skip("MCP OAuth utilities not available") - - mock_request = MagicMock(spec=Request) - mock_request.base_url = "https://gateway.example.com/" - mock_request.headers = { - "X-Forwarded-Proto": "https", - "X-Forwarded-Host": "attacker.example.com", - } - mock_request.client = MagicMock() - mock_request.client.host = "203.0.113.10" - - with ( - patch( - "litellm.proxy.proxy_server.general_settings", - { - "use_x_forwarded_for": True, - "mcp_trusted_proxy_ranges": TRUSTED_PROXY_RANGES, - }, - create=True, - ), - pytest.raises(HTTPException), - ): - validate_trusted_redirect_uri( - mock_request, - "https://attacker.example.com/callback", - ) - - -def test_validate_trusted_redirect_uri_allows_forwarded_origin_from_trusted_proxy( - trusted_proxy_origin_headers, -): - try: - from litellm.proxy._experimental.mcp_server.oauth_utils import ( - validate_trusted_redirect_uri, - ) - from fastapi import Request - except ImportError: - pytest.skip("MCP OAuth utilities not available") - - mock_request = MagicMock(spec=Request) - mock_request.base_url = "http://localhost:4000/" - mock_request.headers = { - "X-Forwarded-Proto": "https", - "X-Forwarded-Host": "proxy.example.com", - } - set_request_from_trusted_proxy(mock_request) - - validate_trusted_redirect_uri( - mock_request, - "https://proxy.example.com/callback", - ) diff --git a/tests/litellm_utils_tests/base_token_counter_test.py b/tests/litellm_utils_tests/base_token_counter_test.py index 9af14dc9f47..ddce27522c2 100644 --- a/tests/litellm_utils_tests/base_token_counter_test.py +++ b/tests/litellm_utils_tests/base_token_counter_test.py @@ -10,16 +10,11 @@ Usage: the abstract methods to provide provider-specific configuration. """ -import os -import sys from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.utils import TokenCountResponse diff --git a/tests/litellm_utils_tests/conftest.py b/tests/litellm_utils_tests/conftest.py index 39ea4299f35..002ed594d3f 100644 --- a/tests/litellm_utils_tests/conftest.py +++ b/tests/litellm_utils_tests/conftest.py @@ -2,14 +2,9 @@ import asyncio import importlib -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402,F401 @@ -38,9 +33,6 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path importlib.reload(litellm) diff --git a/tests/litellm_utils_tests/test_aiohttp_handler.py b/tests/litellm_utils_tests/test_aiohttp_handler.py index 14c80d0e0bd..9fdac5ca23d 100644 --- a/tests/litellm_utils_tests/test_aiohttp_handler.py +++ b/tests/litellm_utils_tests/test_aiohttp_handler.py @@ -1,6 +1,5 @@ import asyncio import copy -import sys import time from datetime import datetime from unittest import mock @@ -10,11 +9,7 @@ from dotenv import load_dotenv from litellm.types.utils import StandardCallbackDynamicParams load_dotenv() -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest import litellm diff --git a/tests/litellm_utils_tests/test_anthropic_token_counter.py b/tests/litellm_utils_tests/test_anthropic_token_counter.py index 028586203a5..df3d198b6cf 100644 --- a/tests/litellm_utils_tests/test_anthropic_token_counter.py +++ b/tests/litellm_utils_tests/test_anthropic_token_counter.py @@ -5,14 +5,10 @@ Tests for the Anthropic token counter implementation using the base test suite. """ import os -import sys from typing import Any, Dict, List import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.llms.anthropic.count_tokens import AnthropicTokenCounter from litellm.llms.base_llm.base_utils import BaseTokenCounter diff --git a/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py b/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py index 2686c28cb1c..50631eb9341 100644 --- a/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py +++ b/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py @@ -5,14 +5,10 @@ Tests for the Azure AI Anthropic token counter implementation using the base tes """ import os -import sys from typing import Any, Dict, List import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.llms.azure_ai.anthropic.count_tokens import AzureAIAnthropicTokenCounter from litellm.llms.base_llm.base_utils import BaseTokenCounter diff --git a/tests/litellm_utils_tests/test_bedrock_token_counter.py b/tests/litellm_utils_tests/test_bedrock_token_counter.py index 9fb2463e8b5..683949fc5c7 100644 --- a/tests/litellm_utils_tests/test_bedrock_token_counter.py +++ b/tests/litellm_utils_tests/test_bedrock_token_counter.py @@ -9,15 +9,11 @@ counting, the test will be skipped. """ import os -import sys from typing import Any, Dict, List from unittest.mock import patch import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter diff --git a/tests/litellm_utils_tests/test_cyberark.py b/tests/litellm_utils_tests/test_cyberark.py index 71daf35a265..9172e33af10 100644 --- a/tests/litellm_utils_tests/test_cyberark.py +++ b/tests/litellm_utils_tests/test_cyberark.py @@ -3,14 +3,12 @@ Integration test for CyberArk Conjur Secret Manager. """ import os -import sys import pytest import yaml from dotenv import load_dotenv load_dotenv() -sys.path.insert(0, os.path.abspath("../..")) from unittest.mock import AsyncMock, MagicMock, patch from litellm._uuid import uuid diff --git a/tests/litellm_utils_tests/test_get_secret.py b/tests/litellm_utils_tests/test_get_secret.py index eec67b5d765..048e668467c 100644 --- a/tests/litellm_utils_tests/test_get_secret.py +++ b/tests/litellm_utils_tests/test_get_secret.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm diff --git a/tests/litellm_utils_tests/test_hashicorp.py b/tests/litellm_utils_tests/test_hashicorp.py index 1d98debef2c..ac9d4af3f53 100644 --- a/tests/litellm_utils_tests/test_hashicorp.py +++ b/tests/litellm_utils_tests/test_hashicorp.py @@ -1,14 +1,10 @@ import os -import sys import pytest from dotenv import load_dotenv load_dotenv() import httpx -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import patch, MagicMock import logging from litellm._logging import verbose_logger diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index 9a17aaeea87..cfdddd20263 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -2,14 +2,10 @@ # This tests if ahealth_check() actually works import os -import sys import pytest from unittest.mock import AsyncMock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import litellm diff --git a/tests/litellm_utils_tests/test_logging_callback_manager.py b/tests/litellm_utils_tests/test_logging_callback_manager.py index 517ba6befd7..ebd5b473ebb 100644 --- a/tests/litellm_utils_tests/test_logging_callback_manager.py +++ b/tests/litellm_utils_tests/test_logging_callback_manager.py @@ -1,14 +1,10 @@ import json import os -import sys import time from datetime import datetime from unittest.mock import AsyncMock, patch, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager diff --git a/tests/litellm_utils_tests/test_proxy_budget_reset.py b/tests/litellm_utils_tests/test_proxy_budget_reset.py index 83891b55fb5..9f6e1f4c3f7 100644 --- a/tests/litellm_utils_tests/test_proxy_budget_reset.py +++ b/tests/litellm_utils_tests/test_proxy_budget_reset.py @@ -1,6 +1,4 @@ import asyncio -import os -import sys from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -11,9 +9,6 @@ load_dotenv() from litellm.proxy._types import LiteLLM_BudgetTableFull -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob diff --git a/tests/litellm_utils_tests/test_secret_manager.py b/tests/litellm_utils_tests/test_secret_manager.py index 012889ee00c..4ba928dacd7 100644 --- a/tests/litellm_utils_tests/test_secret_manager.py +++ b/tests/litellm_utils_tests/test_secret_manager.py @@ -1,6 +1,5 @@ import base64 import os -import sys import time import traceback from litellm._uuid import uuid @@ -12,9 +11,6 @@ load_dotenv() import tempfile from uuid import uuid4 -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm.llms.azure.azure import get_azure_ad_token_from_oidc diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 0a5327d2662..67f2e1ce06d 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -1,6 +1,5 @@ import copy import logging -import sys import time from datetime import datetime from unittest import mock @@ -12,9 +11,6 @@ from litellm.types.utils import StandardCallbackDynamicParams load_dotenv() import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest import litellm diff --git a/tests/litellm_utils_tests/test_validate_tool_choice.py b/tests/litellm_utils_tests/test_validate_tool_choice.py index 07f8c9ed8f4..b8246fe0deb 100644 --- a/tests/litellm_utils_tests/test_validate_tool_choice.py +++ b/tests/litellm_utils_tests/test_validate_tool_choice.py @@ -1,8 +1,5 @@ import pytest -import sys -import os -sys.path.insert(0, os.path.abspath("../..")) from litellm.utils import validate_chat_completion_tool_choice diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 99ca9fb17b5..74c0478b08b 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -1,17 +1,12 @@ import httpx import json import pytest -import sys from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, Mock, patch -import os from litellm._uuid import uuid import time import base64 -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from abc import ABC, abstractmethod diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index b5884f51275..5501d99cb22 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -2,14 +2,9 @@ import asyncio import importlib -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402 @@ -77,17 +72,12 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path importlib.reload(litellm) try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/llm_responses_api_testing/test_anthropic_responses_api.py b/tests/llm_responses_api_testing/test_anthropic_responses_api.py index 0ca159219df..8ed85aaa209 100644 --- a/tests/llm_responses_api_testing/test_anthropic_responses_api.py +++ b/tests/llm_responses_api_testing/test_anthropic_responses_api.py @@ -1,5 +1,3 @@ -import os -import sys import pytest import asyncio from typing import Optional @@ -13,7 +11,6 @@ from litellm.responses.litellm_completion_transformation.transformation import ( from litellm.types.utils import ModelResponse -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.custom_logger import CustomLogger import json diff --git a/tests/llm_responses_api_testing/test_anthropic_tool_result_empty_call_id.py b/tests/llm_responses_api_testing/test_anthropic_tool_result_empty_call_id.py index 08b1c1784e7..28621c6531f 100644 --- a/tests/llm_responses_api_testing/test_anthropic_tool_result_empty_call_id.py +++ b/tests/llm_responses_api_testing/test_anthropic_tool_result_empty_call_id.py @@ -11,12 +11,9 @@ The issue occurs when: 3. The message is sent to Anthropic without a corresponding tool_use block """ -import os -import sys import pytest from unittest.mock import patch, MagicMock -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, diff --git a/tests/llm_responses_api_testing/test_anthropic_tool_result_fix.py b/tests/llm_responses_api_testing/test_anthropic_tool_result_fix.py index d7c15c7609f..d203b0f6917 100644 --- a/tests/llm_responses_api_testing/test_anthropic_tool_result_fix.py +++ b/tests/llm_responses_api_testing/test_anthropic_tool_result_fix.py @@ -5,13 +5,10 @@ This test verifies that when using previous_response_id with tool_result, the fix ensures tool_calls are added to the previous assistant message. """ -import os -import sys import pytest import json from unittest.mock import patch, AsyncMock -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py index 79990a88496..6f1bb440341 100644 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ b/tests/llm_responses_api_testing/test_azure_responses_api.py @@ -1,10 +1,8 @@ import os -import sys import pytest import asyncio from unittest.mock import patch, AsyncMock -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.custom_logger import CustomLogger import json diff --git a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py index 5388c5aef83..bd617587cf3 100644 --- a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py +++ b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py @@ -13,15 +13,12 @@ response tracking and logging. """ import json -import os -import sys from datetime import datetime from typing import Any, Dict, Optional from unittest.mock import Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj diff --git a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py index 3ed92bd760d..d84e9cc66e3 100644 --- a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py +++ b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py @@ -1,9 +1,7 @@ import os -import sys import pytest from unittest.mock import patch, AsyncMock -sys.path.insert(0, os.path.abspath("../..")) import litellm import json from base_responses_api import BaseResponsesAPITest diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index d614c40f5d0..5f77d5a5477 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1,5 +1,4 @@ import os -import sys import pytest import asyncio from typing import Optional, cast @@ -10,7 +9,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging import time import json -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import StandardLoggingPayload diff --git a/tests/llm_translation/base_audio_transcription_unit_tests.py b/tests/llm_translation/base_audio_transcription_unit_tests.py index 71f2aa79ce5..76401b456fa 100644 --- a/tests/llm_translation/base_audio_transcription_unit_tests.py +++ b/tests/llm_translation/base_audio_transcription_unit_tests.py @@ -1,15 +1,11 @@ import httpx import json import pytest -import sys from typing import Any, Dict, List from unittest.mock import MagicMock, Mock, patch import os from litellm._uuid import uuid -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import transcription from litellm.litellm_core_utils.get_supported_openai_params import ( diff --git a/tests/llm_translation/base_embedding_unit_tests.py b/tests/llm_translation/base_embedding_unit_tests.py index 30a9dcc0da3..1a88f0e9d6b 100644 --- a/tests/llm_translation/base_embedding_unit_tests.py +++ b/tests/llm_translation/base_embedding_unit_tests.py @@ -2,14 +2,10 @@ import asyncio import httpx import json import pytest -import sys from typing import Any, Dict, List from unittest.mock import MagicMock, Mock, patch import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import embedding from litellm.exceptions import BadRequestError diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 6d845f4b2f1..1a33422a31c 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -10,9 +10,6 @@ import time import base64 import inspect -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/llm_translation/base_rerank_unit_tests.py b/tests/llm_translation/base_rerank_unit_tests.py index 57878c8f171..df7dd33d7b0 100644 --- a/tests/llm_translation/base_rerank_unit_tests.py +++ b/tests/llm_translation/base_rerank_unit_tests.py @@ -2,14 +2,10 @@ import asyncio import httpx import json import pytest -import sys from typing import Any, Dict, List from unittest.mock import MagicMock, Mock, patch import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index f5b71236e92..8532af2851c 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -7,14 +7,9 @@ import asyncio import importlib -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402 @@ -123,7 +118,6 @@ def event_loop(): @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(event_loop): # Add event_loop as a dependency - sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/llm_translation/realtime/base_realtime_tests.py b/tests/llm_translation/realtime/base_realtime_tests.py index 1a2c6ff6a9c..964e1d0ac59 100644 --- a/tests/llm_translation/realtime/base_realtime_tests.py +++ b/tests/llm_translation/realtime/base_realtime_tests.py @@ -8,14 +8,12 @@ across different providers (OpenAI, xAI, etc.) import asyncio import json import os -import sys from abc import ABC, abstractmethod from typing import Optional, Tuple, Union import pytest import websockets -sys.path.insert(0, os.path.abspath("../../..")) import litellm diff --git a/tests/llm_translation/realtime/test_openai_realtime.py b/tests/llm_translation/realtime/test_openai_realtime.py index 0e50e2792d6..add22117590 100644 --- a/tests/llm_translation/realtime/test_openai_realtime.py +++ b/tests/llm_translation/realtime/test_openai_realtime.py @@ -1,13 +1,9 @@ import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.types.realtime import RealtimeQueryParams diff --git a/tests/llm_translation/realtime/test_openai_realtime_simple.py b/tests/llm_translation/realtime/test_openai_realtime_simple.py index 073c1ce11af..93451a6617e 100644 --- a/tests/llm_translation/realtime/test_openai_realtime_simple.py +++ b/tests/llm_translation/realtime/test_openai_realtime_simple.py @@ -5,12 +5,9 @@ Tests OpenAI's Realtime API through LiteLLM's realtime interface. Uses the base test class to ensure consistent behavior across providers. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from tests.llm_translation.realtime.base_realtime_tests import BaseRealtimeTest diff --git a/tests/llm_translation/realtime/test_xai_realtime.py b/tests/llm_translation/realtime/test_xai_realtime.py index 8ffcb3db30d..19cf8624c48 100644 --- a/tests/llm_translation/realtime/test_xai_realtime.py +++ b/tests/llm_translation/realtime/test_xai_realtime.py @@ -5,13 +5,10 @@ Tests xAI's Grok Voice Agent API through LiteLLM's realtime interface. Uses the base test class to ensure consistent behavior across providers. """ -import os -import sys from typing import Tuple import pytest -sys.path.insert(0, os.path.abspath("../../..")) from tests.llm_translation.realtime.base_realtime_tests import BaseRealtimeTest diff --git a/tests/llm_translation/test_a2a.py b/tests/llm_translation/test_a2a.py index ec260acd1ae..1f647092abf 100644 --- a/tests/llm_translation/test_a2a.py +++ b/tests/llm_translation/test_a2a.py @@ -6,11 +6,9 @@ streaming and non-streaming requests. """ import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index ab1c67dffbf..8c55014955f 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -3,7 +3,6 @@ import asyncio import os -import sys import traceback from dotenv import load_dotenv @@ -15,9 +14,6 @@ from litellm.llms.anthropic.chat import ModelResponseIterator load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Optional from unittest.mock import MagicMock, patch diff --git a/tests/llm_translation/test_azure_agents.py b/tests/llm_translation/test_azure_agents.py index 6a737cc102b..e0741471582 100644 --- a/tests/llm_translation/test_azure_agents.py +++ b/tests/llm_translation/test_azure_agents.py @@ -25,9 +25,7 @@ See: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/quickstart import json import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import pytest from unittest.mock import MagicMock diff --git a/tests/llm_translation/test_azure_ai.py b/tests/llm_translation/test_azure_ai.py index 553f9102246..5be6ade80ab 100644 --- a/tests/llm_translation/test_azure_ai.py +++ b/tests/llm_translation/test_azure_ai.py @@ -3,7 +3,6 @@ import asyncio import os -import sys import traceback from dotenv import load_dotenv @@ -20,9 +19,6 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Optional from unittest.mock import MagicMock, patch diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index ab122d3ff6a..1a2d672af71 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -1,12 +1,8 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/llm_translation/test_azure_openai.py b/tests/llm_translation/test_azure_openai.py index 0deb20900a7..0fa72b45ed8 100644 --- a/tests/llm_translation/test_azure_openai.py +++ b/tests/llm_translation/test_azure_openai.py @@ -1,9 +1,5 @@ -import sys import os -sys.path.insert( - 0, os.path.abspath("../../") -) # Adds the parent directory to the system path import httpx import pytest @@ -103,7 +99,6 @@ from unittest.mock import MagicMock, patch from openai import AzureOpenAI import litellm from litellm import completion -import os @pytest.mark.parametrize( diff --git a/tests/llm_translation/test_bedrock_agentcore.py b/tests/llm_translation/test_bedrock_agentcore.py index 40774cf3d60..0087eb5b326 100644 --- a/tests/llm_translation/test_bedrock_agentcore.py +++ b/tests/llm_translation/test_bedrock_agentcore.py @@ -2,13 +2,10 @@ Test Bedrock AgentCore integration """ -import os -import sys from dotenv import load_dotenv load_dotenv() -sys.path.insert(0, os.path.abspath("../..")) import litellm from unittest.mock import MagicMock, Mock, patch diff --git a/tests/llm_translation/test_bedrock_agents.py b/tests/llm_translation/test_bedrock_agents.py index 6371224def9..1685dd220d2 100644 --- a/tests/llm_translation/test_bedrock_agents.py +++ b/tests/llm_translation/test_bedrock_agents.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from dotenv import load_dotenv @@ -10,9 +8,6 @@ load_dotenv() import io import json -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, Mock, patch import pytest diff --git a/tests/llm_translation/test_bedrock_anthropic_regression.py b/tests/llm_translation/test_bedrock_anthropic_regression.py index 8b8ce0a6cc8..8f2974f531c 100644 --- a/tests/llm_translation/test_bedrock_anthropic_regression.py +++ b/tests/llm_translation/test_bedrock_anthropic_regression.py @@ -11,13 +11,10 @@ feature parity and prevent regression of previously fixed issues. """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 6ee6e5d1493..550e82fb5bb 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -4,7 +4,6 @@ Tests Bedrock Completion + Rerank endpoints # @pytest.mark.skip(reason="AWS Suspended Account") import os -import sys import traceback from dotenv import load_dotenv @@ -15,9 +14,6 @@ load_dotenv() import io import json -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, Mock, patch import pytest diff --git a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py index 5d2fab15a8f..dad2fdbf065 100644 --- a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py +++ b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py @@ -1,14 +1,9 @@ # tests/llm_translation/test_base_aws_llm.py -import os import json import pytest from unittest.mock import patch from botocore.credentials import Credentials -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler diff --git a/tests/llm_translation/test_bedrock_embedding.py b/tests/llm_translation/test_bedrock_embedding.py index e343b8856a7..56baed141da 100644 --- a/tests/llm_translation/test_bedrock_embedding.py +++ b/tests/llm_translation/test_bedrock_embedding.py @@ -1,15 +1,11 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch import pytest import base64 import httpx -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler diff --git a/tests/llm_translation/test_bedrock_gpt_oss.py b/tests/llm_translation/test_bedrock_gpt_oss.py index 0a595ad7114..4af81ee81f7 100644 --- a/tests/llm_translation/test_bedrock_gpt_oss.py +++ b/tests/llm_translation/test_bedrock_gpt_oss.py @@ -1,13 +1,8 @@ from base_llm_unit_tests import BaseLLMChatTest import json import pytest -import sys -import os from unittest.mock import patch, Mock, MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.llms.custom_httpx.http_handler import HTTPHandler diff --git a/tests/llm_translation/test_bedrock_invoke_tests.py b/tests/llm_translation/test_bedrock_invoke_tests.py index 901b43542f7..cf53899ecf6 100644 --- a/tests/llm_translation/test_bedrock_invoke_tests.py +++ b/tests/llm_translation/test_bedrock_invoke_tests.py @@ -1,11 +1,7 @@ from base_llm_unit_tests import BaseLLMChatTest import pytest -import sys import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.types.llms.bedrock import BedrockInvokeNovaRequest diff --git a/tests/llm_translation/test_bedrock_llama.py b/tests/llm_translation/test_bedrock_llama.py index b18928747eb..6c1a7073c13 100644 --- a/tests/llm_translation/test_bedrock_llama.py +++ b/tests/llm_translation/test_bedrock_llama.py @@ -1,11 +1,6 @@ from base_llm_unit_tests import BaseLLMChatTest import pytest -import sys -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/llm_translation/test_bedrock_mantle.py b/tests/llm_translation/test_bedrock_mantle.py index 46a0c653005..70919a07bb9 100644 --- a/tests/llm_translation/test_bedrock_mantle.py +++ b/tests/llm_translation/test_bedrock_mantle.py @@ -9,14 +9,11 @@ Tests use a fake/mocked HTTP layer to verify the full request pipeline: """ import json -import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index a82d1c6f029..3bf047c51a5 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -14,13 +14,11 @@ This test suite verifies: from base_llm_unit_tests import BaseLLMChatTest import httpx import pytest -import sys import os import json from typing import Optional from unittest.mock import AsyncMock, Mock, patch -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.llms.bedrock.common_utils import get_bedrock_chat_config from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/llm_translation/test_bedrock_nova_embedding.py b/tests/llm_translation/test_bedrock_nova_embedding.py index 9795dc3d8d5..c4fd0724884 100644 --- a/tests/llm_translation/test_bedrock_nova_embedding.py +++ b/tests/llm_translation/test_bedrock_nova_embedding.py @@ -11,15 +11,10 @@ Tests cover: """ import json -import os -import sys from unittest.mock import MagicMock, Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.bedrock.embed.amazon_nova_transformation import ( diff --git a/tests/llm_translation/test_bedrock_nova_json.py b/tests/llm_translation/test_bedrock_nova_json.py index 7531891c4ef..754ef4e3525 100644 --- a/tests/llm_translation/test_bedrock_nova_json.py +++ b/tests/llm_translation/test_bedrock_nova_json.py @@ -1,11 +1,6 @@ from base_llm_unit_tests import BaseLLMChatTest import pytest -import sys -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/llm_translation/test_cohere.py b/tests/llm_translation/test_cohere.py index 0eb0b1b33fe..729f42f8984 100644 --- a/tests/llm_translation/test_cohere.py +++ b/tests/llm_translation/test_cohere.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from dotenv import load_dotenv @@ -7,9 +5,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import json import pytest diff --git a/tests/llm_translation/test_containers_api.py b/tests/llm_translation/test_containers_api.py index 6c7303e7b4d..c5248516a1c 100644 --- a/tests/llm_translation/test_containers_api.py +++ b/tests/llm_translation/test_containers_api.py @@ -5,12 +5,10 @@ Tests the container files endpoints using LiteLLM SDK methods. """ import os -import sys import time import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.containers import ( create_container, @@ -63,7 +61,7 @@ def test_container_files_api(): # 3. Try retrieve non-existent file metadata (should raise error) print("3. Testing retrieve_container_file (expect error)...") - with pytest.raises(Exception, match="(?i)not found|invalid"): + with pytest.raises(Exception, match=r"(?i)not found|invalid"): retrieve_container_file( container_id=container.id, file_id="cfile_nonexistent", diff --git a/tests/llm_translation/test_convert_dict_to_image.py b/tests/llm_translation/test_convert_dict_to_image.py index 62a7eec8cbb..df6e2bcb4a3 100644 --- a/tests/llm_translation/test_convert_dict_to_image.py +++ b/tests/llm_translation/test_convert_dict_to_image.py @@ -1,11 +1,6 @@ import json -import os -import sys from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../../") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/llm_translation/test_databricks.py b/tests/llm_translation/test_databricks.py index 3a224231667..46caae0e7bd 100644 --- a/tests/llm_translation/test_databricks.py +++ b/tests/llm_translation/test_databricks.py @@ -6,11 +6,7 @@ import sys from typing import Any, Dict, List from unittest.mock import MagicMock, Mock, patch, ANY -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/llm_translation/test_deepgram.py b/tests/llm_translation/test_deepgram.py index 204d6c01cf8..855d570488b 100644 --- a/tests/llm_translation/test_deepgram.py +++ b/tests/llm_translation/test_deepgram.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest diff --git a/tests/llm_translation/test_elevenlabs.py b/tests/llm_translation/test_elevenlabs.py index b6c838d2300..9dc4a1d09ed 100644 --- a/tests/llm_translation/test_elevenlabs.py +++ b/tests/llm_translation/test_elevenlabs.py @@ -1,5 +1,4 @@ import os -import sys from typing import Any, Dict @@ -7,9 +6,6 @@ import pytest from unittest.mock import patch, MagicMock import httpx -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest diff --git a/tests/llm_translation/test_evals_api.py b/tests/llm_translation/test_evals_api.py index 4a55663e669..ba6b5edf3cd 100644 --- a/tests/llm_translation/test_evals_api.py +++ b/tests/llm_translation/test_evals_api.py @@ -4,13 +4,11 @@ Tests for Evals API operations across providers import hashlib import os -import sys from abc import ABC, abstractmethod from typing import Optional import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.types.llms.openai_evals import ( diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 27059581e4d..e20134fc1bf 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -1,11 +1,6 @@ -import os -import sys import json import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 310a2e2c20c..0c3eca52dde 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1,11 +1,7 @@ import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system paths from base_llm_unit_tests import BaseLLMChatTest from litellm.llms.vertex_ai.context_caching.transformation import ( diff --git a/tests/llm_translation/test_gpt4o_audio.py b/tests/llm_translation/test_gpt4o_audio.py index a50d07406d4..0f20119e4ef 100644 --- a/tests/llm_translation/test_gpt4o_audio.py +++ b/tests/llm_translation/test_gpt4o_audio.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/llm_translation/test_hosted_vllm_embedding_e2e.py b/tests/llm_translation/test_hosted_vllm_embedding_e2e.py index 4b887013357..23ad63ab6da 100644 --- a/tests/llm_translation/test_hosted_vllm_embedding_e2e.py +++ b/tests/llm_translation/test_hosted_vllm_embedding_e2e.py @@ -5,13 +5,9 @@ This test verifies that the hosted_vllm provider works correctly with real API e """ import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/llm_translation/test_huggingface_chat_completion.py b/tests/llm_translation/test_huggingface_chat_completion.py index cdf3f9ef76f..90e6c2adb8d 100644 --- a/tests/llm_translation/test_huggingface_chat_completion.py +++ b/tests/llm_translation/test_huggingface_chat_completion.py @@ -3,15 +3,10 @@ Test HuggingFace LLM """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch from base_llm_unit_tests import BaseLLMChatTest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/llm_translation/test_hyperbolic.py b/tests/llm_translation/test_hyperbolic.py index 006d31c88e6..78817fbd902 100644 --- a/tests/llm_translation/test_hyperbolic.py +++ b/tests/llm_translation/test_hyperbolic.py @@ -1,13 +1,9 @@ import os -import sys from datetime import datetime from unittest.mock import MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import get_llm_provider @@ -76,7 +72,6 @@ def test_hyperbolic_in_provider_lists(): def test_hyperbolic_models_configuration(): """Test that Hyperbolic models are properly configured""" import json - import os # Load model configuration directly from the JSON file json_path = os.path.join( diff --git a/tests/llm_translation/test_infinity.py b/tests/llm_translation/test_infinity.py index 5ca3d377fd7..1829113e045 100644 --- a/tests/llm_translation/test_infinity.py +++ b/tests/llm_translation/test_infinity.py @@ -1,25 +1,15 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import litellm -import os -import sys from unittest.mock import patch, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from test_rerank import assert_response_shape from base_embedding_unit_tests import BaseLLMEmbeddingTest diff --git a/tests/llm_translation/test_jina_ai.py b/tests/llm_translation/test_jina_ai.py index 00810369ed7..81527293a00 100644 --- a/tests/llm_translation/test_jina_ai.py +++ b/tests/llm_translation/test_jina_ai.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from base_rerank_unit_tests import BaseLLMRerankTest diff --git a/tests/llm_translation/test_langgraph.py b/tests/llm_translation/test_langgraph.py index fa3a7f91b6b..3d0de508e7c 100644 --- a/tests/llm_translation/test_langgraph.py +++ b/tests/llm_translation/test_langgraph.py @@ -19,9 +19,7 @@ Non-streaming: """ import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import pytest diff --git a/tests/llm_translation/test_litellm_proxy_provider.py b/tests/llm_translation/test_litellm_proxy_provider.py index cea0167472e..1cb805bf9ba 100644 --- a/tests/llm_translation/test_litellm_proxy_provider.py +++ b/tests/llm_translation/test_litellm_proxy_provider.py @@ -1,13 +1,9 @@ import json -import os -import sys +import re from datetime import datetime from io import BytesIO from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import litellm from litellm import completion, embedding @@ -578,7 +574,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers(): def test_litellm_gateway_from_sdk_with_thinking_param(): - with pytest.raises(Exception, match="Connection error.") as exc_info: + with pytest.raises(Exception, match=re.escape("Connection error.")) as exc_info: response = litellm.completion( model="litellm_proxy/anthropic.claude-sonnet-4-5-20250929-v1:0", messages=[{"role": "user", "content": "Hello world"}], diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index 8c7390d3d04..b6e30ddc711 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1,11 +1,6 @@ import json -import os -import sys from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../../../") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/llm_translation/test_llm_response_utils/test_get_headers.py b/tests/llm_translation/test_llm_response_utils/test_get_headers.py index f0cc7ca61f1..380f89bbdd4 100644 --- a/tests/llm_translation/test_llm_response_utils/test_get_headers.py +++ b/tests/llm_translation/test_llm_response_utils/test_get_headers.py @@ -1,11 +1,6 @@ import json -import os -import sys from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../../") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/llm_translation/test_minimax_tts.py b/tests/llm_translation/test_minimax_tts.py index e10b32fb39b..660e49b664f 100644 --- a/tests/llm_translation/test_minimax_tts.py +++ b/tests/llm_translation/test_minimax_tts.py @@ -3,15 +3,11 @@ Tests for MiniMax Text-to-Speech integration """ import os -import sys from pathlib import Path from unittest.mock import MagicMock, Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import speech diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py index 62f69e616ab..9e2f726a020 100644 --- a/tests/llm_translation/test_mistral_api.py +++ b/tests/llm_translation/test_mistral_api.py @@ -1,6 +1,4 @@ import asyncio -import os -import sys import traceback from dotenv import load_dotenv @@ -12,9 +10,6 @@ from litellm.llms.anthropic.chat import ModelResponseIterator load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Optional from unittest.mock import MagicMock, patch diff --git a/tests/llm_translation/test_morph.py b/tests/llm_translation/test_morph.py index a24ace5ca6d..b91d1810d38 100644 --- a/tests/llm_translation/test_morph.py +++ b/tests/llm_translation/test_morph.py @@ -1,12 +1,8 @@ """Unit tests for Morph provider integration.""" import os -import sys from unittest.mock import patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import MorphChatConfig, get_llm_provider diff --git a/tests/llm_translation/test_nvidia_nim.py b/tests/llm_translation/test_nvidia_nim.py index 79c792d1644..7ee4f347f72 100644 --- a/tests/llm_translation/test_nvidia_nim.py +++ b/tests/llm_translation/test_nvidia_nim.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 405dbb0e6ec..2b9abdec5d0 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -1,13 +1,8 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, patch from typing import Optional -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/llm_translation/test_openai_o1.py b/tests/llm_translation/test_openai_o1.py index dbaf20717a0..e188a3af647 100644 --- a/tests/llm_translation/test_openai_o1.py +++ b/tests/llm_translation/test_openai_o1.py @@ -1,12 +1,8 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/llm_translation/test_openrouter.py b/tests/llm_translation/test_openrouter.py index 8fbb8803d11..8ecf9b4a8a2 100644 --- a/tests/llm_translation/test_openrouter.py +++ b/tests/llm_translation/test_openrouter.py @@ -1,10 +1,5 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system paths import litellm diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 814f5a235e1..997f5b3b73f 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -2,14 +2,11 @@ # This tests if get_optional_params works as expected import asyncio import inspect -import os -import sys import time import traceback import pytest -sys.path.insert(0, os.path.abspath("../..")) from unittest.mock import MagicMock, patch import litellm diff --git a/tests/llm_translation/test_perplexity_reasoning.py b/tests/llm_translation/test_perplexity_reasoning.py index 2ea28b76696..61fbc9d7824 100644 --- a/tests/llm_translation/test_perplexity_reasoning.py +++ b/tests/llm_translation/test_perplexity_reasoning.py @@ -1,13 +1,9 @@ import json import os -import sys from unittest.mock import patch, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import completion diff --git a/tests/llm_translation/test_prompt_caching.py b/tests/llm_translation/test_prompt_caching.py index eb4703fd677..341973168e8 100644 --- a/tests/llm_translation/test_prompt_caching.py +++ b/tests/llm_translation/test_prompt_caching.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 1b4c8a82cf4..a90a3df584e 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1,11 +1,8 @@ #### What this tests #### # This tests if prompts are being correctly formatted -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from typing import List diff --git a/tests/llm_translation/test_replicate.py b/tests/llm_translation/test_replicate.py index 8972d115882..eb8987f5444 100644 --- a/tests/llm_translation/test_replicate.py +++ b/tests/llm_translation/test_replicate.py @@ -4,13 +4,10 @@ Unit tests for Replicate provider, particularly testing DeepSeek models import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion diff --git a/tests/llm_translation/test_rerank.py b/tests/llm_translation/test_rerank.py index cb254542009..3009928c9bc 100644 --- a/tests/llm_translation/test_rerank.py +++ b/tests/llm_translation/test_rerank.py @@ -1,7 +1,6 @@ import asyncio import json import os -import sys import traceback from dotenv import load_dotenv @@ -10,11 +9,7 @@ load_dotenv() import io from typing import Optional, Dict -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from unittest.mock import AsyncMock, MagicMock, patch import pytest diff --git a/tests/llm_translation/test_router_llm_translation_tests.py b/tests/llm_translation/test_router_llm_translation_tests.py index 26456ab0a35..10807adf356 100644 --- a/tests/llm_translation/test_router_llm_translation_tests.py +++ b/tests/llm_translation/test_router_llm_translation_tests.py @@ -4,13 +4,9 @@ Uses litellm.Router, ensures router.completion and router.acompletion pass BaseL import asyncio import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from base_llm_unit_tests import BaseLLMChatTest diff --git a/tests/llm_translation/test_skills_api.py b/tests/llm_translation/test_skills_api.py index e1830e50ef9..aeab5f0da3e 100644 --- a/tests/llm_translation/test_skills_api.py +++ b/tests/llm_translation/test_skills_api.py @@ -3,7 +3,6 @@ Tests for Skills API operations across providers """ import os -import sys import zipfile from abc import ABC, abstractmethod from contextlib import contextmanager @@ -12,7 +11,6 @@ from typing import Optional import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.types.llms.anthropic_skills import ( @@ -143,7 +141,6 @@ class BaseSkillsAPITest(ABC): """ Test listing skills. """ - import os custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() diff --git a/tests/llm_translation/test_text_completion.py b/tests/llm_translation/test_text_completion.py index 38d2dd95de7..7f81a6a3449 100644 --- a/tests/llm_translation/test_text_completion.py +++ b/tests/llm_translation/test_text_completion.py @@ -1,11 +1,6 @@ import json -import os -import sys from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/llm_translation/test_text_completion_unit_tests.py b/tests/llm_translation/test_text_completion_unit_tests.py index 55026ba0542..d741786ad44 100644 --- a/tests/llm_translation/test_text_completion_unit_tests.py +++ b/tests/llm_translation/test_text_completion_unit_tests.py @@ -1,6 +1,4 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock import pytest @@ -8,9 +6,6 @@ import httpx from respx import MockRouter from unittest.mock import patch, MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.types.utils import TextCompletionResponse diff --git a/tests/llm_translation/test_together_ai.py b/tests/llm_translation/test_together_ai.py index 387e61656ea..c371caefa5e 100644 --- a/tests/llm_translation/test_together_ai.py +++ b/tests/llm_translation/test_together_ai.py @@ -5,13 +5,9 @@ Test TogetherAI LLM from base_llm_unit_tests import BaseLLMChatTest import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/llm_translation/test_triton.py b/tests/llm_translation/test_triton.py index f9ab3bfaff7..a5d66809421 100644 --- a/tests/llm_translation/test_triton.py +++ b/tests/llm_translation/test_triton.py @@ -1,6 +1,4 @@ import json -import os -import sys import traceback from dotenv import load_dotenv @@ -9,9 +7,6 @@ load_dotenv() import io from unittest.mock import AsyncMock, MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm diff --git a/tests/llm_translation/test_unit_test_bedrock_invoke.py b/tests/llm_translation/test_unit_test_bedrock_invoke.py index 586b04384d5..e6cf4695089 100644 --- a/tests/llm_translation/test_unit_test_bedrock_invoke.py +++ b/tests/llm_translation/test_unit_test_bedrock_invoke.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from dotenv import load_dotenv import litellm.types @@ -10,7 +8,6 @@ import json load_dotenv() import io -sys.path.insert(0, os.path.abspath("../..")) from unittest.mock import AsyncMock, Mock, patch diff --git a/tests/llm_translation/test_voyage_ai.py b/tests/llm_translation/test_voyage_ai.py index 30f2844fbfa..208e01110da 100644 --- a/tests/llm_translation/test_voyage_ai.py +++ b/tests/llm_translation/test_voyage_ai.py @@ -1,12 +1,8 @@ import json import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch diff --git a/tests/llm_translation/test_watsonx.py b/tests/llm_translation/test_watsonx.py index 5857394d0ff..0ccc2ba85f3 100644 --- a/tests/llm_translation/test_watsonx.py +++ b/tests/llm_translation/test_watsonx.py @@ -1,10 +1,5 @@ import json -import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import completion, embedding from litellm.llms.custom_httpx.http_handler import HTTPHandler diff --git a/tests/llm_translation/test_xai.py b/tests/llm_translation/test_xai.py index f0945e6e165..7a121afc3fa 100644 --- a/tests/llm_translation/test_xai.py +++ b/tests/llm_translation/test_xai.py @@ -1,12 +1,8 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/load_tests/test_datadog_load_test.py b/tests/load_tests/test_datadog_load_test.py index f4328b71b1b..3dfc3fc6da4 100644 --- a/tests/load_tests/test_datadog_load_test.py +++ b/tests/load_tests/test_datadog_load_test.py @@ -1,7 +1,5 @@ -import sys import os -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py index cf9fe526b74..84400d6974b 100644 --- a/tests/load_tests/test_langsmith_load_test.py +++ b/tests/load_tests/test_langsmith_load_test.py @@ -1,8 +1,6 @@ -import sys import os -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/load_tests/test_memory_usage.py b/tests/load_tests/test_memory_usage.py index 347dbf2bb44..c5b5134a3d7 100644 --- a/tests/load_tests/test_memory_usage.py +++ b/tests/load_tests/test_memory_usage.py @@ -9,9 +9,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm.types @@ -21,7 +18,6 @@ from typing import Optional from unittest.mock import MagicMock, patch import pytest -import os import litellm from typing import Callable, Any diff --git a/tests/load_tests/test_otel_load_test.py b/tests/load_tests/test_otel_load_test.py index f5754c0c402..57dcc53a50b 100644 --- a/tests/load_tests/test_otel_load_test.py +++ b/tests/load_tests/test_otel_load_test.py @@ -1,8 +1,6 @@ -import sys import os -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/load_tests/test_vertex_embeddings_load_test.py b/tests/load_tests/test_vertex_embeddings_load_test.py index 9beee710553..c5b9a80ec6b 100644 --- a/tests/load_tests/test_vertex_embeddings_load_test.py +++ b/tests/load_tests/test_vertex_embeddings_load_test.py @@ -3,10 +3,8 @@ Load test on vertex AI embeddings to ensure vertex median response time is less """ -import sys import os -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/load_tests/test_vertex_load_tests.py b/tests/load_tests/test_vertex_load_tests.py index 9130873b970..93e1ed24f72 100644 --- a/tests/load_tests/test_vertex_load_tests.py +++ b/tests/load_tests/test_vertex_load_tests.py @@ -1,7 +1,5 @@ -import sys import os -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/local_testing/cache_unit_tests.py b/tests/local_testing/cache_unit_tests.py index 27eefb79fae..a1973d477b2 100644 --- a/tests/local_testing/cache_unit_tests.py +++ b/tests/local_testing/cache_unit_tests.py @@ -1,7 +1,5 @@ from abc import ABC, abstractmethod from litellm.caching import LiteLLMCacheType -import os -import sys import time import traceback from litellm._uuid import uuid @@ -10,9 +8,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import hashlib import random diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index d134a7439a8..4f142664827 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -13,13 +13,9 @@ import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # ``litellm.model_cost`` is loaded at import time from the URL pinned to ``main`` @@ -238,7 +234,6 @@ def setup_and_teardown(): Module-scoped setup. Reloads litellm only in single-process mode (skipped under xdist to avoid cross-worker interference). """ - sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/local_testing/create_mock_standard_logging_payload.py b/tests/local_testing/create_mock_standard_logging_payload.py index 106328e95e2..096c8ff8c60 100644 --- a/tests/local_testing/create_mock_standard_logging_payload.py +++ b/tests/local_testing/create_mock_standard_logging_payload.py @@ -1,9 +1,6 @@ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/local_testing/test_acompletion_fallbacks.py b/tests/local_testing/test_acompletion_fallbacks.py index 7cf97eb9b5e..f9ee5a93c32 100644 --- a/tests/local_testing/test_acompletion_fallbacks.py +++ b/tests/local_testing/test_acompletion_fallbacks.py @@ -1,14 +1,10 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import concurrent from dotenv import load_dotenv diff --git a/tests/local_testing/test_acooldowns_router.py b/tests/local_testing/test_acooldowns_router.py index 18dc26bda9a..18c58a5cfac 100644 --- a/tests/local_testing/test_acooldowns_router.py +++ b/tests/local_testing/test_acooldowns_router.py @@ -3,15 +3,11 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import concurrent from dotenv import load_dotenv diff --git a/tests/local_testing/test_add_function_to_prompt.py b/tests/local_testing/test_add_function_to_prompt.py index 43ee3dd41af..507fd99ec59 100644 --- a/tests/local_testing/test_add_function_to_prompt.py +++ b/tests/local_testing/test_add_function_to_prompt.py @@ -4,9 +4,6 @@ import sys, os, pytest import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/local_testing/test_aim_guardrails.py b/tests/local_testing/test_aim_guardrails.py index a6a4a0ad781..2a179ddcf32 100644 --- a/tests/local_testing/test_aim_guardrails.py +++ b/tests/local_testing/test_aim_guardrails.py @@ -1,8 +1,6 @@ import asyncio import contextlib import json -import os -import sys from unittest.mock import AsyncMock, patch, call import pytest @@ -17,9 +15,6 @@ from litellm.proxy.guardrails.guardrail_hooks.aim.aim import ( from litellm.proxy.proxy_server import StreamingCallbackError, UserAPIKeyAuth from litellm.types.utils import ModelResponseStream, ModelResponse -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 diff --git a/tests/local_testing/test_alangfuse.py b/tests/local_testing/test_alangfuse.py index 7c2ec7e9f64..7b1f7f203e3 100644 --- a/tests/local_testing/test_alangfuse.py +++ b/tests/local_testing/test_alangfuse.py @@ -3,12 +3,10 @@ import copy import json import logging import os -import sys from typing import Any, Optional from unittest.mock import MagicMock, patch logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index a52b5975f6e..76ff23a9a1b 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -1,5 +1,4 @@ import os -import sys import traceback from dotenv import load_dotenv @@ -9,12 +8,8 @@ import io from test_streaming import streaming_format_tests -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import json -import os import tempfile from unittest.mock import AsyncMock, MagicMock, patch, ANY from respx import MockRouter diff --git a/tests/local_testing/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py index 3105c0b9eeb..904b3ead92d 100644 --- a/tests/local_testing/test_anthropic_prompt_caching.py +++ b/tests/local_testing/test_anthropic_prompt_caching.py @@ -1,6 +1,5 @@ import json import os -import sys import traceback from dotenv import load_dotenv @@ -10,11 +9,7 @@ import io from test_streaming import streaming_format_tests -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from unittest.mock import AsyncMock, MagicMock, patch import pytest diff --git a/tests/local_testing/test_assistants.py b/tests/local_testing/test_assistants.py index 8dc4f9e48e1..af40e2f62b0 100644 --- a/tests/local_testing/test_assistants.py +++ b/tests/local_testing/test_assistants.py @@ -1,5 +1,3 @@ -import os -import sys import pytest from dotenv import load_dotenv @@ -7,7 +5,6 @@ from openai.types.beta.assistant import Assistant from openai.types.beta.assistant_deleted import AssistantDeleted load_dotenv() -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import create_thread, get_thread diff --git a/tests/local_testing/test_async_fn.py b/tests/local_testing/test_async_fn.py index 40a757a4874..e2b3a62bd28 100644 --- a/tests/local_testing/test_async_fn.py +++ b/tests/local_testing/test_async_fn.py @@ -3,15 +3,10 @@ import asyncio import logging -import os -import sys import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import acompletion, acreate, completion diff --git a/tests/local_testing/test_auth_utils.py b/tests/local_testing/test_auth_utils.py index e1444ed562e..0cc52716ce1 100644 --- a/tests/local_testing/test_auth_utils.py +++ b/tests/local_testing/test_auth_utils.py @@ -7,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm.proxy.auth.auth_utils import ( diff --git a/tests/local_testing/test_azure_openai.py b/tests/local_testing/test_azure_openai.py index 2a2b1e7fc35..d6e08552697 100644 --- a/tests/local_testing/test_azure_openai.py +++ b/tests/local_testing/test_azure_openai.py @@ -1,6 +1,5 @@ import json import os -import sys import traceback from dotenv import load_dotenv @@ -8,11 +7,7 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/local_testing/test_basic_python_version.py b/tests/local_testing/test_basic_python_version.py index a710b5e0ff7..fb06ed6b69d 100644 --- a/tests/local_testing/test_basic_python_version.py +++ b/tests/local_testing/test_basic_python_version.py @@ -1,7 +1,6 @@ import asyncio import os import subprocess -import sys import time import traceback @@ -9,9 +8,6 @@ import pytest PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path def _run_uv(*args: str, **kwargs) -> subprocess.CompletedProcess: diff --git a/tests/local_testing/test_batch_completions.py b/tests/local_testing/test_batch_completions.py index 95bfe5e6e2b..d3296988e8c 100644 --- a/tests/local_testing/test_batch_completions.py +++ b/tests/local_testing/test_batch_completions.py @@ -5,9 +5,6 @@ import sys, os import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from openai import APITimeoutError as Timeout import litellm diff --git a/tests/local_testing/test_blocked_user_list.py b/tests/local_testing/test_blocked_user_list.py index 9b29d3fcfa5..9bbe3fedf46 100644 --- a/tests/local_testing/test_blocked_user_list.py +++ b/tests/local_testing/test_blocked_user_list.py @@ -5,7 +5,6 @@ import asyncio import os import random -import sys import time import traceback from datetime import datetime @@ -15,9 +14,6 @@ from fastapi import Request load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import logging import pytest diff --git a/tests/local_testing/test_braintrust.py b/tests/local_testing/test_braintrust.py index 18c210b6d33..4c1a2d990b1 100644 --- a/tests/local_testing/test_braintrust.py +++ b/tests/local_testing/test_braintrust.py @@ -2,9 +2,7 @@ ## This tests the braintrust integration import asyncio -import os import random -import sys import time import traceback from datetime import datetime @@ -14,9 +12,6 @@ from fastapi import Request load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import logging from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index 90be551ff46..f9deb9c100b 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -1,5 +1,4 @@ import os -import sys import time import traceback from litellm._uuid import uuid @@ -9,9 +8,6 @@ from dotenv import load_dotenv load_dotenv() import json -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import hashlib import random diff --git a/tests/local_testing/test_caching_handler.py b/tests/local_testing/test_caching_handler.py index b26334e9ee0..f17a058b3fe 100644 --- a/tests/local_testing/test_caching_handler.py +++ b/tests/local_testing/test_caching_handler.py @@ -1,5 +1,3 @@ -import os -import sys import time import traceback from litellm._uuid import uuid @@ -7,9 +5,6 @@ from litellm._uuid import uuid from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import hashlib import random diff --git a/tests/local_testing/test_caching_ssl.py b/tests/local_testing/test_caching_ssl.py index 863f227aef1..a8fe45b2d7b 100644 --- a/tests/local_testing/test_caching_ssl.py +++ b/tests/local_testing/test_caching_ssl.py @@ -8,9 +8,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm import embedding, completion, Router diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 3b890273ce7..ef8d6c55148 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1,6 +1,5 @@ import json import os -import sys import traceback from dotenv import load_dotenv @@ -8,12 +7,8 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from unittest.mock import AsyncMock, MagicMock, patch import pytest diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 7dfcb55e29a..f47b40f2ef1 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -1,14 +1,9 @@ import os -import sys import traceback import litellm.cost_calculator -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio -import os import time from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/local_testing/test_completion_with_retries.py b/tests/local_testing/test_completion_with_retries.py index c9b519b2af8..ede07a15225 100644 --- a/tests/local_testing/test_completion_with_retries.py +++ b/tests/local_testing/test_completion_with_retries.py @@ -4,9 +4,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import openai import litellm diff --git a/tests/local_testing/test_config.py b/tests/local_testing/test_config.py index 2a5dc3376ee..6c3c0a093a7 100644 --- a/tests/local_testing/test_config.py +++ b/tests/local_testing/test_config.py @@ -3,7 +3,6 @@ import os -import sys import traceback from dotenv import load_dotenv @@ -11,9 +10,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Literal import pytest diff --git a/tests/local_testing/test_cost_calc.py b/tests/local_testing/test_cost_calc.py index 233b67a6072..0b2e8e39701 100644 --- a/tests/local_testing/test_cost_calc.py +++ b/tests/local_testing/test_cost_calc.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from dotenv import load_dotenv @@ -7,9 +5,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from typing import Literal import pytest diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index cedb5ea1a97..745bfe94e1a 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -3,7 +3,6 @@ import asyncio import inspect import os -import sys import traceback from litellm._uuid import uuid from datetime import datetime @@ -11,7 +10,6 @@ from datetime import datetime import pytest from pydantic import BaseModel -sys.path.insert(0, os.path.abspath("../..")) from typing import List, Literal, Optional, Union from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/local_testing/test_custom_llm.py b/tests/local_testing/test_custom_llm.py index 64a6c8b2587..160d771004c 100644 --- a/tests/local_testing/test_custom_llm.py +++ b/tests/local_testing/test_custom_llm.py @@ -3,18 +3,12 @@ import asyncio -import os -import sys import time import traceback import openai import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from typing import ( diff --git a/tests/local_testing/test_custom_logger.py b/tests/local_testing/test_custom_logger.py index 02a9eaaa9e6..1b627d56717 100644 --- a/tests/local_testing/test_custom_logger.py +++ b/tests/local_testing/test_custom_logger.py @@ -2,13 +2,11 @@ import asyncio import inspect import os -import sys import time import traceback import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion, embedding diff --git a/tests/local_testing/test_dual_cache.py b/tests/local_testing/test_dual_cache.py index cdfa8146420..e60fa5f3746 100644 --- a/tests/local_testing/test_dual_cache.py +++ b/tests/local_testing/test_dual_cache.py @@ -1,5 +1,4 @@ import os -import sys import time import traceback from litellm._uuid import uuid @@ -8,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import hashlib import random diff --git a/tests/local_testing/test_dynamic_rate_limit_handler.py b/tests/local_testing/test_dynamic_rate_limit_handler.py index fe3c8ca260e..7c178113e35 100644 --- a/tests/local_testing/test_dynamic_rate_limit_handler.py +++ b/tests/local_testing/test_dynamic_rate_limit_handler.py @@ -1,9 +1,7 @@ # What is this? ## Unit tests for 'dynamic_rate_limiter.py` import asyncio -import os import random -import sys import time import traceback from litellm._uuid import uuid @@ -14,9 +12,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index ee9d4cdd915..aed2849f056 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -1,6 +1,6 @@ import json import os -import sys +import re import traceback import openai @@ -9,9 +9,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch import litellm @@ -536,13 +533,19 @@ def test_demo_tokens_as_input_to_embeddings_fails_for_titan(): with pytest.raises( litellm.BadRequestError, - match='litellm.BadRequestError: BedrockException - {"message":"Malformed input request: expected type: String, found: JSONArray, please reformat your input and try again."}', + match=re.escape( + 'litellm.BadRequestError: BedrockException - {"message":"Malformed input request: ' + 'expected type: String, found: JSONArray, please reformat your input and try again."}' + ), ): litellm.embedding(model="amazon.titan-embed-text-v1", input=[[1]]) with pytest.raises( litellm.BadRequestError, - match='litellm.BadRequestError: BedrockException - {"message":"Malformed input request: expected type: String, found: Integer, please reformat your input and try again."}', + match=re.escape( + 'litellm.BadRequestError: BedrockException - {"message":"Malformed input request: ' + 'expected type: String, found: Integer, please reformat your input and try again."}' + ), ): litellm.embedding( model="amazon.titan-embed-text-v1", diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index cf89e7bea1d..8370046446d 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1,17 +1,14 @@ import asyncio import os import subprocess -import sys import traceback from typing import Any -from openai import AuthenticationError, BadRequestError, OpenAIError, RateLimitError +import httpx +from openai import AsyncOpenAI, AuthenticationError, BadRequestError, OpenAIError, RateLimitError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from concurrent.futures import ThreadPoolExecutor from unittest.mock import MagicMock, patch @@ -63,23 +60,38 @@ async def test_content_policy_exception_azure(): @pytest.mark.asyncio async def test_content_policy_exception_openai(): - # this is ony a test - we needed some way to invoke the exception :( - litellm.set_verbose = True + def reject_as_safety_system(request: httpx.Request) -> httpx.Response: + return httpx.Response( + status_code=400, + json={ + "error": { + "message": "Your request was rejected as a result of our safety system.", + "type": "invalid_request_error", + "param": None, + "code": "content_policy_violation", + } + }, + request=request, + ) - async def stream_response(): + async def stream_response(rejecting_client: AsyncOpenAI): response = await litellm.acompletion( model="gpt-3.5-turbo", stream=True, - messages=[ - {"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"} - ], + messages=[{"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"}], + client=rejecting_client, ) async for chunk in response: print(chunk) - with pytest.raises(litellm.ContentPolicyViolationError) as exc_info: - await stream_response() + async with AsyncOpenAI( + api_key="sk-test", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(reject_as_safety_system)), + ) as rejecting_client: + with pytest.raises(litellm.ContentPolicyViolationError) as exc_info: + await stream_response(rejecting_client) assert exc_info.value.llm_provider == "openai" + assert exc_info.value.status_code == 400 # Test 1: Context Window Errors @@ -871,7 +883,7 @@ def test_anthropic_tool_calling_exception(): from typing import Optional, Union -from openai import AsyncOpenAI, OpenAI +from openai import OpenAI def _pre_call_utils( diff --git a/tests/local_testing/test_function_call_parsing.py b/tests/local_testing/test_function_call_parsing.py index 57027c670bb..c98f170a98f 100644 --- a/tests/local_testing/test_function_call_parsing.py +++ b/tests/local_testing/test_function_call_parsing.py @@ -1,7 +1,5 @@ # What is this? ## Test to make sure function call response always works with json.loads() -> no extra parsing required. Relevant issue - https://github.com/BerriAI/litellm/issues/2654 -import os -import sys import traceback from dotenv import load_dotenv @@ -9,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import json import warnings from typing import List diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index b5f72264549..5752f29daef 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from dotenv import load_dotenv @@ -7,9 +5,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest from unittest.mock import patch, MagicMock, AsyncMock import litellm diff --git a/tests/local_testing/test_function_setup.py b/tests/local_testing/test_function_setup.py index 92f49589ca2..757aaefc8c6 100644 --- a/tests/local_testing/test_function_setup.py +++ b/tests/local_testing/test_function_setup.py @@ -7,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest, uuid from litellm.utils import function_setup, Rules from litellm.litellm_core_utils.prompt_templates.factory import ( diff --git a/tests/local_testing/test_gcs_bucket.py b/tests/local_testing/test_gcs_bucket.py index ffd466aa809..437a8b8f13b 100644 --- a/tests/local_testing/test_gcs_bucket.py +++ b/tests/local_testing/test_gcs_bucket.py @@ -1,8 +1,6 @@ import io import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import json diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index 0e667b82a66..cc6209f2bf9 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -1,5 +1,4 @@ import os -import sys import traceback from dotenv import load_dotenv @@ -9,9 +8,6 @@ import io from unittest.mock import patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm.types.router import LiteLLM_Params diff --git a/tests/local_testing/test_get_model_file.py b/tests/local_testing/test_get_model_file.py index 17bd2d7ceff..3742dca9dda 100644 --- a/tests/local_testing/test_get_model_file.py +++ b/tests/local_testing/test_get_model_file.py @@ -2,9 +2,6 @@ import os, sys, traceback import importlib.resources import json -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index cef05050ac9..2de83778f1c 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -1,16 +1,12 @@ # What is this? ## Unit testing for the 'get_model_info()' function import os -import sys import traceback import json from typing import List, Dict, Any -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest import litellm diff --git a/tests/local_testing/test_get_optional_params_embeddings.py b/tests/local_testing/test_get_optional_params_embeddings.py index ddf9e877477..60ccfbfaebe 100644 --- a/tests/local_testing/test_get_optional_params_embeddings.py +++ b/tests/local_testing/test_get_optional_params_embeddings.py @@ -7,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm import embedding diff --git a/tests/local_testing/test_google_ai_studio_gemini.py b/tests/local_testing/test_google_ai_studio_gemini.py index 5012717d383..43b64ded1ab 100644 --- a/tests/local_testing/test_google_ai_studio_gemini.py +++ b/tests/local_testing/test_google_ai_studio_gemini.py @@ -1,8 +1,5 @@ import os, sys, traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from dotenv import load_dotenv diff --git a/tests/local_testing/test_guardrails_ai.py b/tests/local_testing/test_guardrails_ai.py index 004ffa0b9e3..bc2db026ecc 100644 --- a/tests/local_testing/test_guardrails_ai.py +++ b/tests/local_testing/test_guardrails_ai.py @@ -1,10 +1,5 @@ -import os -import sys import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 diff --git a/tests/local_testing/test_helicone_integration.py b/tests/local_testing/test_helicone_integration.py index 9bfa29551e3..f34ad33aa9b 100644 --- a/tests/local_testing/test_helicone_integration.py +++ b/tests/local_testing/test_helicone_integration.py @@ -2,13 +2,11 @@ import asyncio import copy import logging import os -import sys import time from typing import Any from unittest.mock import MagicMock, patch logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion diff --git a/tests/local_testing/test_http_parsing_utils.py b/tests/local_testing/test_http_parsing_utils.py index 813460c7e27..db282d6d4be 100644 --- a/tests/local_testing/test_http_parsing_utils.py +++ b/tests/local_testing/test_http_parsing_utils.py @@ -3,12 +3,7 @@ from fastapi import Request from fastapi.testclient import TestClient from starlette.datastructures import Headers from starlette.requests import HTTPConnection -import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy._types import ProxyException diff --git a/tests/local_testing/test_least_busy_routing.py b/tests/local_testing/test_least_busy_routing.py index 0a3b5490131..18ab8bf779d 100644 --- a/tests/local_testing/test_least_busy_routing.py +++ b/tests/local_testing/test_least_busy_routing.py @@ -2,9 +2,7 @@ # This tests the router's ability to identify the least busy deployment import asyncio -import os import random -import sys import time import traceback @@ -12,9 +10,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm diff --git a/tests/local_testing/test_llm_guard.py b/tests/local_testing/test_llm_guard.py index 60fe9c0e020..9e70d48dbda 100644 --- a/tests/local_testing/test_llm_guard.py +++ b/tests/local_testing/test_llm_guard.py @@ -10,9 +10,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest from fastapi import HTTPException diff --git a/tests/local_testing/test_longer_context_fallback.py b/tests/local_testing/test_longer_context_fallback.py index 07e9e8cad74..adb087079c5 100644 --- a/tests/local_testing/test_longer_context_fallback.py +++ b/tests/local_testing/test_longer_context_fallback.py @@ -5,9 +5,6 @@ import sys, os import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import longer_context_model_fallback_dict diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index 6ed1731572a..5bf3a3ee98b 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -9,9 +9,6 @@ from dotenv import load_dotenv load_dotenv() import copy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest from litellm import Router from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler diff --git a/tests/local_testing/test_lowest_latency_routing.py b/tests/local_testing/test_lowest_latency_routing.py index 0a202e0dfb9..598b1dbcaf9 100644 --- a/tests/local_testing/test_lowest_latency_routing.py +++ b/tests/local_testing/test_lowest_latency_routing.py @@ -2,7 +2,6 @@ # This tests the router's ability to pick deployment with lowest latency import asyncio -import os import random import sys import time @@ -14,9 +13,6 @@ from dotenv import load_dotenv load_dotenv() import copy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm diff --git a/tests/local_testing/test_lunary.py b/tests/local_testing/test_lunary.py index 0dbae1b817f..a2e137ed355 100644 --- a/tests/local_testing/test_lunary.py +++ b/tests/local_testing/test_lunary.py @@ -1,8 +1,5 @@ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion diff --git a/tests/local_testing/test_mock_request.py b/tests/local_testing/test_mock_request.py index c9cd14633ba..9cbcafb003b 100644 --- a/tests/local_testing/test_mock_request.py +++ b/tests/local_testing/test_mock_request.py @@ -2,14 +2,10 @@ # This tests mock request calls to litellm import os -import sys import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import time diff --git a/tests/local_testing/test_model_alias_map.py b/tests/local_testing/test_model_alias_map.py index 9ef0448e7c6..675f2345747 100644 --- a/tests/local_testing/test_model_alias_map.py +++ b/tests/local_testing/test_model_alias_map.py @@ -1,13 +1,8 @@ #### What this tests #### # This tests the model alias mapping - if user passes in an alias, and has set an alias, set it to the actual value -import os -import sys import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm diff --git a/tests/local_testing/test_multiple_deployments.py b/tests/local_testing/test_multiple_deployments.py index 72bfd5012c1..1c39bd56a95 100644 --- a/tests/local_testing/test_multiple_deployments.py +++ b/tests/local_testing/test_multiple_deployments.py @@ -4,9 +4,6 @@ import sys, os import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm import completion diff --git a/tests/local_testing/test_ollama.py b/tests/local_testing/test_ollama.py index 7ca8e806529..ad5d7d86501 100644 --- a/tests/local_testing/test_ollama.py +++ b/tests/local_testing/test_ollama.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys import traceback from dotenv import load_dotenv @@ -9,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest import mock import pytest diff --git a/tests/local_testing/test_openai_moderations_hook.py b/tests/local_testing/test_openai_moderations_hook.py index 944ac047e55..530ab714eae 100644 --- a/tests/local_testing/test_openai_moderations_hook.py +++ b/tests/local_testing/test_openai_moderations_hook.py @@ -10,9 +10,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm.proxy.enterprise.enterprise_hooks.openai_moderation import ( @@ -62,7 +59,9 @@ async def test_openai_moderation_error_raising(monkeypatch): llm_router.amoderation = mock_amoderation - setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", llm_router) with pytest.raises(Exception, match="Violated content safety policy") as exc_info: await openai_mod.async_moderation_hook( diff --git a/tests/local_testing/test_opik.py b/tests/local_testing/test_opik.py index 4047a5fefe3..8be4b796360 100644 --- a/tests/local_testing/test_opik.py +++ b/tests/local_testing/test_opik.py @@ -1,8 +1,6 @@ import io import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import logging diff --git a/tests/local_testing/test_pass_through_endpoints.py b/tests/local_testing/test_pass_through_endpoints.py index 793a60efc3f..618354ca31e 100644 --- a/tests/local_testing/test_pass_through_endpoints.py +++ b/tests/local_testing/test_pass_through_endpoints.py @@ -1,5 +1,4 @@ import os -import sys from litellm._uuid import uuid from functools import partial from typing import Optional @@ -9,9 +8,6 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds-the parent directory to the system path import asyncio from unittest.mock import Mock diff --git a/tests/local_testing/test_prometheus_service.py b/tests/local_testing/test_prometheus_service.py index b97fcd096b3..c8acca83d93 100644 --- a/tests/local_testing/test_prometheus_service.py +++ b/tests/local_testing/test_prometheus_service.py @@ -2,11 +2,9 @@ ## Unit Tests for prometheus service monitoring import json -import sys import os import io, asyncio -sys.path.insert(0, os.path.abspath("../..")) import pytest from litellm import acompletion, Cache from litellm._service_logger import ServiceLogging diff --git a/tests/local_testing/test_prompt_caching.py b/tests/local_testing/test_prompt_caching.py index 58b8f560045..f6b3fb89e9e 100644 --- a/tests/local_testing/test_prompt_caching.py +++ b/tests/local_testing/test_prompt_caching.py @@ -1,10 +1,7 @@ """Asserts that prompt caching information is correctly returned for Anthropic, OpenAI, and Deepseek""" import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import litellm import pytest diff --git a/tests/local_testing/test_prompt_injection_detection.py b/tests/local_testing/test_prompt_injection_detection.py index 9f5137630ea..fa35dc5b060 100644 --- a/tests/local_testing/test_prompt_injection_detection.py +++ b/tests/local_testing/test_prompt_injection_detection.py @@ -8,9 +8,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm.proxy.hooks.prompt_injection_detection import ( diff --git a/tests/local_testing/test_provider_specific_config.py b/tests/local_testing/test_provider_specific_config.py index 5587087e40b..a6bad688201 100644 --- a/tests/local_testing/test_provider_specific_config.py +++ b/tests/local_testing/test_provider_specific_config.py @@ -3,14 +3,10 @@ # There are 2 types of tests - changing config dynamically or by setting class variables import os -import sys import traceback import json import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch import litellm diff --git a/tests/local_testing/test_pydantic.py b/tests/local_testing/test_pydantic.py index 436b9d3dd48..155b0345186 100644 --- a/tests/local_testing/test_pydantic.py +++ b/tests/local_testing/test_pydantic.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from dotenv import load_dotenv @@ -7,12 +5,8 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import json -import os import tempfile from unittest.mock import MagicMock, patch diff --git a/tests/local_testing/test_redis_batch_optimizations.py b/tests/local_testing/test_redis_batch_optimizations.py index 4997157bac8..d49939cff1a 100644 --- a/tests/local_testing/test_redis_batch_optimizations.py +++ b/tests/local_testing/test_redis_batch_optimizations.py @@ -8,7 +8,6 @@ Verifies: """ import os -import sys import time from unittest.mock import AsyncMock, patch @@ -16,7 +15,6 @@ import pytest from dotenv import load_dotenv load_dotenv() -sys.path.insert(0, os.path.abspath("../..")) import uuid from litellm.caching.dual_cache import DualCache diff --git a/tests/local_testing/test_register_model.py b/tests/local_testing/test_register_model.py index 44fb440bbbd..eddd697974c 100644 --- a/tests/local_testing/test_register_model.py +++ b/tests/local_testing/test_register_model.py @@ -8,9 +8,6 @@ from pathlib import Path import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index f648b31901a..370c43f8f44 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -3,7 +3,6 @@ import asyncio import os -import sys import time import traceback @@ -13,10 +12,6 @@ import pytest import litellm.types import litellm.types.router -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/local_testing/test_router_batch_completion.py b/tests/local_testing/test_router_batch_completion.py index bb9e1851c61..6fd89065c1d 100644 --- a/tests/local_testing/test_router_batch_completion.py +++ b/tests/local_testing/test_router_batch_completion.py @@ -2,18 +2,12 @@ # This tests litellm router with batch completion import asyncio -import os -import sys import time import traceback import openai import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor diff --git a/tests/local_testing/test_router_budget_limiter.py b/tests/local_testing/test_router_budget_limiter.py index 3bdb3116670..bda1f648076 100644 --- a/tests/local_testing/test_router_budget_limiter.py +++ b/tests/local_testing/test_router_budget_limiter.py @@ -6,9 +6,6 @@ from dotenv import load_dotenv load_dotenv() import copy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest from litellm import Router from litellm.router_strategy.budget_limiter import RouterBudgetLimiting diff --git a/tests/local_testing/test_router_caching.py b/tests/local_testing/test_router_caching.py index cb223b661b4..9675a1299d1 100644 --- a/tests/local_testing/test_router_caching.py +++ b/tests/local_testing/test_router_caching.py @@ -2,16 +2,12 @@ # This tests caching on the router import asyncio import os -import sys import time import traceback from unittest.mock import patch from typing import Union import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import Router from litellm.caching import RedisCache, RedisClusterCache diff --git a/tests/local_testing/test_router_client_init.py b/tests/local_testing/test_router_client_init.py index f2b82b651dd..f27b3848beb 100644 --- a/tests/local_testing/test_router_client_init.py +++ b/tests/local_testing/test_router_client_init.py @@ -6,7 +6,6 @@ import os #### What this tests #### # This tests caching on the router -import sys import time import traceback from typing import Dict @@ -15,9 +14,6 @@ from unittest.mock import MagicMock, PropertyMock, patch import pytest from openai.lib.azure import OpenAIError -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import APIConnectionError, Router from unittest.mock import ANY diff --git a/tests/local_testing/test_router_cooldown_handlers.py b/tests/local_testing/test_router_cooldown_handlers.py index 55510df5b9e..e1e3df1e4a5 100644 --- a/tests/local_testing/test_router_cooldown_handlers.py +++ b/tests/local_testing/test_router_cooldown_handlers.py @@ -4,15 +4,11 @@ import asyncio import os import random -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/local_testing/test_router_custom_routing.py b/tests/local_testing/test_router_custom_routing.py index 3ebd79a7b2a..bd624f7a19f 100644 --- a/tests/local_testing/test_router_custom_routing.py +++ b/tests/local_testing/test_router_custom_routing.py @@ -1,15 +1,10 @@ import asyncio -import os -import sys import time from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Dict, List, Optional, Union import pytest diff --git a/tests/local_testing/test_router_debug_logs.py b/tests/local_testing/test_router_debug_logs.py index 04e8dc6c77c..0fce5c824c7 100644 --- a/tests/local_testing/test_router_debug_logs.py +++ b/tests/local_testing/test_router_debug_logs.py @@ -1,14 +1,10 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import logging diff --git a/tests/local_testing/test_router_fallback_handlers.py b/tests/local_testing/test_router_fallback_handlers.py index 0bd455463b7..65994f0a4cf 100644 --- a/tests/local_testing/test_router_fallback_handlers.py +++ b/tests/local_testing/test_router_fallback_handlers.py @@ -1,14 +1,10 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch import litellm diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index 1cafd2c709d..82b832f89fd 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -3,15 +3,11 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch import litellm diff --git a/tests/local_testing/test_router_get_deployments.py b/tests/local_testing/test_router_get_deployments.py index 78503b36c74..a4d4359a3e9 100644 --- a/tests/local_testing/test_router_get_deployments.py +++ b/tests/local_testing/test_router_get_deployments.py @@ -3,15 +3,11 @@ # These are fast Tests, and make no API calls import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from collections import defaultdict from concurrent.futures import ThreadPoolExecutor diff --git a/tests/local_testing/test_router_max_parallel_requests.py b/tests/local_testing/test_router_max_parallel_requests.py index 7bb40dd7a2f..65602c968bc 100644 --- a/tests/local_testing/test_router_max_parallel_requests.py +++ b/tests/local_testing/test_router_max_parallel_requests.py @@ -2,15 +2,12 @@ ## Unit tests for the max_parallel_requests feature on Router import asyncio import inspect -import os -import sys import time import traceback from datetime import datetime import pytest -sys.path.insert(0, os.path.abspath("../..")) from typing import Optional import litellm diff --git a/tests/local_testing/test_router_pattern_matching.py b/tests/local_testing/test_router_pattern_matching.py index d02582a2a99..6ffc5316f2e 100644 --- a/tests/local_testing/test_router_pattern_matching.py +++ b/tests/local_testing/test_router_pattern_matching.py @@ -9,9 +9,6 @@ import json import traceback, asyncio import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import Router from litellm.router import Deployment, LiteLLM_Params diff --git a/tests/local_testing/test_router_retries.py b/tests/local_testing/test_router_retries.py index 7d1ad012745..d5374a3da0f 100644 --- a/tests/local_testing/test_router_retries.py +++ b/tests/local_testing/test_router_retries.py @@ -3,15 +3,11 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx import openai diff --git a/tests/local_testing/test_router_timeout.py b/tests/local_testing/test_router_timeout.py index 9971e540024..9992fa03bcd 100644 --- a/tests/local_testing/test_router_timeout.py +++ b/tests/local_testing/test_router_timeout.py @@ -3,18 +3,13 @@ import asyncio import os -import sys import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import patch, MagicMock, AsyncMock -import os from dotenv import load_dotenv diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index f2fd2fdf559..45fe42f4cd3 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -5,9 +5,6 @@ import sys, os, time import traceback, asyncio import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import Router from litellm.router import Deployment, LiteLLM_Params diff --git a/tests/local_testing/test_rules.py b/tests/local_testing/test_rules.py index b075821e205..2e9472c8678 100644 --- a/tests/local_testing/test_rules.py +++ b/tests/local_testing/test_rules.py @@ -1,16 +1,12 @@ #### What this tests #### # This tests setting rules before / after making llm api calls import asyncio -import os -import sys +import re import time import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import acompletion, completion @@ -82,7 +78,7 @@ def test_post_call_rule(): litellm.post_call_rules = [my_post_call_rule] ### completion - with pytest.raises(Exception, match="This violates LiteLLM Proxy Rules. Response too short") as exc_info: + with pytest.raises(Exception, match=re.escape("This violates LiteLLM Proxy Rules. Response too short")) as exc_info: completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "say sorry"}], @@ -118,7 +114,7 @@ def test_post_call_rule_streaming(): stream=True, ) - with pytest.raises(Exception, match="This violates LiteLLM Proxy Rules. Response too short") as exc_info: + with pytest.raises(Exception, match=re.escape("This violates LiteLLM Proxy Rules. Response too short")) as exc_info: list(response) assert "This violates LiteLLM Proxy Rules. Response too short" in exc_info.value.message diff --git a/tests/local_testing/test_sagemaker.py b/tests/local_testing/test_sagemaker.py index bf17d9dce21..a01c8c217c6 100644 --- a/tests/local_testing/test_sagemaker.py +++ b/tests/local_testing/test_sagemaker.py @@ -1,6 +1,4 @@ import json -import os -import sys import traceback from dotenv import load_dotenv @@ -10,11 +8,7 @@ import io import litellm from test_streaming import streaming_format_tests -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os from unittest.mock import AsyncMock, MagicMock, patch import pytest diff --git a/tests/local_testing/test_scheduler.py b/tests/local_testing/test_scheduler.py index 178983f02d6..027a400dfc9 100644 --- a/tests/local_testing/test_scheduler.py +++ b/tests/local_testing/test_scheduler.py @@ -6,9 +6,6 @@ import traceback, asyncio import pytest from typing import List -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm import Router from litellm.scheduler import FlowItem, Scheduler, SchedulerCacheKeys from litellm import ModelResponse diff --git a/tests/local_testing/test_secret_detect_hook.py b/tests/local_testing/test_secret_detect_hook.py index 8a93b72dce2..0ee0f596177 100644 --- a/tests/local_testing/test_secret_detect_hook.py +++ b/tests/local_testing/test_secret_detect_hook.py @@ -2,12 +2,10 @@ ## This tests the llm guard integration import asyncio -import os import random # What is this? ## Unit test for presidio pii masking -import sys import time import traceback from datetime import datetime @@ -16,9 +14,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest from fastapi import Request, Response from starlette.datastructures import URL diff --git a/tests/local_testing/test_spend_calculate_endpoint.py b/tests/local_testing/test_spend_calculate_endpoint.py index 8f7434e40b9..3bedab794e2 100644 --- a/tests/local_testing/test_spend_calculate_endpoint.py +++ b/tests/local_testing/test_spend_calculate_endpoint.py @@ -1,5 +1,3 @@ -import os -import sys import pytest from dotenv import load_dotenv @@ -13,9 +11,6 @@ from litellm.router import Router # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path @pytest.mark.asyncio diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 9dab6e60c35..6d62dd52b89 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -1,6 +1,5 @@ import asyncio import os -import sys import time import traceback @@ -15,13 +14,9 @@ def check_non_streaming_response(response): assert isinstance( response.choices[0].message.audio, ChatCompletionAudioResponse ), "Invalid audio response type" - assert len(completion.choices[0].message.audio.data) > 0, "Audio data is empty" + assert len(response.choices[0].message.audio.data) > 0, "Audio data is empty" -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import os import dotenv from openai import OpenAI diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index ba1f4e7d51c..07d693af447 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -4,7 +4,6 @@ import asyncio import json import os -import sys import time import traceback from litellm._uuid import uuid @@ -19,9 +18,6 @@ import litellm.litellm_core_utils.litellm_logging from litellm.utils import ModelResponseListIterator from litellm.types.utils import ModelResponseStream -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from dotenv import load_dotenv load_dotenv() diff --git a/tests/local_testing/test_supabase_integration.py b/tests/local_testing/test_supabase_integration.py index 96d2889a795..5331de86303 100644 --- a/tests/local_testing/test_supabase_integration.py +++ b/tests/local_testing/test_supabase_integration.py @@ -4,9 +4,6 @@ import sys, os import traceback import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import embedding, completion diff --git a/tests/local_testing/test_text_completion.py b/tests/local_testing/test_text_completion.py index 227d8e5096a..a814ce6d303 100644 --- a/tests/local_testing/test_text_completion.py +++ b/tests/local_testing/test_text_completion.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys import traceback from dotenv import load_dotenv @@ -9,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch import pytest diff --git a/tests/local_testing/test_timeout.py b/tests/local_testing/test_timeout.py index 6b490f1cef2..66054a0930a 100644 --- a/tests/local_testing/test_timeout.py +++ b/tests/local_testing/test_timeout.py @@ -2,12 +2,8 @@ # This tests the timeout decorator import os -import sys import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import time from litellm._uuid import uuid diff --git a/tests/local_testing/test_tpm_rpm_routing_v2.py b/tests/local_testing/test_tpm_rpm_routing_v2.py index c6917775d4b..7478bd253b6 100644 --- a/tests/local_testing/test_tpm_rpm_routing_v2.py +++ b/tests/local_testing/test_tpm_rpm_routing_v2.py @@ -4,7 +4,6 @@ import asyncio import os import random -import sys import time import traceback from datetime import datetime @@ -13,9 +12,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch from litellm.types.utils import StandardLoggingPayload import pytest diff --git a/tests/local_testing/test_ui_sso_helper_utils.py b/tests/local_testing/test_ui_sso_helper_utils.py index c7206363278..bb446c54738 100644 --- a/tests/local_testing/test_ui_sso_helper_utils.py +++ b/tests/local_testing/test_ui_sso_helper_utils.py @@ -3,9 +3,7 @@ import asyncio -import os import random -import sys import time import traceback from datetime import datetime @@ -15,9 +13,6 @@ from fastapi import Request load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import logging from litellm.proxy.management_endpoints.sso_helper_utils import ( diff --git a/tests/local_testing/test_unit_test_caching.py b/tests/local_testing/test_unit_test_caching.py index e25b75e658f..fd9f4bb9e89 100644 --- a/tests/local_testing/test_unit_test_caching.py +++ b/tests/local_testing/test_unit_test_caching.py @@ -1,5 +1,3 @@ -import os -import sys import time import traceback from litellm._uuid import uuid @@ -7,9 +5,6 @@ from litellm._uuid import uuid from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import hashlib import random diff --git a/tests/local_testing/test_update_spend.py b/tests/local_testing/test_update_spend.py index 7894f330796..b492a752c2c 100644 --- a/tests/local_testing/test_update_spend.py +++ b/tests/local_testing/test_update_spend.py @@ -5,7 +5,6 @@ import asyncio import os import random -import sys import time import traceback from datetime import datetime @@ -15,9 +14,6 @@ from fastapi import Request load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import logging import pytest diff --git a/tests/local_testing/test_validate_environment.py b/tests/local_testing/test_validate_environment.py index dce61b3abbb..289c2bb7c99 100644 --- a/tests/local_testing/test_validate_environment.py +++ b/tests/local_testing/test_validate_environment.py @@ -4,9 +4,6 @@ import sys, os import traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import time import litellm diff --git a/tests/local_testing/test_wandb.py b/tests/local_testing/test_wandb.py index 58a9c9f5ddf..02ab2787cf3 100644 --- a/tests/local_testing/test_wandb.py +++ b/tests/local_testing/test_wandb.py @@ -1,10 +1,8 @@ -import sys import os import io, asyncio # import logging # logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) from litellm import completion import litellm diff --git a/tests/logging_callback_tests/base_test.py b/tests/logging_callback_tests/base_test.py index 0d1e7dfcf77..68faf4bdb35 100644 --- a/tests/logging_callback_tests/base_test.py +++ b/tests/logging_callback_tests/base_test.py @@ -2,14 +2,9 @@ import asyncio import httpx import json import pytest -import sys from typing import Any, Dict, List from unittest.mock import MagicMock, Mock, patch -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index dedff9a5aee..66d0ee01f8e 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -10,13 +10,9 @@ import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from tests._vcr_conftest_common import ( # noqa: E402,F401 @@ -180,7 +176,6 @@ def setup_and_teardown(): Module-scoped setup. Reloads litellm only in single-process mode (skipped under xdist to avoid cross-worker interference). """ - sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/logging_callback_tests/create_mock_standard_logging_payload.py b/tests/logging_callback_tests/create_mock_standard_logging_payload.py index 106328e95e2..096c8ff8c60 100644 --- a/tests/logging_callback_tests/create_mock_standard_logging_payload.py +++ b/tests/logging_callback_tests/create_mock_standard_logging_payload.py @@ -1,9 +1,6 @@ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/logging_callback_tests/test_alerting.py b/tests/logging_callback_tests/test_alerting.py index 83513107ad3..3074e973a8e 100644 --- a/tests/logging_callback_tests/test_alerting.py +++ b/tests/logging_callback_tests/test_alerting.py @@ -6,7 +6,6 @@ import io import json import os import random -import sys import time from litellm._uuid import uuid from datetime import datetime, timedelta @@ -18,8 +17,6 @@ from litellm.types.integrations.slack_alerting import AlertType # import logging # logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) -import os import unittest.mock from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/logging_callback_tests/test_amazing_s3_logs.py b/tests/logging_callback_tests/test_amazing_s3_logs.py index 08b9ac7d01a..befc5ae3996 100644 --- a/tests/logging_callback_tests/test_amazing_s3_logs.py +++ b/tests/logging_callback_tests/test_amazing_s3_logs.py @@ -1,11 +1,8 @@ -import sys -import os import io, asyncio from collections import defaultdict # import logging # logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) from litellm import completion import litellm diff --git a/tests/logging_callback_tests/test_assemble_streaming_responses.py b/tests/logging_callback_tests/test_assemble_streaming_responses.py index 919b76e95a6..d6905ce3565 100644 --- a/tests/logging_callback_tests/test_assemble_streaming_responses.py +++ b/tests/logging_callback_tests/test_assemble_streaming_responses.py @@ -9,14 +9,9 @@ Testing for _assemble_complete_response_from_streaming_chunks """ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index d6d0652ed77..3f9f2bacdd3 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -1,9 +1,7 @@ import io import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py b/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py index 942c26438c8..53fe493ad9f 100644 --- a/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py +++ b/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from litellm._uuid import uuid import pytest @@ -14,9 +12,6 @@ import json # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import asyncio from typing import Optional diff --git a/tests/logging_callback_tests/test_custom_callback_router.py b/tests/logging_callback_tests/test_custom_callback_router.py index 70da10ffeeb..8cbe5fc6ccc 100644 --- a/tests/logging_callback_tests/test_custom_callback_router.py +++ b/tests/logging_callback_tests/test_custom_callback_router.py @@ -3,14 +3,12 @@ import asyncio import inspect import os -import sys import time import traceback from datetime import datetime import pytest -sys.path.insert(0, os.path.abspath("../..")) from typing import List, Literal, Optional from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index bc7a9a211a4..83a652e8884 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -1,6 +1,5 @@ import io import os -import sys from litellm.integrations.datadog.datadog_handler import ( get_datadog_source, @@ -11,7 +10,6 @@ from litellm.integrations.datadog.datadog_handler import ( get_datadog_tags, ) -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/logging_callback_tests/test_datadog_llm_obs.py b/tests/logging_callback_tests/test_datadog_llm_obs.py index 56aae7aa8bf..bed1a214b44 100644 --- a/tests/logging_callback_tests/test_datadog_llm_obs.py +++ b/tests/logging_callback_tests/test_datadog_llm_obs.py @@ -3,11 +3,8 @@ Test the DataDogLLMObsLogger """ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/logging_callback_tests/test_dynamic_otel_keys.py b/tests/logging_callback_tests/test_dynamic_otel_keys.py index 2a463fddc0d..f91f9b166ed 100644 --- a/tests/logging_callback_tests/test_dynamic_otel_keys.py +++ b/tests/logging_callback_tests/test_dynamic_otel_keys.py @@ -1,7 +1,4 @@ -import sys -import os -sys.path.insert(0, os.path.abspath("../..")) from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params, diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 2f6cdb63192..10957fa2f92 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -1,9 +1,7 @@ import io import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm @@ -42,6 +40,7 @@ ignored_keys = [ "metadata.cold_storage_object_key", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", + "metadata.autorouter_savings", "metadata.eval_information", ] diff --git a/tests/logging_callback_tests/test_generic_api_callback.py b/tests/logging_callback_tests/test_generic_api_callback.py index 9ad17b3d6e2..29d8f9e5694 100644 --- a/tests/logging_callback_tests/test_generic_api_callback.py +++ b/tests/logging_callback_tests/test_generic_api_callback.py @@ -1,9 +1,7 @@ import io import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import litellm diff --git a/tests/logging_callback_tests/test_humanloop_unit_tests.py b/tests/logging_callback_tests/test_humanloop_unit_tests.py index 9b45c24b81e..edea2098127 100644 --- a/tests/logging_callback_tests/test_humanloop_unit_tests.py +++ b/tests/logging_callback_tests/test_humanloop_unit_tests.py @@ -1,11 +1,6 @@ -import os -import sys import threading from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest from litellm.integrations.humanloop import HumanLoopPromptManager diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py index bc64e30738f..5682d3720d8 100644 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ b/tests/logging_callback_tests/test_langfuse_e2e_test.py @@ -3,7 +3,6 @@ import copy import json import logging import os -import sys import threading from typing import Any, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -11,7 +10,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 547e9d15f0b..1c25b169243 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -1,9 +1,5 @@ import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest from litellm.integrations.langfuse.langfuse import ( diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index 9cc1acd1ee4..17cd63d8974 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -1,9 +1,7 @@ import io import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip @@ -52,7 +50,6 @@ async def test_get_credentials_from_env(): assert credentials["LANGSMITH_TENANT_ID"] == "test-tenant-id" # Test tenant_id from environment variable - import os os.environ["LANGSMITH_TENANT_ID"] = "env-tenant-id" credentials = logger.get_credentials_from_env() diff --git a/tests/logging_callback_tests/test_log_db_redis_services.py b/tests/logging_callback_tests/test_log_db_redis_services.py index a8c3929be16..e3bc8383c46 100644 --- a/tests/logging_callback_tests/test_log_db_redis_services.py +++ b/tests/logging_callback_tests/test_log_db_redis_services.py @@ -1,8 +1,5 @@ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 3b42595b959..c754c7b8c2a 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -1,10 +1,7 @@ import io -import os -import sys from typing import Optional, Union -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/logging_callback_tests/test_moderations_api_logging.py b/tests/logging_callback_tests/test_moderations_api_logging.py index 9190f2aebe5..a2a356d3665 100644 --- a/tests/logging_callback_tests/test_moderations_api_logging.py +++ b/tests/logging_callback_tests/test_moderations_api_logging.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from litellm._uuid import uuid import pytest @@ -12,9 +10,6 @@ import io import time import json -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.router import Router import asyncio diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index 767f840a003..fcbd6dbc531 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -10,9 +10,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from unittest.mock import patch, MagicMock, AsyncMock diff --git a/tests/logging_callback_tests/test_otel_logging.py b/tests/logging_callback_tests/test_otel_logging.py index b6d7ef4be4e..ff85a320904 100644 --- a/tests/logging_callback_tests/test_otel_logging.py +++ b/tests/logging_callback_tests/test_otel_logging.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import pytest import litellm diff --git a/tests/logging_callback_tests/test_pagerduty_alerting.py b/tests/logging_callback_tests/test_pagerduty_alerting.py index 108a1ead1a4..1426dc32081 100644 --- a/tests/logging_callback_tests/test_pagerduty_alerting.py +++ b/tests/logging_callback_tests/test_pagerduty_alerting.py @@ -1,11 +1,8 @@ import asyncio -import os import random -import sys from datetime import datetime, timedelta from typing import Optional -sys.path.insert(0, os.path.abspath("../..")) import pytest import litellm diff --git a/tests/logging_callback_tests/test_posthog.py b/tests/logging_callback_tests/test_posthog.py index b3f346bcf9d..92bbc255730 100644 --- a/tests/logging_callback_tests/test_posthog.py +++ b/tests/logging_callback_tests/test_posthog.py @@ -1,7 +1,5 @@ import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import pytest diff --git a/tests/logging_callback_tests/test_spend_logs.py b/tests/logging_callback_tests/test_spend_logs.py index 709aa81f421..feecfc9f4ab 100644 --- a/tests/logging_callback_tests/test_spend_logs.py +++ b/tests/logging_callback_tests/test_spend_logs.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from litellm._uuid import uuid @@ -13,9 +11,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import datetime import json diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index 6a632c32fc2..da1fbbaa04f 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -3,14 +3,9 @@ Unit tests for StandardLoggingPayloadSetup """ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from datetime import datetime as dt_object import time import pytest diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py index 4088bdd2cf7..d8c45d832ce 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -13,15 +13,12 @@ Example config: standard_logging_payload_excluded_fields: ["response", "messages"] """ -import os -import sys from copy import deepcopy from typing import Dict, List, Optional from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/logging_callback_tests/test_token_counting.py b/tests/logging_callback_tests/test_token_counting.py index e2160076b00..c942a9d2686 100644 --- a/tests/logging_callback_tests/test_token_counting.py +++ b/tests/logging_callback_tests/test_token_counting.py @@ -1,5 +1,4 @@ import os -import sys import traceback from litellm._uuid import uuid import pytest @@ -14,9 +13,6 @@ import json # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import asyncio from typing import Optional diff --git a/tests/logging_callback_tests/test_unit_test_litellm_logging.py b/tests/logging_callback_tests/test_unit_test_litellm_logging.py index f82813b7475..42ba4ff35f1 100644 --- a/tests/logging_callback_tests/test_unit_test_litellm_logging.py +++ b/tests/logging_callback_tests/test_unit_test_litellm_logging.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from typing import Literal diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index b2243eed049..f8917ddee78 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -1,12 +1,8 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from typing import Literal diff --git a/tests/logging_callback_tests/test_view_request_resp_logs.py b/tests/logging_callback_tests/test_view_request_resp_logs.py index 37b65855774..249e84286d5 100644 --- a/tests/logging_callback_tests/test_view_request_resp_logs.py +++ b/tests/logging_callback_tests/test_view_request_resp_logs.py @@ -1,8 +1,5 @@ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import json diff --git a/tests/mcp_tests/conftest.py b/tests/mcp_tests/conftest.py index a3b425f72c3..d1dc3ec7216 100644 --- a/tests/mcp_tests/conftest.py +++ b/tests/mcp_tests/conftest.py @@ -2,13 +2,9 @@ import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import asyncio @@ -29,9 +25,6 @@ def setup_and_teardown(): This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ curr_dir = os.getcwd() # Get the current working directory - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path from litellm import Router diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index 6da8ce598a9..7a48c366003 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -1,11 +1,9 @@ import logging import os -import sys import pytest from typing import List, Any, cast from unittest.mock import AsyncMock, patch -sys.path.insert(0, os.path.abspath("../../..")) # Import required modules import litellm diff --git a/tests/mcp_tests/test_mcp_client_unit.py b/tests/mcp_tests/test_mcp_client_unit.py index 43260eda1b7..aadaadd510e 100644 --- a/tests/mcp_tests/test_mcp_client_unit.py +++ b/tests/mcp_tests/test_mcp_client_unit.py @@ -3,13 +3,10 @@ Unit tests for the MCPClient class - critical functionality only. """ import base64 -import os -import sys import pytest from unittest.mock import AsyncMock, MagicMock, patch, ANY # Add the project root to the path -sys.path.insert(0, os.path.abspath("../../..")) import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import MCPClient diff --git a/tests/mcp_tests/test_mcp_guardrails.py b/tests/mcp_tests/test_mcp_guardrails.py index 42f4aa6778b..04401992449 100644 --- a/tests/mcp_tests/test_mcp_guardrails.py +++ b/tests/mcp_tests/test_mcp_guardrails.py @@ -7,14 +7,11 @@ including various guardrail types and proper exception handling. import asyncio import pytest -import sys -import os from datetime import datetime from typing import Optional, Dict, Any from unittest.mock import MagicMock, AsyncMock, patch # Add the project root to the path -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException diff --git a/tests/mcp_tests/test_mcp_litellm_client.py b/tests/mcp_tests/test_mcp_litellm_client.py index e197673ab10..cfc0692c8fa 100644 --- a/tests/mcp_tests/test_mcp_litellm_client.py +++ b/tests/mcp_tests/test_mcp_litellm_client.py @@ -1,16 +1,11 @@ # Create server parameters for stdio connection import os -import sys import pytest import asyncio -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client -import os from litellm import experimental_mcp_client import litellm import json diff --git a/tests/mcp_tests/test_mcp_logging.py b/tests/mcp_tests/test_mcp_logging.py index 55b49aa0d29..7ee745b311e 100644 --- a/tests/mcp_tests/test_mcp_logging.py +++ b/tests/mcp_tests/test_mcp_logging.py @@ -1,14 +1,10 @@ import os -import sys import pytest import asyncio from typing import Optional from unittest.mock import AsyncMock, patch -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.types.utils import StandardLoggingPayload from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 434a9bc3809..e06c33263fb 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1,13 +1,9 @@ # Create server parameters for stdio connection import os -import sys import pytest from unittest.mock import AsyncMock, MagicMock, patch from contextlib import asynccontextmanager -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, diff --git a/tests/mcp_tests/test_semantic_tool_filter_e2e.py b/tests/mcp_tests/test_semantic_tool_filter_e2e.py index f71067fde6d..aa25c98107e 100644 --- a/tests/mcp_tests/test_semantic_tool_filter_e2e.py +++ b/tests/mcp_tests/test_semantic_tool_filter_e2e.py @@ -4,12 +4,10 @@ End-to-end test for MCP Semantic Tool Filtering import asyncio import os -import sys from unittest.mock import Mock import pytest -sys.path.insert(0, os.path.abspath("../..")) from mcp.types import Tool as MCPTool diff --git a/tests/multi_instance_e2e_tests/test_update_team_e2e.py b/tests/multi_instance_e2e_tests/test_update_team_e2e.py index 13091fd3df6..ce88e976ce0 100644 --- a/tests/multi_instance_e2e_tests/test_update_team_e2e.py +++ b/tests/multi_instance_e2e_tests/test_update_team_e2e.py @@ -143,7 +143,7 @@ async def test_team_blocking_behavior_multi_instance(): assert team_info_4001["blocked"] is True, "Team should be blocked after update" # 8. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked. - with pytest.raises(Exception, match="(?i)blocked") as excinfo: + with pytest.raises(Exception, match=r"(?i)blocked") as excinfo: await chat_completion_on_port( session, key=key, @@ -157,7 +157,7 @@ async def test_team_blocking_behavior_multi_instance(): ), f"Expected error indicating team blocked, got: {error_msg}" # 9. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked. - with pytest.raises(Exception, match="(?i)blocked") as excinfo: + with pytest.raises(Exception, match=r"(?i)blocked") as excinfo: await chat_completion_on_port( session, key=key, @@ -171,7 +171,7 @@ async def test_team_blocking_behavior_multi_instance(): ), f"Expected error indicating team blocked, got: {error_msg}" # 9. Repeat the chat completion request with another new prompt; expect it to be blocked. - with pytest.raises(Exception, match="(?i)blocked") as excinfo_second: + with pytest.raises(Exception, match=r"(?i)blocked") as excinfo_second: await chat_completion_on_port( session, key=key, diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py index 09d535dee4b..259aad5f782 100644 --- a/tests/ocr_tests/conftest.py +++ b/tests/ocr_tests/conftest.py @@ -5,12 +5,9 @@ # Vertex AI OCR) are replayed for 24h. See tests/llm_translation/Readme.md # for the design overview. -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, diff --git a/tests/otel_tests/test_prometheus.py b/tests/otel_tests/test_prometheus.py index 90c71037609..84d5f48a706 100644 --- a/tests/otel_tests/test_prometheus.py +++ b/tests/otel_tests/test_prometheus.py @@ -6,14 +6,9 @@ import pytest import aiohttp import asyncio from litellm._uuid import uuid -import os -import sys from openai import AsyncOpenAI from typing import Dict, Any -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path END_USER_ID = "my-test-user-34" diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py index e8d14b00681..520b31513f5 100644 --- a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py @@ -17,7 +17,6 @@ import sys from abc import ABC, abstractmethod from typing import Any, Dict, List -sys.path.insert(0, os.path.abspath("../../..")) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) import pytest diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py index 64acc68c264..6a5bf627ac7 100644 --- a/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py @@ -8,12 +8,9 @@ Reference: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-se """ import json -import os -import sys from abc import ABC, abstractmethod from typing import Any, Dict, List -sys.path.insert(0, os.path.abspath("../../..")) import pytest import litellm diff --git a/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py b/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py index 821cb59887f..153c72e4a11 100644 --- a/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py @@ -1,15 +1,10 @@ import json -import os -import sys from datetime import datetime from typing import AsyncIterator, Dict, Any import asyncio import unittest.mock from unittest.mock import AsyncMock, MagicMock -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm import pytest from dotenv import load_dotenv diff --git a/tests/pass_through_unit_tests/conftest.py b/tests/pass_through_unit_tests/conftest.py index 10615ddcb73..e6e98f790e8 100644 --- a/tests/pass_through_unit_tests/conftest.py +++ b/tests/pass_through_unit_tests/conftest.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py b/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py index ce5e8aa25fe..8f27fa000f6 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py @@ -6,12 +6,9 @@ by making actual API calls and validating JSON response format. """ import json -import os -import sys from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional -sys.path.insert(0, os.path.abspath("../../..")) import pytest import litellm diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py index 261c7d18d65..6f87aed4393 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py @@ -7,10 +7,7 @@ by making actual API calls and validating JSON response format. Requires ANTHROPIC_API_KEY environment variable. """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from .base_anthropic_messages_structured_output_test import ( BaseAnthropicMessagesStructuredOutputTest, diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py index b2470bf6b67..1ca4213a2b1 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py @@ -8,10 +8,8 @@ Requires Azure AI credentials and model deployment. """ import os -import sys from typing import Optional -sys.path.insert(0, os.path.abspath("../../../..")) from .base_anthropic_messages_structured_output_test import ( BaseAnthropicMessagesStructuredOutputTest, diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py index 7af7e8e38eb..bb7aa3dec35 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py @@ -7,10 +7,7 @@ by making actual API calls and validating JSON response format. Requires AWS credentials and Bedrock model access. """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from .base_anthropic_messages_structured_output_test import ( BaseAnthropicMessagesStructuredOutputTest, diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py index 09813507058..05a78d9ea00 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py @@ -7,12 +7,9 @@ by making actual API calls and validating JSON response format. Requires AWS credentials and Bedrock model access. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from .base_anthropic_messages_structured_output_test import ( BaseAnthropicMessagesStructuredOutputTest, diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index a53efdd8255..940c9624ec4 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -1,15 +1,11 @@ import json import os -import sys from datetime import datetime from typing import AsyncIterator, Dict, Any import asyncio import unittest.mock from unittest.mock import AsyncMock, MagicMock -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm import pytest from dotenv import load_dotenv @@ -41,7 +37,6 @@ def event_loop(): @pytest.fixture(scope="function", autouse=True) def setup_and_teardown(event_loop): # Add event_loop as a dependency curr_dir = os.getcwd() - sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import Router diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py b/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py index a194ded12fd..e64218b677f 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py @@ -11,10 +11,7 @@ Per AWS docs (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-cachin - Claude 3.5 Haiku: GA, 2048 min tokens """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) import pytest from base_anthropic_messages_prompt_caching_test import ( diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py b/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py index c8b91c3c49f..9006356ff2a 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py @@ -13,10 +13,7 @@ Supported providers: Reference: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) import pytest from base_anthropic_messages_tool_search_test import ( diff --git a/tests/pass_through_unit_tests/test_assemblyai_unit_tests_passthrough.py b/tests/pass_through_unit_tests/test_assemblyai_unit_tests_passthrough.py index 6fdd4cc0f24..bbc6b6b5937 100644 --- a/tests/pass_through_unit_tests/test_assemblyai_unit_tests_passthrough.py +++ b/tests/pass_through_unit_tests/test_assemblyai_unit_tests_passthrough.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import httpx @@ -15,12 +10,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from litellm.proxy.pass_through_endpoints.llm_provider_handlers.assembly_passthrough_logging_handler import ( AssemblyAIPassthroughLoggingHandler, diff --git a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py index dcc44cae77e..e86c32f916d 100644 --- a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py +++ b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py @@ -1,6 +1,5 @@ import json import os -import sys from datetime import datetime from typing import AsyncIterator, Dict, Any import asyncio @@ -9,9 +8,6 @@ from unittest.mock import MagicMock import pytest from litellm.router import Router -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from base_anthropic_unified_messages_test import BaseAnthropicMessagesTest diff --git a/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py b/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py index ed7f38cba4b..a28b8a147af 100644 --- a/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py +++ b/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py @@ -5,11 +5,8 @@ Tests that LiteLLM correctly filters out the advanced-tool-use-2025-11-20 beta h for Bedrock Invoke API, which doesn't support it and returns a 400 "invalid beta flag" error. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/pass_through_unit_tests/test_claude_code_marketplace.py b/tests/pass_through_unit_tests/test_claude_code_marketplace.py index 1a225b44b50..2ca81f1d5d3 100644 --- a/tests/pass_through_unit_tests/test_claude_code_marketplace.py +++ b/tests/pass_through_unit_tests/test_claude_code_marketplace.py @@ -7,15 +7,12 @@ Tests: """ import json -import os -import sys import time from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index 6e6507f9826..e70f2cf4430 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -1,6 +1,5 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch, MagicMock from typing import Optional @@ -8,9 +7,6 @@ from fastapi import Request import pytest import asyncio -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 77fb924c085..ed04b63000f 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -1,13 +1,8 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch, MagicMock from typing import Optional -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import fastapi from fastapi import FastAPI diff --git a/tests/pass_through_unit_tests/test_passthrough_managed_ids.py b/tests/pass_through_unit_tests/test_passthrough_managed_ids.py index cbbf9257118..0fc0e0e751c 100644 --- a/tests/pass_through_unit_tests/test_passthrough_managed_ids.py +++ b/tests/pass_through_unit_tests/test_passthrough_managed_ids.py @@ -18,14 +18,11 @@ from __future__ import annotations import base64 import json -import sys -import os from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.llms.base_llm.managed_resources.utils import ( diff --git a/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py b/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py index 5ab0319da47..8c59ce77451 100644 --- a/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py +++ b/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx diff --git a/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py b/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py index ee1f8772568..2b5bb6cf284 100644 --- a/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py +++ b/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py @@ -1,10 +1,8 @@ import json import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch, MagicMock -sys.path.insert(0, os.path.abspath("../..")) # import unittest from litellm.proxy.pass_through_endpoints.passthrough_endpoint_router import ( diff --git a/tests/pass_through_unit_tests/test_unit_test_streaming.py b/tests/pass_through_unit_tests/test_unit_test_streaming.py index ed98b720b37..376c9208aa1 100644 --- a/tests/pass_through_unit_tests/test_unit_test_streaming.py +++ b/tests/pass_through_unit_tests/test_unit_test_streaming.py @@ -1,12 +1,7 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch, MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx import pytest diff --git a/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py b/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py index ac754aefaea..498f0a734a3 100644 --- a/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py +++ b/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py @@ -6,12 +6,9 @@ for Vertex AI streamRawPredict endpoints when include_cost_in_streaming_usage is """ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch -sys.path.insert(0, os.path.abspath("../..")) import httpx import pytest diff --git a/tests/pass_through_unit_tests/test_vertex_ai_live_passthrough.py b/tests/pass_through_unit_tests/test_vertex_ai_live_passthrough.py index f25d9e7c1d3..e2eb6d0b68b 100644 --- a/tests/pass_through_unit_tests/test_vertex_ai_live_passthrough.py +++ b/tests/pass_through_unit_tests/test_vertex_ai_live_passthrough.py @@ -6,8 +6,6 @@ including the logging handler, cost tracking, and WebSocket message processing. """ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, Mock, patch, MagicMock from typing import Dict, List, Any, Optional @@ -16,7 +14,6 @@ import pytest import httpx # Add the parent directory to the system path -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import ( VertexAILivePassthroughLoggingHandler, diff --git a/tests/pass_through_unit_tests/test_websearch_interception_e2e.py b/tests/pass_through_unit_tests/test_websearch_interception_e2e.py index d4cf997ab58..091ea106b91 100644 --- a/tests/pass_through_unit_tests/test_websearch_interception_e2e.py +++ b/tests/pass_through_unit_tests/test_websearch_interception_e2e.py @@ -5,10 +5,8 @@ Makes actual calls to test WebSearch interception with Perplexity. Tests both streaming and non-streaming requests. """ -import os import sys -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.websearch_interception import ( diff --git a/tests/proxy_admin_ui_tests/conftest.py b/tests/proxy_admin_ui_tests/conftest.py index 67365f4745d..93f00db8f79 100644 --- a/tests/proxy_admin_ui_tests/conftest.py +++ b/tests/proxy_admin_ui_tests/conftest.py @@ -2,13 +2,9 @@ import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm @@ -18,9 +14,6 @@ def setup_and_teardown(): This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ curr_dir = os.getcwd() # Get the current working directory - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path from litellm import Router diff --git a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py index f7092d3ec00..b72a1453576 100644 --- a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py +++ b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py @@ -10,7 +10,6 @@ suite, which is the only place a `NOT (... = ANY(...))` guard going missing show import asyncio import os -import sys from contextlib import asynccontextmanager from datetime import timedelta from types import SimpleNamespace @@ -18,7 +17,6 @@ from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.management_helpers.access_group_team_sync import ( reconcile_team_access_group_membership, diff --git a/tests/proxy_admin_ui_tests/test_key_management.py b/tests/proxy_admin_ui_tests/test_key_management.py index 9fff120bba1..979ba31bffa 100644 --- a/tests/proxy_admin_ui_tests/test_key_management.py +++ b/tests/proxy_admin_ui_tests/test_key_management.py @@ -1,5 +1,4 @@ import os -import sys import traceback from litellm._uuid import uuid import datetime as dt @@ -16,9 +15,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_admin_ui_tests/test_role_based_access.py b/tests/proxy_admin_ui_tests/test_role_based_access.py index b5a076d0185..92e731b8c23 100644 --- a/tests/proxy_admin_ui_tests/test_role_based_access.py +++ b/tests/proxy_admin_ui_tests/test_role_based_access.py @@ -3,7 +3,7 @@ RBAC tests """ import os -import sys +import re import traceback from litellm._uuid import uuid from datetime import datetime @@ -18,9 +18,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging from unittest.mock import MagicMock @@ -411,7 +408,7 @@ async def test_org_admin_create_user_team_wrong_org_permissions(prisma_client): request.body = return_body with pytest.raises( - Exception, match="You do not have a role within the selected organization. Passed organization_id" + Exception, match=re.escape("You do not have a role within the selected organization. Passed organization_id") ) as exc_info: response = await user_api_key_auth(request=request, api_key="Bearer " + new_key) e = exc_info.value diff --git a/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py b/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py index 6396a92cf80..a31c0b923e3 100644 --- a/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py +++ b/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py @@ -1,5 +1,3 @@ -import os -import sys import traceback from litellm._uuid import uuid import datetime as dt @@ -16,9 +14,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_admin_ui_tests/test_sso_sign_in.py b/tests/proxy_admin_ui_tests/test_sso_sign_in.py index 294a5c56199..dd618cf3836 100644 --- a/tests/proxy_admin_ui_tests/test_sso_sign_in.py +++ b/tests/proxy_admin_ui_tests/test_sso_sign_in.py @@ -3,18 +3,13 @@ from fastapi.testclient import TestClient from fastapi import Request, Header from unittest.mock import patch, MagicMock, AsyncMock -import sys import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.proxy_server import app from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.proxy.management_endpoints.ui_sso import auth_callback from litellm.proxy._types import LitellmUserRoles -import os import jwt import time from litellm.caching.caching import DualCache diff --git a/tests/proxy_admin_ui_tests/test_usage_endpoints.py b/tests/proxy_admin_ui_tests/test_usage_endpoints.py index 0d1fa3afa0c..0831902c290 100644 --- a/tests/proxy_admin_ui_tests/test_usage_endpoints.py +++ b/tests/proxy_admin_ui_tests/test_usage_endpoints.py @@ -14,7 +14,6 @@ For all tests - test the following: """ import os -import sys import traceback from litellm._uuid import uuid from datetime import datetime @@ -29,9 +28,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_unit_tests/conftest.py b/tests/proxy_unit_tests/conftest.py index a0326f64ed7..148751c33f2 100644 --- a/tests/proxy_unit_tests/conftest.py +++ b/tests/proxy_unit_tests/conftest.py @@ -3,15 +3,10 @@ import asyncio import copy import inspect -import os -import sys import warnings import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm import litellm.proxy.proxy_server diff --git a/tests/proxy_unit_tests/test_aproxy_startup.py b/tests/proxy_unit_tests/test_aproxy_startup.py index 324a881a7c3..98bf6ef8eb7 100644 --- a/tests/proxy_unit_tests/test_aproxy_startup.py +++ b/tests/proxy_unit_tests/test_aproxy_startup.py @@ -9,9 +9,6 @@ import io # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest, logging, asyncio import litellm from litellm.proxy.proxy_server import ( diff --git a/tests/proxy_unit_tests/test_audit_logs_proxy.py b/tests/proxy_unit_tests/test_audit_logs_proxy.py index a5332213886..878e19f5b6f 100644 --- a/tests/proxy_unit_tests/test_audit_logs_proxy.py +++ b/tests/proxy_unit_tests/test_audit_logs_proxy.py @@ -1,5 +1,4 @@ import os -import sys import traceback from litellm._uuid import uuid from datetime import datetime @@ -14,9 +13,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index 3dc39969024..d436c99cd20 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -7,9 +7,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest, litellm import httpx from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/proxy_unit_tests/test_banned_keyword_list.py b/tests/proxy_unit_tests/test_banned_keyword_list.py index acf4bdbb8e0..35e625a6b9e 100644 --- a/tests/proxy_unit_tests/test_banned_keyword_list.py +++ b/tests/proxy_unit_tests/test_banned_keyword_list.py @@ -9,9 +9,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm from litellm.proxy.enterprise.enterprise_hooks.banned_keywords import ( diff --git a/tests/proxy_unit_tests/test_custom_callback_input.py b/tests/proxy_unit_tests/test_custom_callback_input.py index a032b8706bc..8b7a8a8973b 100644 --- a/tests/proxy_unit_tests/test_custom_callback_input.py +++ b/tests/proxy_unit_tests/test_custom_callback_input.py @@ -3,8 +3,6 @@ import asyncio import inspect import json -import os -import sys import time import traceback from litellm._uuid import uuid @@ -13,7 +11,6 @@ from datetime import datetime import pytest from pydantic import BaseModel -sys.path.insert(0, os.path.abspath("../..")) from typing import List, Literal, Optional, Union from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/proxy_unit_tests/test_default_end_user_budget_simple.py b/tests/proxy_unit_tests/test_default_end_user_budget_simple.py index 6170b0a972e..edd0409343a 100644 --- a/tests/proxy_unit_tests/test_default_end_user_budget_simple.py +++ b/tests/proxy_unit_tests/test_default_end_user_budget_simple.py @@ -5,14 +5,11 @@ Tests the core scenarios where litellm.max_end_user_budget_id applies a default budget to end users without explicit budgets. """ -import sys -import os import uuid from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy._types import LiteLLM_BudgetTable, LiteLLM_EndUserTable diff --git a/tests/proxy_unit_tests/test_e2e_pod_lock_manager.py b/tests/proxy_unit_tests/test_e2e_pod_lock_manager.py index b1e5fd29cde..6fac731a60d 100644 --- a/tests/proxy_unit_tests/test_e2e_pod_lock_manager.py +++ b/tests/proxy_unit_tests/test_e2e_pod_lock_manager.py @@ -1,5 +1,4 @@ import os -import sys import traceback from litellm._uuid import uuid from typing import List @@ -19,9 +18,6 @@ import fakeredis # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_unit_tests/test_gemini_agents_endpoints.py b/tests/proxy_unit_tests/test_gemini_agents_endpoints.py index bdac9348f71..cddb0e526b4 100644 --- a/tests/proxy_unit_tests/test_gemini_agents_endpoints.py +++ b/tests/proxy_unit_tests/test_gemini_agents_endpoints.py @@ -9,15 +9,12 @@ longer accepted — they would appear in server logs. """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import Request from fastapi.datastructures import Headers, QueryParams -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.google_endpoints.agents_endpoints import ( _merge_query_params_into_data, diff --git a/tests/proxy_unit_tests/test_get_favicon.py b/tests/proxy_unit_tests/test_get_favicon.py index ddc8b1230a7..ad18bc90a1e 100644 --- a/tests/proxy_unit_tests/test_get_favicon.py +++ b/tests/proxy_unit_tests/test_get_favicon.py @@ -1,7 +1,5 @@ import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import httpx import pytest diff --git a/tests/proxy_unit_tests/test_get_image.py b/tests/proxy_unit_tests/test_get_image.py index 57e472f86c4..9b7f3da8a7b 100644 --- a/tests/proxy_unit_tests/test_get_image.py +++ b/tests/proxy_unit_tests/test_get_image.py @@ -1,9 +1,6 @@ -import os -import sys from unittest import mock # Standard path insertion -sys.path.insert(0, os.path.abspath("../..")) import httpx import pytest diff --git a/tests/proxy_unit_tests/test_google_endpoint_routing.py b/tests/proxy_unit_tests/test_google_endpoint_routing.py index b978077c730..3dcfede92ea 100644 --- a/tests/proxy_unit_tests/test_google_endpoint_routing.py +++ b/tests/proxy_unit_tests/test_google_endpoint_routing.py @@ -1,12 +1,10 @@ import json import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest import yaml -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.google_endpoints.endpoints import google_generate_content diff --git a/tests/proxy_unit_tests/test_google_gemini_proxy_request.py b/tests/proxy_unit_tests/test_google_gemini_proxy_request.py index dbe30037313..6f8f90efc73 100644 --- a/tests/proxy_unit_tests/test_google_gemini_proxy_request.py +++ b/tests/proxy_unit_tests/test_google_gemini_proxy_request.py @@ -8,8 +8,6 @@ The request payload is correctly processed and forwarded to the httpx client. """ import json -import os -import sys import unittest.mock from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -18,7 +16,6 @@ import httpx import pytest # Add the parent directory to the system path -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/proxy_unit_tests/test_jwt.py index abd91113f96..6ad253f33e8 100644 --- a/tests/proxy_unit_tests/test_jwt.py +++ b/tests/proxy_unit_tests/test_jwt.py @@ -6,7 +6,6 @@ import base64 import logging import os import random -import sys import time import traceback from litellm._uuid import uuid @@ -15,9 +14,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from datetime import datetime, timedelta from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index 16507aaaf55..a3deeb46f6e 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -20,7 +20,7 @@ # function to validate a request - async def user_auth(request: Request): import os -import sys +import re import traceback from litellm._uuid import uuid from datetime import datetime, timezone @@ -37,9 +37,6 @@ import time # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging @@ -1498,7 +1495,9 @@ def test_key_generate_with_custom_auth(prisma_client): await litellm.proxy.proxy_server.prisma_client.connect() request = GenerateKeyRequest() - with pytest.raises(Exception, match="This violates LiteLLM Proxy Rules. No team id provided.") as exc_info: + with pytest.raises( + Exception, match=re.escape("This violates LiteLLM Proxy Rules. No team id provided.") + ) as exc_info: key = await generate_key_fn( request, user_api_key_dict=UserAPIKeyAuth( @@ -3045,7 +3044,9 @@ async def test_custom_api_key_header_name(prisma_client): "headers": [], } ) - with pytest.raises(Exception, match="Malformed API Key passed in. Ensure Key has `Bearer ` prefix") as exc_info: + with pytest.raises( + Exception, match=re.escape("Malformed API Key passed in. Ensure Key has `Bearer ` prefix") + ) as exc_info: result = await user_api_key_auth(request=request, api_key="Bearer sk-1234") e = exc_info.value print("failed with error", e) diff --git a/tests/proxy_unit_tests/test_prisma_client_backoff_retry.py b/tests/proxy_unit_tests/test_prisma_client_backoff_retry.py index b49bef3632d..8fe1c68da59 100644 --- a/tests/proxy_unit_tests/test_prisma_client_backoff_retry.py +++ b/tests/proxy_unit_tests/test_prisma_client_backoff_retry.py @@ -11,10 +11,8 @@ import time from unittest.mock import AsyncMock, MagicMock, patch, call from unittest.mock import Mock import sys -import os # Add project root to path -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.utils import PrismaClient, ProxyLogging from prisma.errors import PrismaError, ClientNotConnectedError diff --git a/tests/proxy_unit_tests/test_proxy_config_unit_test.py b/tests/proxy_unit_tests/test_proxy_config_unit_test.py index a567ad2b025..81648dc1158 100644 --- a/tests/proxy_unit_tests/test_proxy_config_unit_test.py +++ b/tests/proxy_unit_tests/test_proxy_config_unit_test.py @@ -1,5 +1,4 @@ import os -import sys import traceback from unittest import mock import pytest @@ -14,7 +13,6 @@ import io # this file is to test litellm/proxy -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_unit_tests/test_proxy_custom_auth.py b/tests/proxy_unit_tests/test_proxy_custom_auth.py index 0582cacb42d..b575e4c85c6 100644 --- a/tests/proxy_unit_tests/test_proxy_custom_auth.py +++ b/tests/proxy_unit_tests/test_proxy_custom_auth.py @@ -1,5 +1,4 @@ import os -import sys import traceback from dotenv import load_dotenv @@ -9,9 +8,6 @@ import io # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import pytest diff --git a/tests/proxy_unit_tests/test_proxy_custom_logger.py b/tests/proxy_unit_tests/test_proxy_custom_logger.py index 20b9678c7fa..2516df2d58d 100644 --- a/tests/proxy_unit_tests/test_proxy_custom_logger.py +++ b/tests/proxy_unit_tests/test_proxy_custom_logger.py @@ -7,9 +7,6 @@ import io, asyncio # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest, time import litellm from litellm import embedding, completion, completion_cost, Timeout diff --git a/tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py b/tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py index 396a34e9b85..88ee64b6c4b 100644 --- a/tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py +++ b/tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py @@ -1,5 +1,4 @@ import os -import sys import pytest from dotenv import load_dotenv @@ -7,9 +6,6 @@ from dotenv import load_dotenv load_dotenv() import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds-the parent directory to the system path from litellm.proxy import proxy_server from litellm.proxy.common_utils.encrypt_decrypt_utils import ( diff --git a/tests/proxy_unit_tests/test_proxy_exception_mapping.py b/tests/proxy_unit_tests/test_proxy_exception_mapping.py index e9884f8b269..efaaa181600 100644 --- a/tests/proxy_unit_tests/test_proxy_exception_mapping.py +++ b/tests/proxy_unit_tests/test_proxy_exception_mapping.py @@ -2,7 +2,6 @@ import json import os -import sys from unittest import mock from dotenv import load_dotenv @@ -11,9 +10,6 @@ load_dotenv() import asyncio import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import openai import pytest from fastapi import Response diff --git a/tests/proxy_unit_tests/test_proxy_pass_user_config.py b/tests/proxy_unit_tests/test_proxy_pass_user_config.py index 73998253f32..91911c142ea 100644 --- a/tests/proxy_unit_tests/test_proxy_pass_user_config.py +++ b/tests/proxy_unit_tests/test_proxy_pass_user_config.py @@ -7,9 +7,6 @@ import io # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest, logging, asyncio import litellm from litellm import embedding, completion, completion_cost, Timeout @@ -24,7 +21,6 @@ logging.basicConfig( # test /chat/completion request to the proxy from fastapi.testclient import TestClient from fastapi import FastAPI -import os from litellm.proxy.proxy_server import ( router, save_worker_config, diff --git a/tests/proxy_unit_tests/test_proxy_reject_logging.py b/tests/proxy_unit_tests/test_proxy_reject_logging.py index 440f2362276..eb5c5a52f0a 100644 --- a/tests/proxy_unit_tests/test_proxy_reject_logging.py +++ b/tests/proxy_unit_tests/test_proxy_reject_logging.py @@ -5,12 +5,10 @@ ## This tests the llm guard integration import asyncio -import os import random # What is this? ## Unit test for presidio pii masking -import sys import time import traceback from datetime import datetime @@ -19,9 +17,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Literal import pytest diff --git a/tests/proxy_unit_tests/test_proxy_routes.py b/tests/proxy_unit_tests/test_proxy_routes.py index 9d9c02257c2..129a93ea08d 100644 --- a/tests/proxy_unit_tests/test_proxy_routes.py +++ b/tests/proxy_unit_tests/test_proxy_routes.py @@ -1,5 +1,3 @@ -import os -import sys from dotenv import load_dotenv @@ -8,9 +6,6 @@ import io # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index bb8127a8b91..21dbf3e090f 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -1,5 +1,4 @@ import os -import sys import traceback from unittest import mock @@ -14,9 +13,6 @@ import json # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import logging diff --git a/tests/proxy_unit_tests/test_proxy_setting_guardrails.py b/tests/proxy_unit_tests/test_proxy_setting_guardrails.py index d16546249a4..71b7783f5ee 100644 --- a/tests/proxy_unit_tests/test_proxy_setting_guardrails.py +++ b/tests/proxy_unit_tests/test_proxy_setting_guardrails.py @@ -1,6 +1,5 @@ import json import os -import sys from unittest import mock from dotenv import load_dotenv @@ -9,9 +8,6 @@ load_dotenv() import asyncio import io -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import openai import pytest from fastapi import Response diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/proxy_unit_tests/test_proxy_token_counter.py index 1079a5228a1..39ec4bb1887 100644 --- a/tests/proxy_unit_tests/test_proxy_token_counter.py +++ b/tests/proxy_unit_tests/test_proxy_token_counter.py @@ -5,7 +5,6 @@ import json import logging import os -import sys import tempfile from unittest.mock import AsyncMock, MagicMock, patch @@ -17,9 +16,6 @@ load_dotenv() # this file is to test litellm/proxy -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from fastapi import HTTPException, Request diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index de2a9282300..3bde72ccd49 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1,7 +1,6 @@ import asyncio import json import os -import sys from datetime import datetime from typing import Any, Dict, List, Optional, Union from unittest.mock import Mock @@ -14,9 +13,6 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy.utils import _get_docs_url, _get_openapi_url, _get_redoc_url from litellm.types.guardrails import GuardrailEventHooks -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch import litellm diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/proxy_unit_tests/test_response_polling_handler.py index 8d9c7a6a095..772d3622745 100644 --- a/tests/proxy_unit_tests/test_response_polling_handler.py +++ b/tests/proxy_unit_tests/test_response_polling_handler.py @@ -15,15 +15,12 @@ following the OpenAI Response API format. """ import json -import os -import sys from datetime import datetime, timezone from typing import Any, Dict, Optional from unittest.mock import AsyncMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index fe411b1d858..459834d0fd2 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -6,14 +6,11 @@ BEFORE a polling ID is created, so rate-limited requests get a synchronous error instead of a polling ID that immediately fails. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException, Request, Response -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/proxy_unit_tests/test_search_api_logging.py b/tests/proxy_unit_tests/test_search_api_logging.py index 71bbe5351a2..5a833d37615 100644 --- a/tests/proxy_unit_tests/test_search_api_logging.py +++ b/tests/proxy_unit_tests/test_search_api_logging.py @@ -8,14 +8,12 @@ model_group, spend, etc.) import asyncio import os -import sys import time from datetime import datetime from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import Router from litellm.caching import DualCache diff --git a/tests/proxy_unit_tests/test_skills_db.py b/tests/proxy_unit_tests/test_skills_db.py index 9548e78d6ed..8eb07a5ad48 100644 --- a/tests/proxy_unit_tests/test_skills_db.py +++ b/tests/proxy_unit_tests/test_skills_db.py @@ -10,7 +10,6 @@ Tests the SDK-level skills methods when using the LiteLLM database backend: """ import os -import sys import zipfile from contextlib import contextmanager from io import BytesIO @@ -18,7 +17,6 @@ from pathlib import Path import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.caching.caching import DualCache diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index 8b5e6c5497b..3785ccdcfba 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -1,8 +1,5 @@ -import os -import sys from unittest.mock import AsyncMock, patch -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system-path import pytest diff --git a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py index 492b4803af4..e6ffea35e52 100644 --- a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py +++ b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py @@ -1,13 +1,10 @@ import asyncio -import os -import sys from unittest.mock import Mock, patch, AsyncMock import pytest from fastapi import Request from litellm.proxy.utils import _get_redoc_url, _get_docs_url from datetime import datetime -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/proxy_unit_tests/test_update_spend.py b/tests/proxy_unit_tests/test_update_spend.py index 2df381c8190..a28a78cc4a1 100644 --- a/tests/proxy_unit_tests/test_update_spend.py +++ b/tests/proxy_unit_tests/test_update_spend.py @@ -1,15 +1,10 @@ import asyncio -import os -import sys from unittest.mock import Mock from litellm.proxy.utils import _get_redoc_url, _get_docs_url import pytest from fastapi import Request -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from unittest.mock import MagicMock, patch, AsyncMock diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index 49ec29d3ac5..cc7de71aa56 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -1,13 +1,10 @@ # What is this? ## Unit tests for user_api_key_auth helper functions -import os -import sys import litellm.proxy import litellm.proxy.proxy_server -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path from typing import Dict, List, Optional from unittest.mock import MagicMock, patch, AsyncMock diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index db6a722a926..cca1028aec7 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -2,14 +2,9 @@ import asyncio import importlib -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402,F401 @@ -44,9 +39,6 @@ def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -58,8 +50,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/router_unit_tests/create_mock_standard_logging_payload.py b/tests/router_unit_tests/create_mock_standard_logging_payload.py index 106328e95e2..096c8ff8c60 100644 --- a/tests/router_unit_tests/create_mock_standard_logging_payload.py +++ b/tests/router_unit_tests/create_mock_standard_logging_payload.py @@ -1,9 +1,6 @@ import io -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import asyncio import gzip diff --git a/tests/router_unit_tests/test_completion_no_copy.py b/tests/router_unit_tests/test_completion_no_copy.py index 28f40779496..ef157d3b903 100644 --- a/tests/router_unit_tests/test_completion_no_copy.py +++ b/tests/router_unit_tests/test_completion_no_copy.py @@ -5,11 +5,8 @@ Verifies that spreading deployment["litellm_params"] directly (without copy) doesn't cause side effects that mutate the deployment in router.model_list. """ -import sys -import os import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm import Router from unittest.mock import AsyncMock, Mock, patch diff --git a/tests/router_unit_tests/test_default_deployment_copy.py b/tests/router_unit_tests/test_default_deployment_copy.py index 90401479308..3cb9c3683d6 100644 --- a/tests/router_unit_tests/test_default_deployment_copy.py +++ b/tests/router_unit_tests/test_default_deployment_copy.py @@ -5,10 +5,7 @@ Tests the critical side effect: ensure modifying returned deployment doesn't corrupt the original default_deployment instance. """ -import sys -import os -sys.path.insert(0, os.path.abspath("../..")) from litellm import Router diff --git a/tests/router_unit_tests/test_prompt_management_check.py b/tests/router_unit_tests/test_prompt_management_check.py index 81c6c6f0138..313ba2c3340 100644 --- a/tests/router_unit_tests/test_prompt_management_check.py +++ b/tests/router_unit_tests/test_prompt_management_check.py @@ -5,10 +5,7 @@ Verifies that the early return for models without "/" doesn't break prompt management model detection. """ -import sys -import os -sys.path.insert(0, os.path.abspath("../..")) from litellm import Router diff --git a/tests/router_unit_tests/test_router_acancel_batch.py b/tests/router_unit_tests/test_router_acancel_batch.py index 016da592e94..c15658d5d14 100644 --- a/tests/router_unit_tests/test_router_acancel_batch.py +++ b/tests/router_unit_tests/test_router_acancel_batch.py @@ -4,10 +4,7 @@ Test router.acancel_batch() functionality This ensures the router's batch cancellation method has test coverage. """ -import sys -import os -sys.path.insert(0, os.path.abspath("../..")) import pytest from unittest.mock import patch, AsyncMock, MagicMock diff --git a/tests/router_unit_tests/test_router_adding_deployments.py b/tests/router_unit_tests/test_router_adding_deployments.py index 6200cc6ebcc..dfbaf1257c6 100644 --- a/tests/router_unit_tests/test_router_adding_deployments.py +++ b/tests/router_unit_tests/test_router_adding_deployments.py @@ -1,9 +1,6 @@ import sys, os import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm import Router from litellm.router import Deployment, LiteLLM_Params from unittest.mock import patch diff --git a/tests/router_unit_tests/test_router_aresponses_streaming_fallback.py b/tests/router_unit_tests/test_router_aresponses_streaming_fallback.py index 17124a94a8f..ee4750e9db8 100644 --- a/tests/router_unit_tests/test_router_aresponses_streaming_fallback.py +++ b/tests/router_unit_tests/test_router_aresponses_streaming_fallback.py @@ -10,14 +10,11 @@ Targets the four helpers introduced on Router: - _aresponses_streaming_iterator """ -import os -import sys from typing import Any, AsyncIterator, List from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm import Router from litellm.types.llms.openai import ( diff --git a/tests/router_unit_tests/test_router_batch_utils.py b/tests/router_unit_tests/test_router_batch_utils.py index b8760906645..c9f19731372 100644 --- a/tests/router_unit_tests/test_router_batch_utils.py +++ b/tests/router_unit_tests/test_router_batch_utils.py @@ -1,9 +1,4 @@ -import sys -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import json diff --git a/tests/router_unit_tests/test_router_cooldown_utils.py b/tests/router_unit_tests/test_router_cooldown_utils.py index a51b0dc21af..242709708e3 100644 --- a/tests/router_unit_tests/test_router_cooldown_utils.py +++ b/tests/router_unit_tests/test_router_cooldown_utils.py @@ -2,9 +2,6 @@ import sys, os, time import traceback, asyncio import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import Router from litellm.router import Deployment, LiteLLM_Params diff --git a/tests/router_unit_tests/test_router_embedding_headers.py b/tests/router_unit_tests/test_router_embedding_headers.py index 5bf98243dcc..738f09e6ece 100644 --- a/tests/router_unit_tests/test_router_embedding_headers.py +++ b/tests/router_unit_tests/test_router_embedding_headers.py @@ -9,13 +9,10 @@ just like router.completion() does, which properly sets up metadata and allows default_litellm_params (including headers) to be propagated. """ -import os -import sys from unittest.mock import MagicMock, patch, AsyncMock import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm import Router diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py index 6f5781336eb..75dacbaf08e 100644 --- a/tests/router_unit_tests/test_router_embedding_integration.py +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -5,13 +5,10 @@ These tests simulate real-world scenarios where headers and configuration need to be properly propagated through the router to the LLM API. """ -import os -import sys from unittest.mock import MagicMock, patch, AsyncMock import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm import Router diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index 658ad4f3b5c..d37af5b456a 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -1,4 +1,3 @@ -import sys import os import json import traceback @@ -8,9 +7,6 @@ from fastapi import Request from datetime import datetime from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm import Router, CustomLogger from litellm.types.utils import StandardLoggingPayload diff --git a/tests/router_unit_tests/test_router_handle_error.py b/tests/router_unit_tests/test_router_handle_error.py index a84c90ccb78..6b57efc7f37 100644 --- a/tests/router_unit_tests/test_router_handle_error.py +++ b/tests/router_unit_tests/test_router_handle_error.py @@ -3,9 +3,6 @@ import traceback, asyncio import pytest from typing import List -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm import Router from litellm.router import Deployment, LiteLLM_Params diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 755405c8b21..dcd2e9edf7b 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1,13 +1,9 @@ -import sys import os import traceback from dotenv import load_dotenv from fastapi import Request from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm import Router import pytest import litellm @@ -1832,7 +1828,7 @@ def test_init_auto_router_deployment_duplicate_model_name(mock_auto_router, mode ) with pytest.raises( - ValueError, match="Auto-router deployment test-auto-router with tags .* already exists" + ValueError, match=r"Auto-router deployment test-auto-router with tags .* already exists" ): router.init_auto_router_deployment(deployment) diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 3f0a185e8bf..87ddaadaf3d 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -1,11 +1,7 @@ -import sys import os import pytest import ast -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm import Router diff --git a/tests/router_unit_tests/test_router_prompt_caching.py b/tests/router_unit_tests/test_router_prompt_caching.py index 574eccda162..5c36c30e818 100644 --- a/tests/router_unit_tests/test_router_prompt_caching.py +++ b/tests/router_unit_tests/test_router_prompt_caching.py @@ -1,14 +1,9 @@ -import sys -import os import traceback import asyncio from dotenv import load_dotenv from fastapi import Request from datetime import datetime -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm import Router import pytest import litellm diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py index 78ba19a7724..deef6527a8f 100644 --- a/tests/search_tests/conftest.py +++ b/tests/search_tests/conftest.py @@ -6,12 +6,9 @@ # are replayed for 24h. See tests/llm_translation/Readme.md for the # design overview. -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from tests._vcr_conftest_common import ( # noqa: E402,F401 VerboseReporterState, diff --git a/tests/search_tests/test_duckduckgo_search.py b/tests/search_tests/test_duckduckgo_search.py index 635e26e1c0c..69d19edded7 100644 --- a/tests/search_tests/test_duckduckgo_search.py +++ b/tests/search_tests/test_duckduckgo_search.py @@ -3,11 +3,9 @@ Tests for DuckDuckGo Search API integration. """ import os -import sys import pytest from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert(0, os.path.abspath("../..")) import litellm from tests.search_tests.base_search_unit_tests import BaseSearchTest diff --git a/tests/search_tests/test_google_pse_search.py b/tests/search_tests/test_google_pse_search.py index 21d58a95491..12b1a714709 100644 --- a/tests/search_tests/test_google_pse_search.py +++ b/tests/search_tests/test_google_pse_search.py @@ -2,11 +2,8 @@ Tests for Google Programmable Search Engine (PSE) API integration. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from tests.search_tests.base_search_unit_tests import BaseSearchTest diff --git a/tests/search_tests/test_linkup_search.py b/tests/search_tests/test_linkup_search.py index 5e1fe4ddd9b..ab9bffc5633 100644 --- a/tests/search_tests/test_linkup_search.py +++ b/tests/search_tests/test_linkup_search.py @@ -3,11 +3,9 @@ Tests for Linkup Search API integration. """ import os -import sys import pytest from unittest.mock import Mock, patch -sys.path.insert(0, os.path.abspath("../..")) import litellm from tests.search_tests.base_search_unit_tests import BaseSearchTest diff --git a/tests/search_tests/test_nimble_search.py b/tests/search_tests/test_nimble_search.py index c83b7236a09..df432f8ae84 100644 --- a/tests/search_tests/test_nimble_search.py +++ b/tests/search_tests/test_nimble_search.py @@ -3,13 +3,10 @@ Tests for Nimble Search API integration. """ import json -import os -import sys from unittest.mock import AsyncMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from tests.search_tests.base_search_unit_tests import BaseSearchTest diff --git a/tests/search_tests/test_perplexity_search.py b/tests/search_tests/test_perplexity_search.py index c9e09ed404e..e1189a71355 100644 --- a/tests/search_tests/test_perplexity_search.py +++ b/tests/search_tests/test_perplexity_search.py @@ -3,10 +3,8 @@ Tests for Perplexity Search API integration. """ import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from tests.search_tests.base_search_unit_tests import BaseSearchTest diff --git a/tests/search_tests/test_search_tool_name_filtering.py b/tests/search_tests/test_search_tool_name_filtering.py index 5424582a90c..902e95c7a4b 100644 --- a/tests/search_tests/test_search_tool_name_filtering.py +++ b/tests/search_tests/test_search_tool_name_filtering.py @@ -6,10 +6,7 @@ which search tool configuration to use, but should not be sent to external search provider APIs. """ -import sys -import os -sys.path.insert(0, os.path.abspath("../..")) from litellm.types.utils import all_litellm_params from litellm.utils import filter_out_litellm_params diff --git a/tests/search_tests/test_searchapi_search.py b/tests/search_tests/test_searchapi_search.py index d16868502a4..58ba6aa018a 100644 --- a/tests/search_tests/test_searchapi_search.py +++ b/tests/search_tests/test_searchapi_search.py @@ -10,13 +10,11 @@ Tests the SearchAPI.io search provider implementation including: import json import os -import sys from unittest.mock import MagicMock, Mock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.llms.searchapi.search.transformation import SearchAPIConfig from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult diff --git a/tests/search_tests/test_serper_search.py b/tests/search_tests/test_serper_search.py index 99aae0f64e6..02e9d734443 100644 --- a/tests/search_tests/test_serper_search.py +++ b/tests/search_tests/test_serper_search.py @@ -3,11 +3,9 @@ Tests for Serper Search API integration. """ import os -import sys import pytest from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/search_tests/test_tavily_search.py b/tests/search_tests/test_tavily_search.py index a737685916c..4a5338deadb 100644 --- a/tests/search_tests/test_tavily_search.py +++ b/tests/search_tests/test_tavily_search.py @@ -3,11 +3,9 @@ Tests for Tavily Search API integration. """ import os -import sys import pytest from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/test_keys.py b/tests/test_keys.py index 2d8ff2232a1..e39c715de03 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -8,9 +8,6 @@ from openai import AsyncOpenAI import sys, os from typing import Optional -sys.path.insert( - 0, os.path.abspath("../") -) # Adds the parent directory to the system path import litellm from litellm.proxy._types import LitellmUserRoles diff --git a/tests/test_litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py b/tests/test_litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py index 717a7c902b5..c5626afa954 100644 --- a/tests/test_litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py +++ b/tests/test_litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py @@ -4,12 +4,9 @@ Tests for Pydantic AI agents transformation. Tests the helper functions and response transformation without making real API calls. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import ( PydanticAITransformation, diff --git a/tests/test_litellm/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py b/tests/test_litellm/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py index 7968eed4146..43dfdaba02d 100644 --- a/tests/test_litellm/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py +++ b/tests/test_litellm/a2a_protocol/providers/watsonx_orchestrate/test_watsonx_orchestrate_transformation.py @@ -1,14 +1,11 @@ import asyncio import json -import os -import sys import time from pathlib import Path import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager from litellm.a2a_protocol.providers.watsonx_orchestrate import handler as wxo_handler diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index ebe093c591c..41b4bb8cf76 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -16,15 +16,12 @@ deterministic stand-ins so the arithmetic under test is the only variable. import json import logging -import os -import sys from types import MappingProxyType import httpx import pytest import respx -sys.path.insert(0, os.path.abspath("../../../..")) import litellm import litellm.batches.batch_utils as bu @@ -800,6 +797,43 @@ async def test_output_file_content_vertex_unified_file_id_extracts_gcs_uri(monke assert captured["custom_llm_provider"] == "vertex_ai" +@pytest.mark.asyncio +async def test_output_file_content_model_encoded_file_id_decoded_to_provider_id(monkeypatch): + import litellm.files.main as files_main + from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model + + captured: dict = {} + + async def fake_afile_content(**kw): + captured.update(kw) + return type("R", (), {"content": b'{"a": 1}'})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + encoded_id = encode_file_id_with_model("file-Y3FHrMpi7uCkDpY6fgWGeR", "my-batch-model") + + await bu._fetch_batch_output_file_content(_batch(encoded_id), custom_llm_provider="openai") + + assert captured["file_id"] == "file-Y3FHrMpi7uCkDpY6fgWGeR" + assert captured["custom_llm_provider"] == "openai" + + +@pytest.mark.asyncio +async def test_output_file_content_raw_openai_file_id_passes_through(monkeypatch): + import litellm.files.main as files_main + + captured: dict = {} + + async def fake_afile_content(**kw): + captured.update(kw) + return type("R", (), {"content": b'{"a": 1}'})() + + monkeypatch.setattr(files_main, "afile_content", fake_afile_content) + + await bu._fetch_batch_output_file_content(_batch("file-abc123"), custom_llm_provider="openai") + + assert captured["file_id"] == "file-abc123" + + def _vertex_predictions_row(custom_id, prompt_tokens, completion_tokens): return { "request": { diff --git a/tests/test_litellm/batches/test_main.py b/tests/test_litellm/batches/test_main.py index 17e9ee29d4d..c3edb40c819 100644 --- a/tests/test_litellm/batches/test_main.py +++ b/tests/test_litellm/batches/test_main.py @@ -23,8 +23,6 @@ production. Provider env vars are not required: missing creds resolve to None an flow through harmlessly because the handler is mocked. """ -import os -import sys from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Dict @@ -33,7 +31,6 @@ from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm import litellm.batches.main as bm diff --git a/tests/test_litellm/caching/test_azure_blob_cache.py b/tests/test_litellm/caching/test_azure_blob_cache.py index c5c85e1551d..63f4681fd06 100644 --- a/tests/test_litellm/caching/test_azure_blob_cache.py +++ b/tests/test_litellm/caching/test_azure_blob_cache.py @@ -1,13 +1,8 @@ -import os -import sys from unittest.mock import MagicMock, patch, AsyncMock import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.caching.azure_blob_cache import AzureBlobCache diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 9684e82f550..6c60aa6e220 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys import time from unittest.mock import MagicMock, patch @@ -10,9 +8,6 @@ import pytest import respx from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from datetime import datetime from unittest.mock import AsyncMock diff --git a/tests/test_litellm/caching/test_embedding_router.py b/tests/test_litellm/caching/test_embedding_router.py index 9ebe669d32d..00a80c63303 100644 --- a/tests/test_litellm/caching/test_embedding_router.py +++ b/tests/test_litellm/caching/test_embedding_router.py @@ -1,8 +1,5 @@ -import os -import sys from unittest.mock import MagicMock -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.caching._embedding_router import ( diff --git a/tests/test_litellm/caching/test_gcs_cache.py b/tests/test_litellm/caching/test_gcs_cache.py index 40bfa447d63..6222cf4760a 100644 --- a/tests/test_litellm/caching/test_gcs_cache.py +++ b/tests/test_litellm/caching/test_gcs_cache.py @@ -1,10 +1,7 @@ -import os -import sys from unittest.mock import MagicMock, AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.caching.gcs_cache import GCSCache diff --git a/tests/test_litellm/caching/test_in_memory_cache.py b/tests/test_litellm/caching/test_in_memory_cache.py index 7be03d23fbe..85e8308ae91 100644 --- a/tests/test_litellm/caching/test_in_memory_cache.py +++ b/tests/test_litellm/caching/test_in_memory_cache.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys import threading import time from concurrent.futures import ThreadPoolExecutor @@ -12,9 +10,6 @@ import pytest import respx from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock from litellm.caching.in_memory_cache import InMemoryCache diff --git a/tests/test_litellm/caching/test_llm_caching_handler.py b/tests/test_litellm/caching/test_llm_caching_handler.py index 5f0e82dbb80..dd81b877c0e 100644 --- a/tests/test_litellm/caching/test_llm_caching_handler.py +++ b/tests/test_litellm/caching/test_llm_caching_handler.py @@ -9,15 +9,10 @@ See: https://github.com/BerriAI/litellm/pull/22247 """ import asyncio -import os -import sys import warnings import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.caching.evicted_client_closer import EvictedClientCloser from litellm.caching.llm_caching_handler import LLMClientCache diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index a5fbaf151ca..e07578dd7e5 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -1,13 +1,9 @@ -import os import sys import types from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path def test_qdrant_semantic_cache_initialization(monkeypatch): diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 6a76decd5b1..decf59130fe 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1,13 +1,8 @@ import asyncio -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock from litellm.caching.redis_cache import RedisCache diff --git a/tests/test_litellm/caching/test_redis_cluster_cache.py b/tests/test_litellm/caching/test_redis_cluster_cache.py index 26878865187..372425aa9fa 100644 --- a/tests/test_litellm/caching/test_redis_cluster_cache.py +++ b/tests/test_litellm/caching/test_redis_cluster_cache.py @@ -1,14 +1,9 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.caching.redis_cache import RedisCache from litellm.caching.redis_cluster_cache import RedisClusterCache diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 66271579d31..be4367fd8bd 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -1,12 +1,8 @@ -import os import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path # Tests for RedisSemanticCache @@ -893,7 +889,6 @@ async def test_redis_semantic_cache_async_paths_set_similarity_on_misses(): def test_redis_get_embedding_routes_through_router(monkeypatch): - import sys import types from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -928,7 +923,6 @@ def test_redis_get_embedding_routes_through_router(monkeypatch): def test_redis_get_embedding_falls_back_to_direct(monkeypatch): - import sys import types from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1138,7 +1132,6 @@ def test_redis_sync_get_cache_passes_precomputed_vector(): @pytest.mark.asyncio async def test_redis_async_embedding_forwards_full_metadata(monkeypatch): - import sys import types from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1169,7 +1162,6 @@ LONG_PROMPT = " ".join(f"token{i}" for i in range(300)) def _proxy_with_router(monkeypatch: pytest.MonkeyPatch, router: MagicMock, model_name: str) -> None: - import sys import types fake_proxy = types.ModuleType("litellm.proxy.proxy_server") @@ -1223,7 +1215,6 @@ async def test_redis_async_embedding_explicit_limit_beats_deployment_limit(monke def test_redis_get_embedding_truncates_direct_path_with_explicit_limit(monkeypatch): - import sys import types from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1342,7 +1333,6 @@ def _router_proxy_module(router, model_name): def test_redis_sync_embedding_call_is_bounded(monkeypatch): - import sys from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1366,7 +1356,6 @@ def test_redis_sync_embedding_call_is_bounded(monkeypatch): @pytest.mark.asyncio async def test_redis_async_embedding_call_is_bounded(monkeypatch): - import sys from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1391,7 +1380,6 @@ async def test_redis_async_embedding_call_is_bounded(monkeypatch): @pytest.mark.asyncio async def test_redis_async_embedding_gives_up_on_unresponsive_endpoint(monkeypatch): import asyncio - import sys import time from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -1422,7 +1410,6 @@ async def test_redis_async_embedding_gives_up_on_unresponsive_endpoint(monkeypat @pytest.mark.asyncio async def test_redis_async_get_cache_fails_open_when_embedding_hangs(monkeypatch): import asyncio - import sys import time from litellm.caching.redis_semantic_cache import RedisSemanticCache diff --git a/tests/test_litellm/caching/test_s3_cache.py b/tests/test_litellm/caching/test_s3_cache.py index 795511c5bc2..f9a0b165e12 100644 --- a/tests/test_litellm/caching/test_s3_cache.py +++ b/tests/test_litellm/caching/test_s3_cache.py @@ -1,5 +1,3 @@ -import os -import sys from unittest.mock import MagicMock, patch import json import datetime @@ -7,9 +5,6 @@ import asyncio import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.caching.s3_cache import S3Cache diff --git a/tests/test_litellm/caching/test_valkey_semantic_cache.py b/tests/test_litellm/caching/test_valkey_semantic_cache.py index acf5a914e5c..749658784ac 100644 --- a/tests/test_litellm/caching/test_valkey_semantic_cache.py +++ b/tests/test_litellm/caching/test_valkey_semantic_cache.py @@ -9,7 +9,6 @@ from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.caching.valkey_semantic_cache import ValkeySemanticCache diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index 42b5ba235bc..c5d7ca96a21 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -1,11 +1,8 @@ -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.completion_extras.litellm_responses_transformation.handler import ( diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 382b41807d4..4ff92aaf87d 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1,7 +1,6 @@ import datetime import json import os -import sys import unittest from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from unittest.mock import ANY, MagicMock, Mock, patch @@ -9,9 +8,6 @@ from unittest.mock import ANY, MagicMock, Mock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system-path import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, @@ -1508,7 +1504,7 @@ def test_multiple_tool_calls_in_single_choice(): print("✓ Multiple tool calls are correctly grouped in a single choice") -def test_map_reasoning_effort_adds_summary_detailed(): +def test_map_reasoning_effort_adds_summary_detailed(monkeypatch): """ Test that _map_reasoning_effort behavior with reasoning_auto_summary flag. @@ -1518,7 +1514,6 @@ def test_map_reasoning_effort_adds_summary_detailed(): When flag is enabled (flag=True or env var), summary="detailed" is added. """ - import os import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( @@ -1571,7 +1566,7 @@ def test_map_reasoning_effort_adds_summary_detailed(): # Test 3: With env var enabled (flag disabled) - summary IS added litellm.reasoning_auto_summary = False - os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") result = handler._map_reasoning_effort("high") assert ( @@ -1603,7 +1598,7 @@ def test_map_reasoning_effort_adds_summary_detailed(): # Restore original values litellm.reasoning_auto_summary = original_flag if original_env is not None: - os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = original_env + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", original_env) elif "LITELLM_REASONING_AUTO_SUMMARY" in os.environ: del os.environ["LITELLM_REASONING_AUTO_SUMMARY"] diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index ceb491e3d11..1fe73b552da 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -9,13 +9,9 @@ import importlib import os -import sys from pathlib import Path import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import asyncio import litellm @@ -462,7 +458,6 @@ def setup_and_teardown(): Use this sparingly - most state should be handled by isolate_litellm_state. Only reload modules here if absolutely necessary. """ - sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/test_litellm/containers/test_azure_container_transformation.py b/tests/test_litellm/containers/test_azure_container_transformation.py index cdcccf7c04e..1c990220e11 100644 --- a/tests/test_litellm/containers/test_azure_container_transformation.py +++ b/tests/test_litellm/containers/test_azure_container_transformation.py @@ -1,12 +1,9 @@ -import os -import sys from unittest.mock import AsyncMock, MagicMock from urllib.parse import parse_qs, urlparse import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../")) import litellm from litellm.llms.azure.containers.transformation import AzureContainerConfig diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index de6fd1bc8ce..885c4cd294a 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -1,15 +1,10 @@ import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.containers.main import ( diff --git a/tests/test_litellm/containers/test_container_integration.py b/tests/test_litellm/containers/test_container_integration.py index 062d0359f60..6c3a876fc45 100644 --- a/tests/test_litellm/containers/test_container_integration.py +++ b/tests/test_litellm/containers/test_container_integration.py @@ -1,14 +1,10 @@ import json import os -import sys from unittest.mock import MagicMock, patch import pytest import httpx -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.types.containers.main import ( diff --git a/tests/test_litellm/containers/test_container_regional_api_base.py b/tests/test_litellm/containers/test_container_regional_api_base.py index d450d7f9cf0..055f7d4b166 100644 --- a/tests/test_litellm/containers/test_container_regional_api_base.py +++ b/tests/test_litellm/containers/test_container_regional_api_base.py @@ -7,13 +7,11 @@ US Data Residency instead of defaulting to https://api.openai.com/v1. """ import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm diff --git a/tests/test_litellm/containers/test_container_transformation.py b/tests/test_litellm/containers/test_container_transformation.py index 555fe7773f0..8bc3ffda544 100644 --- a/tests/test_litellm/containers/test_container_transformation.py +++ b/tests/test_litellm/containers/test_container_transformation.py @@ -1,14 +1,10 @@ import json import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.openai.containers.transformation import OpenAIContainerConfig @@ -341,10 +337,10 @@ class TestOpenAIContainerTransformation: assert data["expires_after"] is None assert data["file_ids"] is None - def test_container_create_response_includes_cost(self): + def test_container_create_response_includes_cost(self, monkeypatch): """Test that container create response includes code interpreter cost calculation.""" # Force use of local model cost map for CI/CD consistency - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( diff --git a/tests/test_litellm/containers/test_container_utils.py b/tests/test_litellm/containers/test_container_utils.py index 35e9ed36916..a81d1263d6b 100644 --- a/tests/test_litellm/containers/test_container_utils.py +++ b/tests/test_litellm/containers/test_container_utils.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.containers.utils import ( diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py index 61303340570..8b89c592f02 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py @@ -1,7 +1,6 @@ import asyncio import json import os -import sys import unittest.mock as mock from unittest.mock import patch @@ -13,7 +12,6 @@ from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( BaseEmailLogger, ) -sys.path.insert(0, os.path.abspath("../../..")) from litellm_enterprise.types.enterprise_callbacks.send_emails import ( EmailEvent, SendKeyCreatedEmailEvent, diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_endpoints.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_endpoints.py index d1e8f37184a..f0e1461c616 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_endpoints.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_endpoints.py @@ -1,13 +1,10 @@ import json -import os -import sys import unittest.mock as mock import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import ( _get_email_settings, diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_resend_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_resend_email.py index 88cc2275ae2..6bf77ac2d28 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_resend_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_resend_email.py @@ -1,11 +1,9 @@ import os -import sys import unittest.mock as mock import pytest from httpx import Response -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( @@ -88,49 +86,44 @@ async def test_send_email_success(mock_env_vars): @pytest.mark.asyncio -async def test_send_email_missing_api_key(): +async def test_send_email_missing_api_key(monkeypatch): # Remove the API key from environment before initializing logger - original_key = os.environ.pop("RESEND_API_KEY", None) + monkeypatch.delenv("RESEND_API_KEY", raising=False) - try: - # Initialize the logger after removing the API key - logger = ResendEmailLogger() + # Initialize the logger after removing the API key + logger = ResendEmailLogger() - # Test data - from_email = "test@example.com" - to_email = ["recipient@example.com"] - subject = "Test Subject" - html_body = "

Test email body

" + # Test data + from_email = "test@example.com" + to_email = ["recipient@example.com"] + subject = "Test Subject" + html_body = "

Test email body

" - # Create mock HTTP client and inject it directly into the logger - # This ensures the mock is used regardless of any caching issues - mock_response = mock.Mock(spec=Response) - mock_response.raise_for_status.return_value = None - mock_response.status_code = 200 - mock_response.json.return_value = {"id": "test_email_id"} + # Create mock HTTP client and inject it directly into the logger + # This ensures the mock is used regardless of any caching issues + mock_response = mock.Mock(spec=Response) + mock_response.raise_for_status.return_value = None + mock_response.status_code = 200 + mock_response.json.return_value = {"id": "test_email_id"} - mock_async_client = mock.AsyncMock() - mock_async_client.post.return_value = mock_response + mock_async_client = mock.AsyncMock() + mock_async_client.post.return_value = mock_response - # Directly inject the mock client to bypass any caching - logger.async_httpx_client = mock_async_client + # Directly inject the mock client to bypass any caching + logger.async_httpx_client = mock_async_client - # Send email - await logger.send_email( - from_email=from_email, - to_email=to_email, - subject=subject, - html_body=html_body, - ) + # Send email + await logger.send_email( + from_email=from_email, + to_email=to_email, + subject=subject, + html_body=html_body, + ) - # Verify the HTTP client was called with None as the API key - mock_async_client.post.assert_called_once() - call_args = mock_async_client.post.call_args - assert call_args[1]["headers"] == {"Authorization": "Bearer None"} - finally: - # Restore the original key if it existed - if original_key is not None: - os.environ["RESEND_API_KEY"] = original_key + # Verify the HTTP client was called with None as the API key + mock_async_client.post.assert_called_once() + call_args = mock_async_client.post.call_args + assert call_args[1]["headers"] == {"Authorization": "Bearer None"} @pytest.mark.asyncio diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py index 5fe4b217e4f..465a03cfff7 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py @@ -1,11 +1,9 @@ import os -import sys import unittest.mock as mock import pytest from httpx import Response -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( @@ -98,22 +96,18 @@ async def test_send_email_success(mock_env_vars, mock_async_client): @pytest.mark.asyncio -async def test_send_email_missing_api_key(): - original_key = os.environ.pop("SENDGRID_API_KEY", None) +async def test_send_email_missing_api_key(monkeypatch): + monkeypatch.delenv("SENDGRID_API_KEY", raising=False) - try: - logger = SendGridEmailLogger() + logger = SendGridEmailLogger() - with pytest.raises(ValueError, match='SENDGRID_API_KEY is not set'): - await logger.send_email( - from_email="test@example.com", - to_email=["recipient@example.com"], - subject="Test Subject", - html_body="

Test email body

", - ) - finally: - if original_key is not None: - os.environ["SENDGRID_API_KEY"] = original_key + with pytest.raises(ValueError, match='SENDGRID_API_KEY is not set'): + await logger.send_email( + from_email="test@example.com", + to_email=["recipient@example.com"], + subject="Test Subject", + html_body="

Test email body

", + ) @pytest.mark.asyncio diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 1ddb2cc1c8d..51fdfa4ce31 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -21,7 +21,6 @@ from mcp.types import ( ) # Add the parent directory to the path so we can import litellm -sys.path.insert(0, "../../../") import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import ( diff --git a/tests/test_litellm/experimental_mcp_client/test_tools.py b/tests/test_litellm/experimental_mcp_client/test_tools.py index 804e99b6f4e..89f67452f29 100644 --- a/tests/test_litellm/experimental_mcp_client/test_tools.py +++ b/tests/test_litellm/experimental_mcp_client/test_tools.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from mcp.types import ( CallToolRequestParams, diff --git a/tests/test_litellm/google_genai/test_google_genai_adapter.py b/tests/test_litellm/google_genai/test_google_genai_adapter.py index 8f5f4d41f3c..81834451859 100644 --- a/tests/test_litellm/google_genai/test_google_genai_adapter.py +++ b/tests/test_litellm/google_genai/test_google_genai_adapter.py @@ -3,20 +3,13 @@ Test to verify the Google GenAI generate_content adapter functionality """ import json -import os -import sys import unittest import pytest from litellm.google_genai.main import agenerate_content -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path -import os -import sys import litellm diff --git a/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py b/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py index 36022dcb5db..8ea9dcfb990 100644 --- a/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py +++ b/tests/test_litellm/google_genai/test_google_genai_adapter_fixes.py @@ -3,16 +3,11 @@ Test to verify the Google GenAI adapter fixes """ import json -import os -import sys import unittest from unittest.mock import patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.google_genai.adapters.handler import GenerateContentToCompletionHandler diff --git a/tests/test_litellm/google_genai/test_google_genai_handler.py b/tests/test_litellm/google_genai/test_google_genai_handler.py index 0dc218d297b..bf037c59854 100644 --- a/tests/test_litellm/google_genai/test_google_genai_handler.py +++ b/tests/test_litellm/google_genai/test_google_genai_handler.py @@ -3,15 +3,10 @@ Test to verify the Google GenAI generate_content handler functionality """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.google_genai.adapters.handler import GenerateContentToCompletionHandler diff --git a/tests/test_litellm/google_genai/test_google_genai_main.py b/tests/test_litellm/google_genai/test_google_genai_main.py index 8441b62e559..238fff7deca 100644 --- a/tests/test_litellm/google_genai/test_google_genai_main.py +++ b/tests/test_litellm/google_genai/test_google_genai_main.py @@ -4,17 +4,10 @@ Test to verify the Google GenAI generate_content adapter functionality """ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path -import os -import sys import litellm diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py index 6b0cd500a82..f0d0fc6126d 100644 --- a/tests/test_litellm/google_genai/test_google_genai_transformation.py +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -2,12 +2,7 @@ """ Test to verify the Google GenAI transformation logic for generateContent parameters """ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/images/test_image_generation_extra_headers.py b/tests/test_litellm/images/test_image_generation_extra_headers.py index a6e5031c7db..a65bdeb892b 100644 --- a/tests/test_litellm/images/test_image_generation_extra_headers.py +++ b/tests/test_litellm/images/test_image_generation_extra_headers.py @@ -6,13 +6,10 @@ to the OpenAI SDK on the openai/litellm_proxy/openai_compatible_providers code paths. """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.images.main import image_generation diff --git a/tests/test_litellm/integrations/SlackAlerting/test_hanging_request_check.py b/tests/test_litellm/integrations/SlackAlerting/test_hanging_request_check.py index 063aabd309b..4b579dfe82f 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_hanging_request_check.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_hanging_request_check.py @@ -1,6 +1,4 @@ import json -import os -import sys import time from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -8,7 +6,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.SlackAlerting.hanging_request_check import ( AlertingHangingRequestCheck, diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index fd54d26c1f6..997e80b45df 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -1,14 +1,11 @@ """Tests for the Slack alerting model deprecation hook.""" import asyncio -import os -import sys from itertools import chain, repeat from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index 23a35098697..cfbd3e76a88 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -1,8 +1,6 @@ import asyncio import datetime import json -import os -import sys import time import unittest from typing import Final, List, Optional, Tuple @@ -10,7 +8,6 @@ from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system-path import litellm from litellm.caching.caching import DualCache from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py index b3fee1f045b..edce5c5f3a2 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py @@ -10,11 +10,9 @@ Verifies that: """ import os -import sys import unittest from datetime import datetime, timedelta -sys.path.insert(0, os.path.abspath("../../..")) from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_utils.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_utils.py index 027fed1b5ff..403cd51701d 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_utils.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_utils.py @@ -1,13 +1,10 @@ import json -import os -import sys from typing import Optional from unittest.mock import MagicMock import pytest # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.langfuse.langfuse_prompt_management import ( diff --git a/tests/test_litellm/integrations/arize/test_arize.py b/tests/test_litellm/integrations/arize/test_arize.py index 1ca3349eeb7..cdafd856b49 100644 --- a/tests/test_litellm/integrations/arize/test_arize.py +++ b/tests/test_litellm/integrations/arize/test_arize.py @@ -1,11 +1,8 @@ import json -import os -import sys from typing import Optional from unittest.mock import MagicMock, Mock, patch # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) import asyncio diff --git a/tests/test_litellm/integrations/arize/test_arize_health_check.py b/tests/test_litellm/integrations/arize/test_arize_health_check.py index 3f10e9dcbd7..f7364dc27eb 100644 --- a/tests/test_litellm/integrations/arize/test_arize_health_check.py +++ b/tests/test_litellm/integrations/arize/test_arize_health_check.py @@ -4,11 +4,9 @@ Test Arize health check functionality and proxy integration. import json import os -import sys from unittest.mock import patch, MagicMock # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) import asyncio import pytest diff --git a/tests/test_litellm/integrations/arize/test_arize_utils.py b/tests/test_litellm/integrations/arize/test_arize_utils.py index b02fe35cad0..50f2823d632 100644 --- a/tests/test_litellm/integrations/arize/test_arize_utils.py +++ b/tests/test_litellm/integrations/arize/test_arize_utils.py @@ -1,10 +1,7 @@ import json -import os -import sys from typing import Optional # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) import asyncio diff --git a/tests/test_litellm/integrations/azure_storage/test_azure_storage.py b/tests/test_litellm/integrations/azure_storage/test_azure_storage.py index 5d7c55e81af..16c518ff412 100644 --- a/tests/test_litellm/integrations/azure_storage/test_azure_storage.py +++ b/tests/test_litellm/integrations/azure_storage/test_azure_storage.py @@ -1,12 +1,8 @@ -import os import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger from litellm.types.utils import StandardLoggingPayload diff --git a/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py b/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py index 142be536f6b..955821f66e0 100644 --- a/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py +++ b/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.integrations.bitbucket import BitBucketPromptManager diff --git a/tests/test_litellm/integrations/bitbucket/test_bitbucket_prompt_manager.py b/tests/test_litellm/integrations/bitbucket/test_bitbucket_prompt_manager.py index dd97de24df3..d6668bf9ad8 100644 --- a/tests/test_litellm/integrations/bitbucket/test_bitbucket_prompt_manager.py +++ b/tests/test_litellm/integrations/bitbucket/test_bitbucket_prompt_manager.py @@ -1,13 +1,9 @@ import json -import os -import sys +import re from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.integrations.bitbucket.bitbucket_client import BitBucketClient from litellm.integrations.bitbucket.bitbucket_prompt_manager import ( @@ -158,7 +154,7 @@ def test_bitbucket_client_get_file_content_access_denied(mock_get): client = BitBucketClient(config) - with pytest.raises(Exception, match="Access denied to file 'test.prompt'"): + with pytest.raises(Exception, match=re.escape("Access denied to file 'test.prompt'")): client.get_file_content("test.prompt") diff --git a/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py b/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py index a715116e5ee..1a95e45b2d5 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py +++ b/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py @@ -1,5 +1,3 @@ -import os -import sys import zoneinfo from datetime import datetime, timezone from unittest.mock import MagicMock, Mock, patch @@ -8,7 +6,6 @@ import httpx import polars as pl import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.integrations.cloudzero.cz_stream_api import CloudZeroStreamer diff --git a/tests/test_litellm/integrations/cloudzero/test_dry_run_endpoint.py b/tests/test_litellm/integrations/cloudzero/test_dry_run_endpoint.py index c5f377aa09b..795692f2cdf 100644 --- a/tests/test_litellm/integrations/cloudzero/test_dry_run_endpoint.py +++ b/tests/test_litellm/integrations/cloudzero/test_dry_run_endpoint.py @@ -2,14 +2,11 @@ Test the CloudZero dry run endpoint functionality """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import polars as pl import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger diff --git a/tests/test_litellm/integrations/cloudzero/test_transform.py b/tests/test_litellm/integrations/cloudzero/test_transform.py index 416eacdc63a..3ec2fe6779e 100644 --- a/tests/test_litellm/integrations/cloudzero/test_transform.py +++ b/tests/test_litellm/integrations/cloudzero/test_transform.py @@ -1,12 +1,9 @@ -import os -import sys from datetime import datetime from unittest.mock import MagicMock, patch import polars as pl import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.integrations.cloudzero.transform import CBFTransformer from litellm.types.integrations.cloudzero import CBFRecord diff --git a/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py b/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py index cb786d9c292..1a50a6991da 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_cost_management.py @@ -1,4 +1,3 @@ -import os import time from unittest.mock import AsyncMock @@ -12,34 +11,13 @@ from litellm.types.utils import StandardLoggingPayload @pytest.fixture -def clean_env(): - # Save original env - original_api_key = os.environ.get("DD_API_KEY") - original_app_key = os.environ.get("DD_APP_KEY") - original_site = os.environ.get("DD_SITE") - - # Set test env - os.environ["DD_API_KEY"] = "test_api_key" - os.environ["DD_APP_KEY"] = "test_app_key" - os.environ["DD_SITE"] = "test.datadoghq.com" - - yield - - # Restore original env - if original_api_key: - os.environ["DD_API_KEY"] = original_api_key - else: - del os.environ["DD_API_KEY"] - - if original_app_key: - os.environ["DD_APP_KEY"] = original_app_key - else: - del os.environ["DD_APP_KEY"] - - if original_site: - os.environ["DD_SITE"] = original_site - else: - del os.environ["DD_SITE"] +def clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key, value in ( + ("DD_API_KEY", "test_api_key"), + ("DD_APP_KEY", "test_app_key"), + ("DD_SITE", "test.datadoghq.com"), + ): + monkeypatch.setenv(key, value) @pytest.mark.asyncio diff --git a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py index a4a4ca334b0..eade92d6672 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_metrics.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_metrics.py @@ -1,4 +1,3 @@ -import os import time from datetime import datetime, timedelta from unittest.mock import AsyncMock @@ -11,25 +10,16 @@ from litellm.types.utils import StandardLoggingPayload @pytest.fixture -def clean_env(): - """Set test env vars and restore originals after test.""" - keys = ["DD_API_KEY", "DD_APP_KEY", "DD_SITE", "DD_ENV", "DD_SERVICE", "DD_VERSION"] - originals = {k: os.environ.get(k) for k in keys} - - os.environ["DD_API_KEY"] = "test_api_key" - os.environ["DD_APP_KEY"] = "test_app_key" - os.environ["DD_SITE"] = "test.datadoghq.com" - os.environ["DD_ENV"] = "test-env" - os.environ["DD_SERVICE"] = "test-service" - os.environ["DD_VERSION"] = "1.0.0" - - yield - - for k, v in originals.items(): - if v is not None: - os.environ[k] = v - elif k in os.environ: - del os.environ[k] +def clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key, value in ( + ("DD_API_KEY", "test_api_key"), + ("DD_APP_KEY", "test_app_key"), + ("DD_SITE", "test.datadoghq.com"), + ("DD_ENV", "test-env"), + ("DD_SERVICE", "test-service"), + ("DD_VERSION", "1.0.0"), + ): + monkeypatch.setenv(key, value) @pytest.mark.asyncio diff --git a/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py b/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py index 624995085aa..110ac75e73b 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py @@ -1,11 +1,9 @@ import datetime import os -import sys from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../../../")) from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_handler import get_datadog_tags, normalize_datadog_tag_value diff --git a/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py b/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py index d849582b3c4..b92ed13302e 100644 --- a/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py +++ b/tests/test_litellm/integrations/dotprompt/test_prompt_manager.py @@ -1,15 +1,10 @@ import json -import os -import sys import tempfile from pathlib import Path import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, Mock, patch diff --git a/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py b/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py index a4e16500aee..8d662311da1 100644 --- a/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py +++ b/tests/test_litellm/integrations/gcs_bucket/test_gcs_bucket_base.py @@ -8,10 +8,10 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase class TestGCSBucketBase: - def test_construct_request_headers_with_project_id(self): + def test_construct_request_headers_with_project_id(self, monkeypatch): """Test that construct_request_headers correctly uses project_id if passed from env""" test_project_id = "test-project" - os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = test_project_id + monkeypatch.setenv("GOOGLE_SECRET_MANAGER_PROJECT_ID", test_project_id) try: # Create handler diff --git a/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py b/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py index 7ff28bfe831..3c7f577d1d8 100644 --- a/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py +++ b/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py @@ -1,7 +1,6 @@ import datetime import json import os -import sys import unittest from typing import List, Optional, Tuple from unittest.mock import ANY, MagicMock, Mock, patch @@ -9,9 +8,6 @@ from unittest.mock import ANY, MagicMock, Mock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system-path import litellm diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_client.py b/tests/test_litellm/integrations/gitlab/test_gitlab_client.py index 529868ca06a..d6f588c4965 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_client.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_client.py @@ -1,13 +1,8 @@ import base64 import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.integrations.gitlab.gitlab_client import GitLabClient diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py b/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py index 8118af56b0e..7d5b490fea4 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py @@ -1,12 +1,7 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch from litellm.integrations.gitlab.gitlab_prompt_manager import GitLabPromptManager diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py b/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py index 1f7706882f6..120cc877b51 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py @@ -1,12 +1,8 @@ -import os -import sys +import re from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.integrations.gitlab.gitlab_client import GitLabClient from litellm.integrations.gitlab.gitlab_prompt_manager import ( @@ -172,7 +168,7 @@ def test_gitlab_client_get_file_content_access_denied(mock_get): mock_get.side_effect = err client = GitLabClient({"project": "g/s/r", "access_token": "tok"}) - with pytest.raises(Exception, match="Access denied to file 'test.prompt'"): + with pytest.raises(Exception, match=re.escape("Access denied to file 'test.prompt'")): client.get_file_content("test.prompt") diff --git a/tests/test_litellm/integrations/open_telemetry/conftest.py b/tests/test_litellm/integrations/open_telemetry/conftest.py index b29335aedd8..367e9fba07f 100644 --- a/tests/test_litellm/integrations/open_telemetry/conftest.py +++ b/tests/test_litellm/integrations/open_telemetry/conftest.py @@ -11,8 +11,6 @@ emitter in isolation. See ``LIT-3193_test_matrix.md`` (same directory) for the cell list. """ -import os -import sys from datetime import datetime from typing import Optional, Tuple from unittest.mock import MagicMock @@ -24,7 +22,6 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.integrations.opentelemetry import OpenTelemetry diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py index ca62253aa2f..a9a78dcb2f1 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py @@ -1,10 +1,7 @@ """Per-request multi-tenant credential routing (V1 parity).""" import base64 -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from opentelemetry.trace import NoOpTracer diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py index 7240d49d022..0cd71db4ae1 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py @@ -4,12 +4,9 @@ surface and the server-span + shared-provider behavior it produces. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) pytest.importorskip("opentelemetry") pytest.importorskip("opentelemetry.instrumentation.fastapi") diff --git a/tests/test_litellm/integrations/test_agentops.py b/tests/test_litellm/integrations/test_agentops.py index 85ee34a0d8c..5d4055ac75f 100644 --- a/tests/test_litellm/integrations/test_agentops.py +++ b/tests/test_litellm/integrations/test_agentops.py @@ -1,12 +1,8 @@ import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from litellm.integrations.agentops.agentops import AgentOps, AgentOpsConfig diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 7bf15f59eb9..b6e063a6d94 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -12,7 +12,6 @@ from unittest.mock import ANY, MagicMock, Mock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system-path import litellm from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, @@ -37,7 +36,7 @@ def _rendered_log_message(call): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_system_message(): +async def test_anthropic_cache_control_hook_system_message(monkeypatch: pytest.MonkeyPatch): # Use patch.dict to mock environment variables instead of setting them directly with patch.dict( os.environ, @@ -48,7 +47,7 @@ async def test_anthropic_cache_control_hook_system_message(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -116,7 +115,7 @@ async def test_anthropic_cache_control_hook_system_message(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_user_message(): +async def test_anthropic_cache_control_hook_user_message(monkeypatch: pytest.MonkeyPatch): # Use patch.dict to mock environment variables instead of setting them directly with patch.dict( os.environ, @@ -127,7 +126,7 @@ async def test_anthropic_cache_control_hook_user_message(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -188,7 +187,7 @@ async def test_anthropic_cache_control_hook_user_message(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_negative_indices(): +async def test_anthropic_cache_control_hook_negative_indices(monkeypatch: pytest.MonkeyPatch): """ Test the bug fix for handling negative indices in cache control injection points. This test verifies that negative indices (-1, -2) are properly converted to positive indices @@ -204,7 +203,7 @@ async def test_anthropic_cache_control_hook_negative_indices(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -302,7 +301,7 @@ async def test_anthropic_cache_control_hook_negative_indices(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_out_of_bounds_logging(): +async def test_anthropic_cache_control_hook_out_of_bounds_logging(monkeypatch: pytest.MonkeyPatch): """ Test that warning logs are generated when out-of-bounds indices are used. This verifies that the verbose_logger.warning is called with the correct message. @@ -316,7 +315,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -365,7 +364,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): +async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(monkeypatch: pytest.MonkeyPatch): """ Test that warning logs are generated for negative indices that are out of bounds. """ @@ -378,7 +377,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -431,7 +430,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_multiple_user_messages(): +async def test_anthropic_cache_control_hook_multiple_user_messages(monkeypatch: pytest.MonkeyPatch): """ Test cache control injection on multiple user messages specifically. Note: Bedrock API combines consecutive user messages into a single message with multiple content blocks. @@ -445,7 +444,7 @@ async def test_anthropic_cache_control_hook_multiple_user_messages(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -523,7 +522,7 @@ async def test_anthropic_cache_control_hook_multiple_user_messages(): @pytest.mark.asyncio @pytest.mark.parametrize("bad_index", [10, -10]) -async def test_anthropic_cache_control_hook_out_of_bounds(bad_index): +async def test_anthropic_cache_control_hook_out_of_bounds(bad_index, monkeypatch: pytest.MonkeyPatch): """ Verify the hook does not raise an error and makes no changes when an out-of-bounds index is provided. @@ -537,7 +536,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -586,7 +585,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index): "message_list", [[{"role": "user", "content": "Single message"}]], # Single message only - empty list will fail at API level ) -async def test_anthropic_cache_control_hook_single_message(message_list): +async def test_anthropic_cache_control_hook_single_message(message_list, monkeypatch: pytest.MonkeyPatch): """ Verify the hook runs without error on very short message lists. """ @@ -599,7 +598,7 @@ async def test_anthropic_cache_control_hook_single_message(message_list): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -637,7 +636,7 @@ async def test_anthropic_cache_control_hook_single_message(message_list): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_empty_message_list(): +async def test_anthropic_cache_control_hook_empty_message_list(monkeypatch: pytest.MonkeyPatch): """ Verify that empty message lists are handled appropriately (should fail at API level, not hook level). """ @@ -650,7 +649,7 @@ async def test_anthropic_cache_control_hook_empty_message_list(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) client = AsyncHTTPHandler() with patch.object(client, "post", return_value=MagicMock()) as mock_post: @@ -668,7 +667,7 @@ async def test_anthropic_cache_control_hook_empty_message_list(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_no_op(): +async def test_anthropic_cache_control_hook_no_op(monkeypatch: pytest.MonkeyPatch): """ Verify that if no injection points are specified, messages remain unmodified. """ @@ -681,7 +680,7 @@ async def test_anthropic_cache_control_hook_no_op(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) # Mock response data mock_response = MagicMock() @@ -726,7 +725,7 @@ async def test_anthropic_cache_control_hook_no_op(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_multiple_content_items_last_only(): +async def test_anthropic_cache_control_hook_multiple_content_items_last_only(monkeypatch: pytest.MonkeyPatch): """ Test that cache_control is only applied to the last content item in a list, not all items. This verifies the fix for https://github.com/BerriAI/litellm/issues/15696 @@ -740,7 +739,7 @@ async def test_anthropic_cache_control_hook_multiple_content_items_last_only(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) mock_response = MagicMock() mock_response.json.return_value = { @@ -797,7 +796,7 @@ async def test_anthropic_cache_control_hook_multiple_content_items_last_only(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_document_analysis_multiple_pages(): +async def test_anthropic_cache_control_hook_document_analysis_multiple_pages(monkeypatch: pytest.MonkeyPatch): """ Test cache_control with multiple document pages to ensure only the last page gets cached. This simulates document analysis with 6 content blocks, verifying the fix for issue 15696. @@ -811,7 +810,7 @@ async def test_anthropic_cache_control_hook_document_analysis_multiple_pages(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) mock_response = MagicMock() mock_response.json.return_value = { @@ -969,7 +968,7 @@ def test_gemini_cache_control_injection_list_content_detected(): @pytest.mark.asyncio -async def test_anthropic_cache_control_hook_string_negative_index(): +async def test_anthropic_cache_control_hook_string_negative_index(monkeypatch: pytest.MonkeyPatch): """ Test that string negative indices like "-1" are handled correctly. @@ -986,7 +985,7 @@ async def test_anthropic_cache_control_hook_string_negative_index(): }, ): anthropic_cache_control_hook = AnthropicCacheControlHook() - litellm.callbacks = [anthropic_cache_control_hook] + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) mock_response = MagicMock() mock_response.json.return_value = { @@ -1185,7 +1184,7 @@ def test_cache_control_hook_does_not_overwrite_existing_cache_control(): @pytest.mark.asyncio -async def test_cache_control_hook_bedrock_payload_caps_cachepoints_at_four(): +async def test_cache_control_hook_bedrock_payload_caps_cachepoints_at_four(monkeypatch: pytest.MonkeyPatch): """End-to-end: outgoing Bedrock payload must not exceed 4 cachePoint blocks. Reproduces the customer report where 4 client cache_control system blocks @@ -1199,7 +1198,7 @@ async def test_cache_control_hook_bedrock_payload_caps_cachepoints_at_four(): "AWS_REGION_NAME": "us-east-1", }, ): - litellm.callbacks = [AnthropicCacheControlHook()] + monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) mock_response = MagicMock() mock_response.json.return_value = { @@ -1289,7 +1288,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point(): @pytest.mark.asyncio -async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point(): +async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point(monkeypatch: pytest.MonkeyPatch): """End-to-end: message + tool_config injection must not exceed 4 cachePoints.""" with patch.dict( os.environ, @@ -1299,7 +1298,7 @@ async def test_cache_control_hook_bedrock_payload_caps_with_tool_config_point(): "AWS_REGION_NAME": "us-east-1", }, ): - litellm.callbacks = [AnthropicCacheControlHook()] + monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) mock_response = MagicMock() mock_response.json.return_value = { diff --git a/tests/test_litellm/integrations/test_athina.py b/tests/test_litellm/integrations/test_athina.py index 49d8fc693e7..4f64f26db9a 100644 --- a/tests/test_litellm/integrations/test_athina.py +++ b/tests/test_litellm/integrations/test_athina.py @@ -1,13 +1,8 @@ import datetime import json -import os -import sys import unittest from unittest.mock import ANY, MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path from litellm.integrations.athina import AthinaLogger diff --git a/tests/test_litellm/integrations/test_custom_prompt_management.py b/tests/test_litellm/integrations/test_custom_prompt_management.py index 7d5d02bf4b6..0bf2063a98d 100644 --- a/tests/test_litellm/integrations/test_custom_prompt_management.py +++ b/tests/test_litellm/integrations/test_custom_prompt_management.py @@ -1,7 +1,5 @@ import datetime import json -import os -import sys import unittest from typing import List, Optional, Tuple from unittest.mock import ANY, MagicMock, Mock, patch @@ -9,9 +7,6 @@ from unittest.mock import ANY, MagicMock, Mock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system-path import litellm from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler diff --git a/tests/test_litellm/integrations/test_galileo.py b/tests/test_litellm/integrations/test_galileo.py index 8905795bbc6..d0709b966d4 100644 --- a/tests/test_litellm/integrations/test_galileo.py +++ b/tests/test_litellm/integrations/test_galileo.py @@ -1,11 +1,8 @@ -import os -import sys from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.galileo import GalileoObserve from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse diff --git a/tests/test_litellm/integrations/test_helicone.py b/tests/test_litellm/integrations/test_helicone.py index da07fa1a9bf..64960de050a 100644 --- a/tests/test_litellm/integrations/test_helicone.py +++ b/tests/test_litellm/integrations/test_helicone.py @@ -1,8 +1,6 @@ -import os import sys import types -sys.path.insert(0, os.path.abspath("../../..")) from litellm.integrations.helicone import HeliconeLogger diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 73a62e5594d..747f733a46d 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -1,6 +1,5 @@ import datetime import json -import os import sys import types import unittest @@ -13,7 +12,6 @@ import litellm from litellm.integrations.langfuse import langfuse as langfuse_module from litellm.integrations.langfuse.langfuse import LangFuseLogger -sys.path.insert(0, os.path.abspath("../..")) # Import LangfuseUsageDetails directly from the module where it's defined from litellm.types.integrations.langfuse import * diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index 129dda4abde..d3393ac3d28 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -1,10 +1,8 @@ import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.integrations.langsmith import LangsmithLogger diff --git a/tests/test_litellm/integrations/test_lunary.py b/tests/test_litellm/integrations/test_lunary.py index 0a1ec100594..6491f5c8b82 100644 --- a/tests/test_litellm/integrations/test_lunary.py +++ b/tests/test_litellm/integrations/test_lunary.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) from litellm.integrations.lunary import parse_tool_calls from litellm.types.utils import ( diff --git a/tests/test_litellm/integrations/test_mlflow.py b/tests/test_litellm/integrations/test_mlflow.py index 32358641984..61010f8531c 100644 --- a/tests/test_litellm/integrations/test_mlflow.py +++ b/tests/test_litellm/integrations/test_mlflow.py @@ -1,12 +1,9 @@ import asyncio import json -import os -import sys from datetime import datetime from unittest.mock import MagicMock, patch # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) import pytest diff --git a/tests/test_litellm/integrations/test_openmeter.py b/tests/test_litellm/integrations/test_openmeter.py index 539e3f99cdc..2d09e1572db 100644 --- a/tests/test_litellm/integrations/test_openmeter.py +++ b/tests/test_litellm/integrations/test_openmeter.py @@ -33,7 +33,7 @@ class TestOpenMeterIntegration: def test_openmeter_logger_missing_api_key(self): """Test that OpenMeterLogger raises exception when API key is missing""" os.environ.pop("OPENMETER_API_KEY", None) - with pytest.raises(Exception, match="Missing keys.*OPENMETER_API_KEY"): + with pytest.raises(Exception, match=r"Missing keys.*OPENMETER_API_KEY"): OpenMeterLogger() def test_common_logic_with_string_user(self): @@ -236,9 +236,9 @@ class TestOpenMeterIntegration: assert result["data"]["completion_tokens"] == 8 assert result["data"]["total_tokens"] == 23 - def test_custom_event_type(self): + def test_custom_event_type(self, monkeypatch): """Test that custom event type is used when set""" - os.environ["OPENMETER_EVENT_TYPE"] = "custom_event_type" + monkeypatch.setenv("OPENMETER_EVENT_TYPE", "custom_event_type") logger = OpenMeterLogger() @@ -374,10 +374,10 @@ class TestOpenMeterIntegration: assert isinstance(result["subject"], str) assert result["subject"] == "12345" - def test_common_logic_trust_request_user_false_ignores_request_user(self): + def test_common_logic_trust_request_user_false_ignores_request_user(self, monkeypatch): """OPENMETER_TRUST_REQUEST_USER=false makes the key-bound user_id win over a request-supplied `user` (forge-attribution mitigation).""" - os.environ["OPENMETER_TRUST_REQUEST_USER"] = "false" + monkeypatch.setenv("OPENMETER_TRUST_REQUEST_USER", "false") logger = OpenMeterLogger() kwargs = { @@ -400,11 +400,11 @@ class TestOpenMeterIntegration: assert result["subject"] == "real-tenant-id" assert result["subject"] != "forged-by-client" - def test_common_logic_trust_request_user_false_still_raises_without_key_user(self): + def test_common_logic_trust_request_user_false_still_raises_without_key_user(self, monkeypatch): """OPENMETER_TRUST_REQUEST_USER=false still raises when no user_api_key_user_id is available — the request `user` is not a fallback in this mode.""" - os.environ["OPENMETER_TRUST_REQUEST_USER"] = "false" + monkeypatch.setenv("OPENMETER_TRUST_REQUEST_USER", "false") logger = OpenMeterLogger() kwargs = { diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index a5ad3d771e3..229214bf1e1 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -14,7 +14,6 @@ from parameterized import parameterized from unittest.mock import MagicMock, patch # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) from opentelemetry import trace from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor diff --git a/tests/test_litellm/integrations/test_otel_guardrail_violation_spans.py b/tests/test_litellm/integrations/test_otel_guardrail_violation_spans.py index c3e9d67ddad..16d77fe38a5 100644 --- a/tests/test_litellm/integrations/test_otel_guardrail_violation_spans.py +++ b/tests/test_litellm/integrations/test_otel_guardrail_violation_spans.py @@ -24,8 +24,6 @@ real ``OpenTelemetry`` integration. No monkey patching of the integration under test — only the OTEL exporter is in-memory. """ -import os -import sys import time import unittest from datetime import datetime, timedelta, timezone @@ -35,7 +33,6 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import StatusCode -sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.opentelemetry import ( LITELLM_REQUEST_SPAN_NAME, diff --git a/tests/test_litellm/integrations/test_otel_team_attributes_matrix.py b/tests/test_litellm/integrations/test_otel_team_attributes_matrix.py index 1ce55fa7a58..daf7d0fdaf0 100644 --- a/tests/test_litellm/integrations/test_otel_team_attributes_matrix.py +++ b/tests/test_litellm/integrations/test_otel_team_attributes_matrix.py @@ -31,8 +31,6 @@ Strategy """ import asyncio -import os -import sys import unittest from datetime import datetime from unittest.mock import MagicMock @@ -43,7 +41,6 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) -sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.opentelemetry import ( LITELLM_PROXY_REQUEST_SPAN_NAME, diff --git a/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py b/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py index 5dbe487ab0a..278a4ef1df6 100644 --- a/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py +++ b/tests/test_litellm/integrations/test_prometheus_invalid_key_filtering.py @@ -5,14 +5,11 @@ Tests functionality that prevents invalid API key requests (401 status codes) from being recorded in Prometheus metrics. """ -import os -import sys from unittest.mock import Mock, patch import pytest from prometheus_client import REGISTRY -sys.path.insert(0, os.path.abspath("../../..")) from litellm.integrations.prometheus import PrometheusLogger from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_litellm/integrations/test_prometheus_none_metadata.py b/tests/test_litellm/integrations/test_prometheus_none_metadata.py index fff2e48bf5a..c2d4c831609 100644 --- a/tests/test_litellm/integrations/test_prometheus_none_metadata.py +++ b/tests/test_litellm/integrations/test_prometheus_none_metadata.py @@ -6,14 +6,11 @@ can be None, causing AttributeError: 'NoneType' object has no attribute 'get' in set_llm_deployment_success_metrics. """ -import os -import sys from datetime import datetime import pytest from prometheus_client import REGISTRY -sys.path.insert(0, os.path.abspath("../../..")) from litellm.integrations.prometheus import PrometheusLogger from litellm.types.integrations.prometheus import UserAPIKeyLabelValues diff --git a/tests/test_litellm/integrations/test_prometheus_remaining_tokens_router_fallback.py b/tests/test_litellm/integrations/test_prometheus_remaining_tokens_router_fallback.py index d754de86569..45b378d10fe 100644 --- a/tests/test_litellm/integrations/test_prometheus_remaining_tokens_router_fallback.py +++ b/tests/test_litellm/integrations/test_prometheus_remaining_tokens_router_fallback.py @@ -20,14 +20,11 @@ Tests cover: - llm_router unavailable / model_group missing / router raises → silent no-op. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from prometheus_client import REGISTRY -sys.path.insert(0, os.path.abspath("../../..")) from litellm.integrations.prometheus import PrometheusLogger from litellm.types.integrations.prometheus import UserAPIKeyLabelValues diff --git a/tests/test_litellm/integrations/test_prometheus_services.py b/tests/test_litellm/integrations/test_prometheus_services.py index 2efd226dc9d..2303061ede8 100644 --- a/tests/test_litellm/integrations/test_prometheus_services.py +++ b/tests/test_litellm/integrations/test_prometheus_services.py @@ -1,6 +1,4 @@ import json -import os -import sys import time from unittest.mock import AsyncMock, patch @@ -13,9 +11,6 @@ from litellm.integrations.prometheus_services import ( ServiceTypes, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path def test_is_metric_registered_does_not_use_registry_collect(): diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index 8cccfd937e7..933e41d17a0 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -751,7 +751,7 @@ async def test_strip_base64_mixed_nested_objects(): @pytest.mark.asyncio -async def test_s3_verify_false_handling(): +async def test_s3_verify_false_handling(monkeypatch: pytest.MonkeyPatch): """ Test that s3_verify=False is properly handled and not treated as None. @@ -763,15 +763,19 @@ async def test_s3_verify_false_handling(): import litellm # Set up s3_callback_params with s3_verify=False - litellm.s3_callback_params = { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, # This should NOT be ignored - "s3_use_ssl": False, # This should also NOT be ignored - } + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": "https://localhost:443", + "s3_aws_access_key_id": "minioadmin", + "s3_aws_secret_access_key": "minioadmin", + "s3_region_name": "us-east-1", + "s3_verify": False, # This should NOT be ignored + "s3_use_ssl": False, # This should also NOT be ignored + }, + ) with patch("asyncio.create_task"): with patch( @@ -801,12 +805,9 @@ async def test_s3_verify_false_handling(): "ssl_verify": False }, f"Expected ssl_verify=False in params, got {call_kwargs.get('params')}" - # Clean up - litellm.s3_callback_params = None - @pytest.mark.asyncio -async def test_s3_verify_none_handling(): +async def test_s3_verify_none_handling(monkeypatch: pytest.MonkeyPatch): """ Test that s3_verify=None uses default behavior. """ @@ -815,12 +816,16 @@ async def test_s3_verify_none_handling(): import litellm # Set up s3_callback_params without s3_verify - litellm.s3_callback_params = { - "s3_bucket_name": "test-bucket", - "s3_aws_access_key_id": "test-key", - "s3_aws_secret_access_key": "test-secret", - "s3_region_name": "us-east-1", - } + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_aws_access_key_id": "test-key", + "s3_aws_secret_access_key": "test-secret", + "s3_region_name": "us-east-1", + }, + ) with patch("asyncio.create_task"): with patch( @@ -846,12 +851,9 @@ async def test_s3_verify_none_handling(): assert call_kwargs["params"].get("ssl_verify") is None # Either params is None or params={'ssl_verify': None} is acceptable - # Clean up - litellm.s3_callback_params = None - @pytest.mark.asyncio -async def test_s3_verify_false_creates_httpx_client_with_verify_false(): +async def test_s3_verify_false_creates_httpx_client_with_verify_false(monkeypatch: pytest.MonkeyPatch): """ Test that when s3_verify=False, the actual httpx client has verify=False. @@ -862,14 +864,18 @@ async def test_s3_verify_false_creates_httpx_client_with_verify_false(): import litellm # Set up s3_callback_params with s3_verify=False - litellm.s3_callback_params = { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, - } + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": "https://localhost:443", + "s3_aws_access_key_id": "minioadmin", + "s3_aws_secret_access_key": "minioadmin", + "s3_region_name": "us-east-1", + "s3_verify": False, + }, + ) with patch("asyncio.create_task"): # Create logger - this creates the httpx client @@ -888,12 +894,9 @@ async def test_s3_verify_false_creates_httpx_client_with_verify_false(): httpx_client._verify is False ), f"Expected httpx client _verify=False, got {httpx_client._verify}" - # Clean up - litellm.s3_callback_params = None - @pytest.mark.asyncio -async def test_s3_verify_false_async_client(): +async def test_s3_verify_false_async_client(monkeypatch: pytest.MonkeyPatch): """ Test that the async httpx client respects s3_verify=False. """ @@ -903,14 +906,18 @@ async def test_s3_verify_false_async_client(): from litellm.types.integrations.s3_v2 import s3BatchLoggingElement # Set up s3_callback_params with s3_verify=False - litellm.s3_callback_params = { - "s3_bucket_name": "test-bucket", - "s3_endpoint_url": "https://localhost:443", - "s3_aws_access_key_id": "minioadmin", - "s3_aws_secret_access_key": "minioadmin", - "s3_region_name": "us-east-1", - "s3_verify": False, - } + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "test-bucket", + "s3_endpoint_url": "https://localhost:443", + "s3_aws_access_key_id": "minioadmin", + "s3_aws_secret_access_key": "minioadmin", + "s3_region_name": "us-east-1", + "s3_verify": False, + }, + ) with patch("asyncio.create_task"): logger = S3Logger() @@ -945,9 +952,6 @@ async def test_s3_verify_false_async_client(): httpx_client._verify is False ), f"Expected async httpx client _verify=False, got {httpx_client._verify}" - # Clean up - litellm.s3_callback_params = None - @pytest.mark.asyncio async def test_strip_base64_recursive_redaction(): @@ -1169,26 +1173,22 @@ def test_create_s3_batch_logging_element_flat_key_for_arn_response_id(): # -------------------------------------------------------------- # params_source / s3_callback_params_override (audit-log decoupling) # -------------------------------------------------------------- -def test_s3_callback_params_override_uses_alternate_dict(): +def test_s3_callback_params_override_uses_alternate_dict(monkeypatch): """`s3_callback_params_override` makes the logger read its config from the override dict instead of `litellm.s3_callback_params`.""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = {"s3_bucket_name": "normal-bucket"} - try: - logger = S3Logger( - s3_callback_params_override={ - "s3_bucket_name": "audit-bucket", - "s3_path": "audit-prefix", - "s3_region_name": "us-west-2", - } - ) - assert logger.s3_bucket_name == "audit-bucket" - assert logger.s3_path == "audit-prefix" - assert logger.s3_region_name == "us-west-2" - finally: - litellm.s3_callback_params = original + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-bucket"}) + logger = S3Logger( + s3_callback_params_override={ + "s3_bucket_name": "audit-bucket", + "s3_path": "audit-prefix", + "s3_region_name": "us-west-2", + } + ) + assert logger.s3_bucket_name == "audit-bucket" + assert logger.s3_path == "audit-prefix" + assert logger.s3_region_name == "us-west-2" def test_s3_callback_params_override_does_not_mutate_inputs(monkeypatch): @@ -1198,43 +1198,31 @@ def test_s3_callback_params_override_does_not_mutate_inputs(monkeypatch): monkeypatch.setenv("MY_AUDIT_BUCKET", "resolved-bucket") override = {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"} - original_global = litellm.s3_callback_params - litellm.s3_callback_params = {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"} - try: - logger = S3Logger(s3_callback_params_override=override) - assert logger.s3_bucket_name == "resolved-bucket" - assert override["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" - assert ( - litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" - ) - finally: - litellm.s3_callback_params = original_global + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"}) + logger = S3Logger(s3_callback_params_override=override) + assert logger.s3_bucket_name == "resolved-bucket" + assert override["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" + assert ( + litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" + ) -def test_s3_callback_params_override_none_falls_back_to_global(): +def test_s3_callback_params_override_none_falls_back_to_global(monkeypatch): """No override → behaves exactly as today (reads `litellm.s3_callback_params`).""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = {"s3_bucket_name": "from-global"} - try: - logger = S3Logger() - assert logger.s3_bucket_name == "from-global" - finally: - litellm.s3_callback_params = original + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "from-global"}) + logger = S3Logger() + assert logger.s3_bucket_name == "from-global" -def test_s3_callback_params_override_empty_dict_is_opt_in(): +def test_s3_callback_params_override_empty_dict_is_opt_in(monkeypatch): """An empty override dict skips the global entirely (env/IAM-only config).""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = {"s3_bucket_name": "from-global"} - try: - logger = S3Logger(s3_callback_params_override={}) - assert logger.s3_bucket_name is None - finally: - litellm.s3_callback_params = original + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "from-global"}) + logger = S3Logger(s3_callback_params_override={}) + assert logger.s3_bucket_name is None def _expected_content_md5(payload: dict) -> str: @@ -1374,20 +1362,20 @@ async def test_async_upload_sets_server_side_encryption_header_when_configured() assert headers["x-amz-server-side-encryption"] == "aws:kms" -def test_s3_server_side_encryption_read_from_callback_params(): +def test_s3_server_side_encryption_read_from_callback_params(monkeypatch): """s3_server_side_encryption can be configured via s3_callback_params.""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - } - try: - logger = S3Logger() - assert logger.s3_server_side_encryption == "aws:kms" - finally: - litellm.s3_callback_params = original + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + }, + ) + logger = S3Logger() + assert logger.s3_server_side_encryption == "aws:kms" @pytest.mark.asyncio @@ -1505,21 +1493,21 @@ async def test_async_upload_omits_kms_key_id_header_when_not_configured(): assert "x-amz-server-side-encryption-aws-kms-key-id" not in headers -def test_s3_sse_kms_key_id_read_from_callback_params(): +def test_s3_sse_kms_key_id_read_from_callback_params(monkeypatch): """s3_sse_kms_key_id can be configured via s3_callback_params.""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - } - try: - logger = S3Logger() - assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") - finally: - litellm.s3_callback_params = original + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, + ) + logger = S3Logger() + assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") @pytest.mark.asyncio @@ -1561,83 +1549,79 @@ async def test_async_upload_infers_aws_kms_when_only_key_id_set(): ) -def test_s3_sse_kms_key_id_read_from_audit_override_params(): +def test_s3_sse_kms_key_id_read_from_audit_override_params(monkeypatch): """The audit-log override path must honor s3_sse_kms_key_id too.""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = {"s3_bucket_name": "normal-logs-bucket"} - try: - logger = S3Logger( - s3_callback_params_override={ - "s3_bucket_name": "audit-logs-bucket", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/audit-key-id", - } - ) - assert logger.s3_bucket_name == "audit-logs-bucket" - assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/audit-key-id") - finally: - litellm.s3_callback_params = original + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-logs-bucket"}) + logger = S3Logger( + s3_callback_params_override={ + "s3_bucket_name": "audit-logs-bucket", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/audit-key-id", + } + ) + assert logger.s3_bucket_name == "audit-logs-bucket" + assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/audit-key-id") -def test_kms_key_id_dropped_when_algorithm_is_not_kms(): +def test_kms_key_id_dropped_when_algorithm_is_not_kms(monkeypatch): """ AES256 plus a KMS key id is an invalid S3 combination; the key id must be dropped at init so uploads keep working instead of silently 400ing. """ import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "AES256", - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - } - try: - logger = S3Logger() - assert logger.s3_server_side_encryption == "AES256" - assert logger.s3_sse_kms_key_id is None - finally: - litellm.s3_callback_params = original + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "AES256", + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, + ) + logger = S3Logger() + assert logger.s3_server_side_encryption == "AES256" + assert logger.s3_sse_kms_key_id is None -def test_non_string_algorithm_is_dropped_and_valid_key_id_is_rescued(): +def test_non_string_algorithm_is_dropped_and_valid_key_id_is_rescued(monkeypatch): """ A YAML boolean in s3_server_side_encryption must not crash logger init and must not discard the valid key id; aws:kms is inferred from the key id. """ import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": True, - "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", - } - try: - logger = S3Logger() - assert logger.s3_server_side_encryption == "aws:kms" - assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") - finally: - litellm.s3_callback_params = original + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": True, + "s3_sse_kms_key_id": "arn:aws:kms:us-east-1:111122223333:key/test-key-id", + }, + ) + logger = S3Logger() + assert logger.s3_server_side_encryption == "aws:kms" + assert logger.s3_sse_kms_key_id == ("arn:aws:kms:us-east-1:111122223333:key/test-key-id") -def test_non_string_key_id_is_dropped_and_valid_algorithm_is_kept(): +def test_non_string_key_id_is_dropped_and_valid_algorithm_is_kept(monkeypatch): """A mistyped key id (unquoted YAML number) must not disable the valid algorithm.""" import litellm - original = litellm.s3_callback_params - litellm.s3_callback_params = { - "s3_bucket_name": "from-global", - "s3_server_side_encryption": "aws:kms", - "s3_sse_kms_key_id": 12345, - } - try: - logger = S3Logger() - assert logger.s3_server_side_encryption == "aws:kms" - assert logger.s3_sse_kms_key_id is None - finally: - litellm.s3_callback_params = original + monkeypatch.setattr( + litellm, + "s3_callback_params", + { + "s3_bucket_name": "from-global", + "s3_server_side_encryption": "aws:kms", + "s3_sse_kms_key_id": 12345, + }, + ) + logger = S3Logger() + assert logger.s3_server_side_encryption == "aws:kms" + assert logger.s3_sse_kms_key_id is None _ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE" diff --git a/tests/test_litellm/interactions/test_agents_http_handler.py b/tests/test_litellm/interactions/test_agents_http_handler.py index 6947503e0bb..31b78d7a360 100644 --- a/tests/test_litellm/interactions/test_agents_http_handler.py +++ b/tests/test_litellm/interactions/test_agents_http_handler.py @@ -8,14 +8,11 @@ branches, error mapping, and pre/post logging hooks. No real HTTP traffic is made. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.interactions.agents.http_handler import ( AgentsHTTPHandler, diff --git a/tests/test_litellm/interactions/test_agents_main_and_utils.py b/tests/test_litellm/interactions/test_agents_main_and_utils.py index f5523cf1cf8..395801ff059 100644 --- a/tests/test_litellm/interactions/test_agents_main_and_utils.py +++ b/tests/test_litellm/interactions/test_agents_main_and_utils.py @@ -9,13 +9,10 @@ small helper utilities without touching the network. """ import asyncio -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.interactions.agents import ( diff --git a/tests/test_litellm/interactions/test_gemini_interactions_transformation.py b/tests/test_litellm/interactions/test_gemini_interactions_transformation.py index 524589abf5e..d9b7cc790e6 100644 --- a/tests/test_litellm/interactions/test_gemini_interactions_transformation.py +++ b/tests/test_litellm/interactions/test_gemini_interactions_transformation.py @@ -8,13 +8,10 @@ Covers: - transform_request: response_mime_type coalescing, image_config migration """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.interactions.litellm_responses_transformation.streaming_iterator import ( @@ -86,29 +83,21 @@ class TestValidateEnvironment: assert headers["X-Custom"] == "value" assert headers["x-goog-api-key"] == "test-key" - def test_api_revision_new_schema_by_default(self, config): + def test_api_revision_new_schema_by_default(self, config, monkeypatch: pytest.MonkeyPatch): # Default: use_legacy_interactions_schema=False → new steps schema - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = False - headers = config.validate_environment( - headers={}, model="gemini-2.5-flash", litellm_params=None - ) - assert headers["Api-Revision"] == "2026-05-20" - finally: - litellm.use_legacy_interactions_schema = original + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + headers = config.validate_environment( + headers={}, model="gemini-2.5-flash", litellm_params=None + ) + assert headers["Api-Revision"] == "2026-05-20" - def test_api_revision_legacy_schema_when_flag_set(self, config): + def test_api_revision_legacy_schema_when_flag_set(self, config, monkeypatch: pytest.MonkeyPatch): # Flag on → legacy outputs schema until June 8, 2026 - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = True - headers = config.validate_environment( - headers={}, model="gemini-2.5-flash", litellm_params=None - ) - assert headers["Api-Revision"] == "2026-05-07" - finally: - litellm.use_legacy_interactions_schema = original + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", True) + headers = config.validate_environment( + headers={}, model="gemini-2.5-flash", litellm_params=None + ) + assert headers["Api-Revision"] == "2026-05-07" class TestGetCompleteUrl: @@ -561,23 +550,19 @@ class TestInteractionOperationUrls: class TestTransformRequestSchemaCoalescing: """Test new-schema request coalescing (Api-Revision: 2026-05-20).""" - def test_response_mime_type_folded_into_response_format(self, config): - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = False - body = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="summarise", - optional_params={ - "response_mime_type": "application/json", - "response_format": {"type": "object", "properties": {}}, - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + def test_response_mime_type_folded_into_response_format(self, config, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + body = config.transform_request( + model="gemini/gemini-2.5-flash", + agent=None, + input="summarise", + optional_params={ + "response_mime_type": "application/json", + "response_format": {"type": "object", "properties": {}}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) # response_mime_type must not appear as a top-level body key assert "response_mime_type" not in body @@ -586,25 +571,21 @@ class TestTransformRequestSchemaCoalescing: assert rf["mime_type"] == "application/json" assert "schema" in rf - def test_image_config_moved_to_response_format(self, config): - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = False - body = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="draw a sunset", - optional_params={ - "generation_config": { - "temperature": 0.7, - "image_config": {"aspect_ratio": "1:1", "image_size": "1K"}, - } - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + def test_image_config_moved_to_response_format(self, config, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + body = config.transform_request( + model="gemini/gemini-2.5-flash", + agent=None, + input="draw a sunset", + optional_params={ + "generation_config": { + "temperature": 0.7, + "image_config": {"aspect_ratio": "1:1", "image_size": "1K"}, + } + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) # image_config removed from generation_config assert "image_config" not in body.get("generation_config", {}) @@ -613,95 +594,85 @@ class TestTransformRequestSchemaCoalescing: assert rf["type"] == "image" assert rf["aspect_ratio"] == "1:1" - def test_response_mime_type_skipped_when_response_format_is_list(self, config): + def test_response_mime_type_skipped_when_response_format_is_list(self, config, monkeypatch: pytest.MonkeyPatch): """Lists are already polymorphic; do not wrap them into schema.""" - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = False - rf_list = [ - {"type": "text", "mime_type": "application/json"}, - {"type": "image", "aspect_ratio": "1:1"}, - ] - body = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="multimodal", - optional_params={ - "response_format": rf_list, - "response_mime_type": "application/json", - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + rf_list = [ + {"type": "text", "mime_type": "application/json"}, + {"type": "image", "aspect_ratio": "1:1"}, + ] + body = config.transform_request( + model="gemini/gemini-2.5-flash", + agent=None, + input="multimodal", + optional_params={ + "response_format": rf_list, + "response_mime_type": "application/json", + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) assert body["response_format"] == rf_list assert "response_mime_type" not in body def test_image_config_appended_to_response_format_list_without_mutating_input( - self, config + self, + config, + monkeypatch: pytest.MonkeyPatch, ): """When response_format is already a list, image_config must not mutate optional_params.""" - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = False - text_rf = {"type": "text", "mime_type": "application/json"} - optional_params = { - "response_format": [text_rf], - "generation_config": { - "image_config": {"aspect_ratio": "16:9", "image_size": "2K"}, - }, - } - original_rf = optional_params["response_format"] + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + text_rf = {"type": "text", "mime_type": "application/json"} + optional_params = { + "response_format": [text_rf], + "generation_config": { + "image_config": {"aspect_ratio": "16:9", "image_size": "2K"}, + }, + } + original_rf = optional_params["response_format"] - body = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="draw and summarise", - optional_params=optional_params, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) + body = config.transform_request( + model="gemini/gemini-2.5-flash", + agent=None, + input="draw and summarise", + optional_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) - assert optional_params["response_format"] is original_rf - assert len(optional_params["response_format"]) == 1 - assert body["response_format"] == [ - text_rf, - {"type": "image", "aspect_ratio": "16:9", "image_size": "2K"}, - ] + assert optional_params["response_format"] is original_rf + assert len(optional_params["response_format"]) == 1 + assert body["response_format"] == [ + text_rf, + {"type": "image", "aspect_ratio": "16:9", "image_size": "2K"}, + ] - # Retry must not append a second image entry into the caller's list. - body_retry = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="draw and summarise", - optional_params=optional_params, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - assert len(optional_params["response_format"]) == 1 - assert body_retry["response_format"] == body["response_format"] - finally: - litellm.use_legacy_interactions_schema = original + # Retry must not append a second image entry into the caller's list. + body_retry = config.transform_request( + model="gemini/gemini-2.5-flash", + agent=None, + input="draw and summarise", + optional_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert len(optional_params["response_format"]) == 1 + assert body_retry["response_format"] == body["response_format"] - def test_legacy_schema_passes_fields_unchanged(self, config): - original = litellm.use_legacy_interactions_schema - try: - litellm.use_legacy_interactions_schema = True - body = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="hello", - optional_params={ - "response_mime_type": "application/json", - "generation_config": {"image_config": {"aspect_ratio": "16:9"}}, - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + def test_legacy_schema_passes_fields_unchanged(self, config, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "use_legacy_interactions_schema", True) + body = config.transform_request( + model="gemini/gemini-2.5-flash", + agent=None, + input="hello", + optional_params={ + "response_mime_type": "application/json", + "generation_config": {"image_config": {"aspect_ratio": "16:9"}}, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) assert body["response_mime_type"] == "application/json" assert body["generation_config"]["image_config"]["aspect_ratio"] == "16:9" diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/test_litellm/interactions/test_google_interactions_integration.py index 49cd978c683..93429d64789 100644 --- a/tests/test_litellm/interactions/test_google_interactions_integration.py +++ b/tests/test_litellm/interactions/test_google_interactions_integration.py @@ -10,11 +10,9 @@ Run with: pytest tests/test_litellm/interactions/test_google_interactions_integr import asyncio import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm import litellm.interactions as interactions diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index cf36a2b9b25..052c08a86b5 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -56,8 +56,8 @@ def test_bedrock_guardrail_cost_no_pricing_entry(monkeypatch): assert bedrock_guardrail_cost(usage_units={"contentPolicyUnits": 1}, aws_region_name="us-east-1") == 0.0 -def test_shipped_bedrock_guardrail_prices_match_aws_pricing_page(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_shipped_bedrock_guardrail_prices_match_aws_pricing_page(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") assert litellm.model_cost["bedrock/guardrails"]["guardrail_cost_per_unit"] == { "automatedReasoningPolicyUnits": 0.00017, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index f66056a54e2..c8c36032793 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,6 +1,4 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient @@ -28,10 +26,6 @@ from litellm.types.utils import ( StandardBuiltInToolsParams, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path - from litellm.litellm_core_utils.llm_cost_calc.utils import ( PromptTokensDetailsResult, TokenTypeCostBreakdown, @@ -44,13 +38,17 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( from litellm.types.utils import CacheCreationTokenDetails, Usage -def test_reasoning_tokens_no_price_set(): +@pytest.fixture +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + +def test_reasoning_tokens_no_price_set(_local_model_cost_map): # Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics # (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token) model = "o1" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] usage = Usage( completion_tokens=1578, @@ -87,11 +85,9 @@ def test_reasoning_tokens_no_price_set(): ) -def test_reasoning_tokens_gemini(): +def test_reasoning_tokens_gemini(_local_model_cost_map): model = "gemini-2.5-flash" custom_llm_provider = "gemini" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=1578, @@ -132,12 +128,10 @@ def test_reasoning_tokens_gemini(): ) -def test_reasoning_tokens_gemini_3_1_flash_lite(): +def test_reasoning_tokens_gemini_3_1_flash_lite(_local_model_cost_map): """Test cost calculation for gemini-3.1-flash-lite-preview with reasoning tokens""" model = "gemini-3.1-flash-lite-preview" custom_llm_provider = "gemini" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=1000, @@ -270,11 +264,9 @@ def test_image_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) -def test_video_output_tokens_gemini_omni_flash_preview(): +def test_video_output_tokens_gemini_omni_flash_preview(_local_model_cost_map): """Video output tokens are billed at output_cost_per_video_token, not the text rate and not zero.""" model = "gemini-omni-flash-preview" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") text_tokens = 100 video_tokens = 46336 @@ -310,11 +302,9 @@ def test_video_output_tokens_gemini_omni_flash_preview(): ) -def test_video_input_tokens_gemini_omni_flash_preview(): +def test_video_input_tokens_gemini_omni_flash_preview(_local_model_cost_map): """Video input tokens are billed at the standard input rate instead of being dropped.""" model = "gemini-omni-flash-preview" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( completion_tokens=10, @@ -369,12 +359,10 @@ def test_video_tokens_fallback_to_base_cost(): assert round(completion_cost, 12) == round((600 + 1120) * 2e-6, 12) -def test_generic_cost_per_token_above_200k_tokens(): +def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" custom_llm_provider = "vertex_ai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 220 * 1e6 @@ -420,12 +408,10 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): assert prompt_base_cost == 9e-6 -def test_generic_cost_per_token_gpt54_above_272k_tokens(): +def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 273000 # Above 272K threshold @@ -450,12 +436,10 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(): assert round(completion_cost, 10) == round(expected_completion, 10) -def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): +def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_map): """MiniMax-M3: prompts >512K input tokens priced at 2x input, output, and cache read.""" model = "minimax/MiniMax-M3" custom_llm_provider = "minimax" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] prompt_tokens = 600000 @@ -493,10 +477,8 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): "bedrock_mantle/openai.gpt-5.6-luna", ], ) -def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model): +def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(_local_model_cost_map, model): """Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] assert model_cost_map["max_input_tokens"] == 1000000 @@ -827,12 +809,10 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_gpt55(): +def test_generic_cost_per_token_gpt55(_local_model_cost_map): """gpt-5.5: base pricing — $5/1M input, $30/1M output, $0.50/1M cached input.""" model = "gpt-5.5" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -867,12 +847,10 @@ def test_generic_cost_per_token_gpt55(): ) -def test_generic_cost_per_token_gpt55_pro(): +def test_generic_cost_per_token_gpt55_pro(_local_model_cost_map): """gpt-5.5-pro: responses-only model — $30/1M input, $180/1M output, $3/1M cached input.""" model = "gpt-5.5-pro" custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -913,13 +891,13 @@ def test_generic_cost_per_token_gpt55_pro(): @pytest.mark.parametrize( "model,input_cost,output_cost,cache_read_cost,cache_write_cost", [ - ("gpt-5.6", 5e-6, 3e-5, 5e-7, 6.25e-6), - ("gpt-5.6-sol", 5e-6, 3e-5, 5e-7, 6.25e-6), + ("gpt-5.6", 4e-6, 2e-5, 4e-7, 5e-6), + ("gpt-5.6-sol", 4e-6, 2e-5, 4e-7, 5e-6), ("gpt-5.6-terra", 2e-6, 1.2e-5, 2e-7, 2.5e-6), ("gpt-5.6-luna", 2e-7, 1.2e-6, 2e-8, 2.5e-7), ], ) -def test_generic_cost_per_token_gpt56( +def test_generic_cost_per_token_gpt56(_local_model_cost_map, model, input_cost, output_cost, cache_read_cost, cache_write_cost ): """gpt-5.6 (sol/terra/luna): base pricing + new cache-write cost. @@ -927,8 +905,6 @@ def test_generic_cost_per_token_gpt56( Cache writes are billed at 1.25x the uncached input rate for this family. """ custom_llm_provider = "openai" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] @@ -965,16 +941,31 @@ def test_generic_cost_per_token_gpt56( assert round(completion_cost, 10) == round(output_cost * completion_tokens, 10) +def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map): + """Regression: the bare gpt-5.6 alias routes to GPT-5.6 Sol, so every cost field on + the two entries has to hold the same value. They drifted once before, when Sol took + its promotional cut and gpt-5.6 was left on the pre-cut rates, overbilling callers + who used the alias.""" + alias = litellm.model_cost["gpt-5.6"] + sol = litellm.model_cost["gpt-5.6-sol"] + + cost_fields = sorted(field for field in sol if "cost" in field) + assert len(cost_fields) == 23 + + for field in cost_fields: + assert alias.get(field) == sol.get(field), field + + @pytest.mark.parametrize( "model,flex_long_input_cost,flex_long_output_cost", [ - ("gpt-5.6", 5e-6, 2.25e-5), - ("gpt-5.6-sol", 5e-6, 2.25e-5), + ("gpt-5.6", 4e-6, 1.5e-5), + ("gpt-5.6-sol", 4e-6, 1.5e-5), ("gpt-5.6-terra", 2e-6, 9e-6), ("gpt-5.6-luna", 2e-7, 9e-7), ], ) -def test_generic_cost_per_token_gpt56_flex_above_272k( +def test_generic_cost_per_token_gpt56_flex_above_272k(_local_model_cost_map, model, flex_long_input_cost, flex_long_output_cost ): """A >272K flex request bills the flex long-context rate, not the standard one. @@ -983,8 +974,6 @@ def test_generic_cost_per_token_gpt56_flex_above_272k( ``*_above_272k_tokens_flex`` keys these requests silently fell back to the standard long-context price, billing 2x what OpenAI charges. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") prompt_tokens = 300000 completion_tokens = 1000 @@ -1023,11 +1012,9 @@ def test_generic_cost_per_token_gpt56_flex_above_272k( ("flex", 300000, 2e-6, 2.5e-6, 2e-7), ], ) -def test_generic_cost_per_token_gpt56_terra_cache_costs_by_tier_and_context( +def test_generic_cost_per_token_gpt56_terra_cache_costs_by_tier_and_context(_local_model_cost_map, service_tier, prompt_tokens, input_rate, cache_write_rate, cache_read_rate ): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") cached_tokens = 50000 cache_write_tokens = 40000 @@ -1115,14 +1102,14 @@ def test_generic_cost_per_token_gpt56_cyber( ("azure/eu/gpt-5.6-luna", 2.2e-7, 1.32e-6, 2.2e-8), ], ) -def test_generic_cost_per_token_azure_gpt56( +def test_generic_cost_per_token_azure_gpt56(_local_model_cost_map, model, input_cost, output_cost, cache_read_cost ): - """Azure gpt-5.6 (global + us/eu regional): pricing mirrors the openai - family for global deployments and carries the standard 10% regional uplift. + """Azure gpt-5.6 (global + us/eu regional): Azure prices this family on its own + schedule and carries the standard 10% regional uplift on top. It did not take the + promotional cut OpenAI applied to gpt-5.6-sol, so these rates deliberately sit + above the openai ones and must not be lowered to match them. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_cost_map = litellm.model_cost[model] assert model_cost_map["litellm_provider"] == "azure" @@ -1163,7 +1150,7 @@ def test_generic_cost_per_token_azure_gpt56( ("gpt-5.5-pro-2026-04-23", False, True, False), ], ) -def test_gpt55_reasoning_effort_flags_match_live_openai_api( +def test_gpt55_reasoning_effort_flags_match_live_openai_api(_local_model_cost_map, model, expected_none, expected_xhigh, expected_minimal ): """Pin reasoning_effort capability flags to OpenAI's actual API contract. @@ -1172,8 +1159,6 @@ def test_gpt55_reasoning_effort_flags_match_live_openai_api( ``Unsupported value: 'reasoning_effort' does not support 'minimal' with this model``. gpt-5.5-pro additionally rejects 'none' and 'low'. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert ( @@ -1194,7 +1179,7 @@ def test_gpt55_reasoning_effort_flags_match_live_openai_api( ("gpt-5.5-pro", "gpt-5.5-pro-2026-04-23"), ], ) -def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( +def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities(_local_model_cost_map, base_model, dated_model ): """Dated snapshots must carry the same reasoning_effort capability flags as @@ -1206,8 +1191,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( behavior between ``gpt-5.5`` and ``gpt-5.5-2026-04-23``. Pinning to a dated variant must never lose capabilities relative to the base alias. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") base = litellm.model_cost[base_model] dated = litellm.model_cost[dated_model] @@ -1234,7 +1217,7 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities( ("azure/gpt-5.5-pro-2026-04-23", "responses", 3e-5, 1.8e-4, 3e-6), ], ) -def test_azure_gpt55_entries_present_with_correct_pricing( +def test_azure_gpt55_entries_present_with_correct_pricing(_local_model_cost_map, model, expected_mode, expected_input, expected_output, expected_cache_read ): """Day-0 Azure entries for GPT-5.5 mirror the OpenAI pricing structure. @@ -1243,8 +1226,6 @@ def test_azure_gpt55_entries_present_with_correct_pricing( on 2026-04-24): $5/$30 input/output per 1M for chat, $30/$180 for pro. Cache discount is 10% of input. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert m["litellm_provider"] == "azure" @@ -1269,12 +1250,10 @@ def test_azure_gpt55_entries_present_with_correct_pricing( ("azure/gpt-5.5-pro", False, False, True), ], ) -def test_azure_gpt55_reasoning_effort_flags_match_live_openai_api( +def test_azure_gpt55_reasoning_effort_flags_match_live_openai_api(_local_model_cost_map, model, expected_none, expected_minimal, expected_xhigh ): """Azure entries pin reasoning_effort flags to OpenAI's actual API contract.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") m = litellm.model_cost[model] assert m.get("supports_none_reasoning_effort") is expected_none @@ -1654,11 +1633,9 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): assert round(result, 6) == round(expected, 6) -def test_service_tier_flex_pricing(): +def test_service_tier_flex_pricing(_local_model_cost_map): """Test that flex service tier uses correct pricing (approximately 50% of standard).""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -1711,11 +1688,9 @@ def test_service_tier_flex_pricing(): ), f"Flex total cost mismatch: {flex_total} vs {expected_flex_total}" -def test_service_tier_default_pricing(): +def test_service_tier_default_pricing(_local_model_cost_map): """Test that when no service tier is provided, standard pricing is used.""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano model = "gpt-5-nano" @@ -1762,11 +1737,9 @@ def test_service_tier_default_pricing(): ), f"Standard completion cost mismatch: {default_cost[1]} vs {expected_standard_completion}" -def test_service_tier_fallback_pricing(): +def test_service_tier_fallback_pricing(_local_model_cost_map): """Test that when service tier is provided but model doesn't have those keys, it falls back to standard pricing.""" # Set up environment for local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-4 which doesn't have flex pricing keys model = "gpt-4" @@ -1874,15 +1847,13 @@ def test_service_tier_ultrafast_pricing(): assert completion_cost == pytest.approx(400 * 3e-04) -def test_service_tier_ultrafast_fallback_pricing(): +def test_service_tier_ultrafast_fallback_pricing(_local_model_cost_map): """Without *_ultrafast keys an ultrafast request bills the standard rate, not zero. Guards the suffix fallback in _get_cost_per_unit: "_fast" is a substring of "_ultrafast", so a shortest-first suffix match would strip the wrong suffix and price the request at 0. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) @@ -1909,9 +1880,10 @@ def test_service_tier_ultrafast_fallback_pricing(): [ "gemini-3-pro-image-preview", "gemini-3.1-flash-image-preview", + "gemini-3.1-flash-lite-image", ], ) -def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): +def test_gemini_image_generation_cost_with_zero_text_tokens(_local_model_cost_map, model: str): """ Test that image_tokens are correctly costed when text_tokens=0. @@ -1921,8 +1893,6 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): https://github.com/BerriAI/litellm/issues/17410 """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") custom_llm_provider = "vertex_ai" @@ -1977,13 +1947,11 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): ), f"Expected completion cost ${expected_completion_cost:.6f}, got ${completion_cost:.6f}" -def test_vertex_image_generation_cost_prefers_token_usage_metadata(): +def test_vertex_image_generation_cost_prefers_token_usage_metadata(_local_model_cost_map): """ When usage metadata exists on image responses, Vertex image generation cost should be calculated from token pricing, not flat output_cost_per_image. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3.1-flash-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -2022,13 +1990,11 @@ def test_vertex_image_generation_cost_prefers_token_usage_metadata(): assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): +def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_model_cost_map): """ Without usage metadata, Vertex image generation cost should fall back to output_cost_per_image * number_of_images. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3.1-flash-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -2046,13 +2012,11 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): assert round(cost, 10) == round(expected_cost, 10) -def test_gemini_image_generation_cost_prefers_token_usage_metadata(): +def test_gemini_image_generation_cost_prefers_token_usage_metadata(_local_model_cost_map): """ When usage metadata exists on image responses, Gemini image generation cost should be calculated from token pricing, not flat output_cost_per_image. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -2091,13 +2055,11 @@ def test_gemini_image_generation_cost_prefers_token_usage_metadata(): assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(): +def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(_local_model_cost_map): """ Without usage metadata, Gemini image generation cost should fall back to output_cost_per_image * number_of_images. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -2194,7 +2156,7 @@ def test_reasoning_tokens_without_text_tokens_gpt5_nano(): ), "Bug detected: Cost calculation is using only reasoning_tokens instead of all completion_tokens!" -def test_image_count_prevents_text_tokens_fallback(): +def test_image_count_prevents_text_tokens_fallback(_local_model_cost_map): """ Test that the text_tokens fallback in generic_cost_per_token does not override text_tokens=0 when image_count > 0. @@ -2203,8 +2165,6 @@ def test_image_count_prevents_text_tokens_fallback(): When image_count > 0, text_tokens=0 is intentional (image-only request), not "text_tokens not set by provider." """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Simulate Nova image-only embedding: prompt_tokens estimated from # embedding dimensions (768 for 3072-dim), image_count=1 @@ -2238,20 +2198,6 @@ def test_image_count_prevents_text_tokens_fallback(): # --------------------------------------------------------------------------- -@pytest.fixture -def _local_model_cost_map(): - prev_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - prev_model_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - try: - yield - finally: - litellm.model_cost = prev_model_cost - if prev_env is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = prev_env @pytest.mark.parametrize("model", ["gpt-5.4", "gpt-realtime-2.1", "gpt-realtime-2.1-mini"]) @@ -2585,7 +2531,7 @@ def test_threshold_keys_exclude_service_tier_variants(): ("cerebras/qwen-3-32b", "cerebras", 250, 0), ], ) -def test_token_type_cost_breakdown_is_provider_agnostic( +def test_token_type_cost_breakdown_is_provider_agnostic(_local_model_cost_map, model, custom_llm_provider, reasoning_tokens, cached_tokens ): """ @@ -2597,8 +2543,6 @@ def test_token_type_cost_breakdown_is_provider_agnostic( there - not the top-level cache_read_input_tokens attribute the old breakdown code relied on - is what makes Vertex/OpenAI/Azure cache costs show up at all. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=1000, @@ -2629,10 +2573,8 @@ def test_token_type_cost_breakdown_is_provider_agnostic( assert breakdown.cache_read_cost == pytest.approx(cached_tokens * cache_read_rate) -def test_token_type_cost_breakdown_matches_real_gemini_numbers(): +def test_token_type_cost_breakdown_matches_real_gemini_numbers(_local_model_cost_map): """Hard-coded against the exact gemini-2.5-flash response that exposed the gap.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=209, @@ -2655,9 +2597,7 @@ def test_token_type_cost_breakdown_matches_real_gemini_numbers(): assert breakdown.cache_creation_cost == 0.0 -def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(_local_model_cost_map): usage = Usage( prompt_tokens=200_000, @@ -2679,9 +2619,7 @@ def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(): assert breakdown.cache_read_cost == pytest.approx(50_000 * 4e-07) -def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(_local_model_cost_map): usage = Usage( prompt_tokens=199_999, @@ -2703,14 +2641,12 @@ def test_token_type_cost_breakdown_xai_just_below_200k_uses_base_tier_rates(): assert breakdown.cache_read_cost == pytest.approx(50_000 * 2e-07) -def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage(): +def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage(_local_model_cost_map): """ Bedrock/Anthropic report cache tokens as top-level usage fields; the Usage constructor maps them onto prompt_tokens_details, so the breakdown must still pick up both cache-read and cache-creation costs. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "anthropic.claude-3-5-haiku-20241022-v1:0" usage = Usage( @@ -2734,14 +2670,12 @@ def test_token_type_cost_breakdown_includes_cache_creation_from_top_level_usage( ) -def test_token_type_cost_breakdown_reads_cache_write_tokens(): +def test_token_type_cost_breakdown_reads_cache_write_tokens(_local_model_cost_map): """ Some OpenAI-compatible providers (e.g. kimi-k2) report cache-write tokens under `cache_write_tokens` rather than `cache_creation_tokens`. The breakdown must read it the same way the total-cost normalization does, so the two agree. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "anthropic.claude-3-5-haiku-20241022-v1:0" usage = Usage( @@ -2762,7 +2696,7 @@ def test_token_type_cost_breakdown_reads_cache_write_tokens(): ) -def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): +def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_cost_map): """ Regression: OpenAI gpt-5.6 reports cache-write tokens under prompt_tokens_details.cache_write_tokens (not the Anthropic cache_creation_tokens @@ -2770,8 +2704,6 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): input rate. Customer report: cache creation tokens were never counted for the GPT-5.6 series, so cost was undercounted on cache-write requests. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = Usage( @@ -2793,14 +2725,12 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(): assert prompt_cost > 1000 * info["input_cost_per_token"] -def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): +def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(_local_model_cost_map): """ Regression for #34801: when a provider reports text_tokens covering the whole prompt alongside cache-write tokens (and no cache reads), the cache-write tokens must be backed out of the text total instead of being billed twice. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = Usage( @@ -2819,15 +2749,13 @@ def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(): assert prompt_cost == pytest.approx(expected_prompt) -def test_token_type_cost_breakdown_reconciles_with_generic_total(): +def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_cost_map): """ Both-ways check: the reasoning subset must sum with the remaining (text) output cost to exactly the completion total, and the cache-read subset with the remaining input cost to exactly the prompt total, as computed by generic_cost_per_token. A mismatch here would mean the breakdown misrepresents what was actually billed. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-2.5-flash" custom_llm_provider = "vertex_ai" @@ -2860,9 +2788,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(): assert text_input_cost + breakdown.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_zero_without_special_tokens(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) breakdown = get_token_type_cost_breakdown( @@ -2899,7 +2825,7 @@ def test_token_type_cost_breakdown_zero_without_special_tokens(): ), ], ) -def test_token_type_cost_breakdown_openai_responses_api_cache_write_read( +def test_token_type_cost_breakdown_openai_responses_api_cache_write_read(_local_model_cost_map, raw_usage, expect_read, expect_write ): """Regression for #34309: OpenAI Responses API reports cache tokens under @@ -2908,8 +2834,6 @@ def test_token_type_cost_breakdown_openai_responses_api_cache_write_read( cache_read_cost / cache_creation_cost from the transformed usage.""" from litellm.responses.utils import ResponseAPILoggingUtils - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.6" usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) @@ -2950,15 +2874,13 @@ def test_token_type_cost_breakdown_handles_unknown_model_gracefully(): ) -def test_token_type_cost_breakdown_applies_regional_uplift(): +def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map): """ Regional OpenAI hosts (eu./us.) apply a flat uplift to every token cost. The per-type breakdown must apply the same uplift via data_residency so it stays reconciled with the uplifted input_cost/output_cost totals, instead of being logged at the base rate. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "gpt-5.4" custom_llm_provider = "openai" @@ -3006,15 +2928,13 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_applies_vertex_regional_uplift(): +def test_token_type_cost_breakdown_applies_vertex_regional_uplift(_local_model_cost_map): """ Non-global Vertex endpoints apply a flat 1.1x uplift to every token cost. The per-type breakdown must apply the same uplift via vertex_location so it stays reconciled with the uplifted input_cost/output_cost totals, instead of being logged at the global rate. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-haiku-4-5@20251001" custom_llm_provider = "vertex_ai" @@ -3057,7 +2977,7 @@ def test_token_type_cost_breakdown_applies_vertex_regional_uplift(): assert text_input_cost + regional.cache_read_cost == pytest.approx(prompt_cost) -def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(monkeypatch): +def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(_local_model_cost_map, monkeypatch): """ Anthropic's regional (geo) uplift lives in provider_specific_entry and is applied to every token type in the totals, so the per-type breakdown must @@ -3070,7 +2990,6 @@ def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(monkeypatch) ) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-geo-breakdown-model" litellm.register_model( @@ -3191,9 +3110,7 @@ GEMINI_DAY0_LAUNCH_PRICING = [ @pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_DAY0_LAUNCH_PRICING) -def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, output_cost, cache_read_cost): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_gemini_36_flash_and_35_flash_lite_launch_pricing(_local_model_cost_map, model, input_cost, output_cost, cache_read_cost): model_cost_map = litellm.model_cost[model] assert model_cost_map["input_cost_per_token"] == input_cost @@ -3206,9 +3123,7 @@ def test_gemini_36_flash_and_35_flash_lite_launch_pricing(model, input_cost, out assert model_cost_map["max_input_tokens"] == 1048576 -def test_generic_cost_per_token_gemini_36_flash(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_generic_cost_per_token_gemini_36_flash(_local_model_cost_map): usage = Usage( prompt_tokens=1000, @@ -3274,9 +3189,7 @@ def test_gemini_36_flash_batch_introductory_pricing(model, _local_model_cost_map assert model_cost_map["output_cost_per_token_batches"] == 1.875e-06 -def test_generic_cost_per_token_gemini_35_flash_lite(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_generic_cost_per_token_gemini_35_flash_lite(_local_model_cost_map): usage = Usage( prompt_tokens=1000, @@ -3300,8 +3213,8 @@ def test_generic_cost_per_token_gemini_35_flash_lite(): @pytest.mark.parametrize( "service_tier,input_rate,cache_read_rate,cache_write_rate,output_rate", [ - ("flex", 2.5e-6, 2.5e-7, 3.125e-6, 1.5e-5), - ("priority", 1e-5, 1e-6, 1.25e-5, 6e-5), + ("flex", 2e-6, 2e-7, 2.5e-6, 1e-5), + ("priority", 8e-6, 8e-7, 1e-5, 4e-5), ], ) def test_service_tier_cache_creation_rates_for_gpt_5_6( @@ -3314,7 +3227,7 @@ def test_service_tier_cache_creation_rates_for_gpt_5_6( ): """Regression: gpt-5.6 publishes cache_creation_input_token_cost_flex/_priority, so a flex or priority request must bill cache writes at that tier's rate instead of falling - back to the standard 6.25e-6 rate.""" + back to the standard cache-write rate.""" usage = Usage( prompt_tokens=10_000, completion_tokens=500, @@ -3361,8 +3274,8 @@ def test_fast_service_tier_bills_at_the_priority_rate(_local_model_cost_map): model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast" ) - expected_prompt = 800 * 1e-05 + 200 * 1e-06 - expected_completion = 500 * 6e-05 + expected_prompt = 800 * 8e-06 + 200 * 8e-07 + expected_completion = 500 * 4e-05 assert fast == priority assert fast[0] == pytest.approx(expected_prompt, rel=1e-9) @@ -3397,8 +3310,8 @@ def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_m ) assert fast == priority - assert fast[0] == pytest.approx(300_000 * 1e-05, rel=1e-9) - assert fast[1] == pytest.approx(1_000 * 4.5e-05, rel=1e-9) + assert fast[0] == pytest.approx(300_000 * 8e-06, rel=1e-9) + assert fast[1] == pytest.approx(1_000 * 3e-05, rel=1e-9) def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 2f32145580d..9bdded94513 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,5 +1,4 @@ import os -import sys import pytest @@ -10,9 +9,6 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( from litellm.types.llms.openai import FileSearchTool, WebSearchOptions from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path @@ -377,12 +373,12 @@ def test_get_cost_for_vertex_ai_gemini_web_search(model, custom_llm_provider): assert cost == 0.035, f"Expected $0.035 grounding cost, got ${cost}" -def test_azure_assistant_features_integrated_cost_tracking(): +def test_azure_assistant_features_integrated_cost_tracking(monkeypatch): """ Test integrated cost tracking for Azure assistant features. """ # Force use of local model cost map for CI/CD consistency - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "azure/gpt-4o" diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking_dict_safety.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking_dict_safety.py index 4eee6b59d34..61b94139bb8 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking_dict_safety.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking_dict_safety.py @@ -5,12 +5,9 @@ either a ``dict`` or a ``ServerToolUse`` pydantic instance. See https://github.com/BerriAI/litellm/issues/26153. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 293e5de304f..304d732c518 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index f9311497729..44fa8fc8ae0 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1,13 +1,9 @@ import json import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index a10dc46eb42..3a7e06d085a 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2844,7 +2844,7 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control(): assert text_block["cache_control"]["type"] == "ephemeral" -def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(): +def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): """ Tools with cache_control ttl should preserve the ttl in the cachePoint block for Claude 4.5+ models on Bedrock, matching the behavior of system @@ -2867,7 +2867,7 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: tool_with_1h = { @@ -2927,10 +2927,10 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) -def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(): +def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(monkeypatch): """ End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl for Claude 4.5+ models when tools have cache_control with ttl. @@ -2944,7 +2944,7 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: tools = [ @@ -2980,7 +2980,7 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_convert_to_anthropic_tool_result_openai_file_pdf_becomes_document(): diff --git a/tests/test_litellm/litellm_core_utils/specialty_caches/test_dynamic_logging_cache.py b/tests/test_litellm/litellm_core_utils/specialty_caches/test_dynamic_logging_cache.py index f21cd56750b..5ed9dca68fd 100644 --- a/tests/test_litellm/litellm_core_utils/specialty_caches/test_dynamic_logging_cache.py +++ b/tests/test_litellm/litellm_core_utils/specialty_caches/test_dynamic_logging_cache.py @@ -1,14 +1,9 @@ import json -import os -import sys import time from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.litellm_core_utils.specialty_caches.dynamic_logging_cache import ( diff --git a/tests/test_litellm/litellm_core_utils/test_anthropic_dedup_factory.py b/tests/test_litellm/litellm_core_utils/test_anthropic_dedup_factory.py index df1458b0f95..bafca04ad38 100644 --- a/tests/test_litellm/litellm_core_utils/test_anthropic_dedup_factory.py +++ b/tests/test_litellm/litellm_core_utils/test_anthropic_dedup_factory.py @@ -1,8 +1,5 @@ -import sys -import os import pytest -sys.path.insert(0, os.path.abspath(".")) from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt diff --git a/tests/test_litellm/litellm_core_utils/test_bedrock_converse_dedup_factory.py b/tests/test_litellm/litellm_core_utils/test_bedrock_converse_dedup_factory.py index c32917efe87..73fa1a07d63 100644 --- a/tests/test_litellm/litellm_core_utils/test_bedrock_converse_dedup_factory.py +++ b/tests/test_litellm/litellm_core_utils/test_bedrock_converse_dedup_factory.py @@ -1,8 +1,5 @@ -import sys -import os import pytest -sys.path.insert(0, os.path.abspath(".")) from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, diff --git a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py index cc16ad558e4..434daab6ab5 100644 --- a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -20,14 +20,11 @@ removed, so `test_internal_control_fields_never_leak_into_provider_body` proves they stay out of the body even without it. """ -import os -import sys from typing import Any, Dict, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/test_litellm/litellm_core_utils/test_dd_tracing.py b/tests/test_litellm/litellm_core_utils/test_dd_tracing.py index 455ad033afd..b55ade5225d 100644 --- a/tests/test_litellm/litellm_core_utils/test_dd_tracing.py +++ b/tests/test_litellm/litellm_core_utils/test_dd_tracing.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.dd_tracing import ( _should_use_dd_profiler, diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 38f46b26eea..cc0a52247a4 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -1,14 +1,9 @@ -import os -import sys import httpx import pytest import litellm -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index b2a4263fade..882429fd7cd 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -8,12 +8,9 @@ resolution (get_model_info) including the shipped rules in the bundled cost map. """ import logging -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm._logging import verbose_logger diff --git a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py index fc5b39a2fd7..bda7ab4afc6 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py +++ b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py @@ -10,13 +10,10 @@ server's real provider key to an attacker-controlled host on the outbound request. """ -import os -import sys from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.get_llm_provider_logic import ( _endpoint_matches_api_base, diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 94798d77348..8c0e8ee5d02 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -6,11 +6,9 @@ count actual model entries, not reserved meta keys) and the extraction of the import json import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, diff --git a/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py b/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py index 8587ad1ab01..2285cc83cad 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py +++ b/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, diff --git a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py index f0d91224614..8f4799e3e7d 100644 --- a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py @@ -1,14 +1,9 @@ """Test health check helper functions""" -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index 956f86a9292..f9ddc47cc7c 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params, diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 82de634b488..873da28fc34 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6,9 +6,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import time @@ -64,7 +61,7 @@ def test_post_call_serializes_dict_with_datetime(logging_obj): assert "2026-05-11" in serialized -def test_sentry_sample_rate(): +def test_sentry_sample_rate(monkeypatch): existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE") try: # test with default value by removing the environment variable @@ -76,7 +73,7 @@ def test_sentry_sample_rate(): assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "1.0" # test with custom value - os.environ["SENTRY_API_SAMPLE_RATE"] = "0.5" + monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", "0.5") set_callbacks(["sentry"]) # Check if the custom sample rate is set correctly @@ -86,13 +83,13 @@ def test_sentry_sample_rate(): finally: # Restore the original environment variable if existing_sample_rate: - os.environ["SENTRY_API_SAMPLE_RATE"] = existing_sample_rate + monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", existing_sample_rate) else: if "SENTRY_API_SAMPLE_RATE" in os.environ: del os.environ["SENTRY_API_SAMPLE_RATE"] -def test_sentry_environment(): +def test_sentry_environment(monkeypatch): """Test that SENTRY_ENVIRONMENT is properly handled during Sentry initialization""" existing_environment = os.getenv("SENTRY_ENVIRONMENT") existing_dsn = os.getenv("SENTRY_DSN") @@ -115,7 +112,7 @@ def test_sentry_environment(): try: # Set a mock DSN to allow Sentry initialization - os.environ["SENTRY_DSN"] = "https://test@sentry.io/123456" + monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456") # Test with default value (no environment set) if existing_environment: @@ -129,7 +126,7 @@ def test_sentry_environment(): assert call_kwargs["environment"] == "production" # Test with custom environment value - os.environ["SENTRY_ENVIRONMENT"] = "development" + monkeypatch.setenv("SENTRY_ENVIRONMENT", "development") mock_init.reset_mock() set_callbacks(["sentry"]) @@ -139,7 +136,7 @@ def test_sentry_environment(): assert call_kwargs["environment"] == "development" # Test with staging environment - os.environ["SENTRY_ENVIRONMENT"] = "staging" + monkeypatch.setenv("SENTRY_ENVIRONMENT", "staging") mock_init.reset_mock() set_callbacks(["sentry"]) @@ -154,13 +151,13 @@ def test_sentry_environment(): finally: # Restore the original environment variables if existing_environment: - os.environ["SENTRY_ENVIRONMENT"] = existing_environment + monkeypatch.setenv("SENTRY_ENVIRONMENT", existing_environment) else: if "SENTRY_ENVIRONMENT" in os.environ: del os.environ["SENTRY_ENVIRONMENT"] if existing_dsn: - os.environ["SENTRY_DSN"] = existing_dsn + monkeypatch.setenv("SENTRY_DSN", existing_dsn) else: if "SENTRY_DSN" in os.environ: del os.environ["SENTRY_DSN"] diff --git a/tests/test_litellm/litellm_core_utils/test_max_streaming_duration.py b/tests/test_litellm/litellm_core_utils/test_max_streaming_duration.py index 1eb49f4859f..c768be22a9e 100644 --- a/tests/test_litellm/litellm_core_utils/test_max_streaming_duration.py +++ b/tests/test_litellm/litellm_core_utils/test_max_streaming_duration.py @@ -6,14 +6,11 @@ Covers: - BaseResponsesAPIStreamingIterator (responses) sync + async """ -import os -import sys import time from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper diff --git a/tests/test_litellm/litellm_core_utils/test_model_param_helper.py b/tests/test_litellm/litellm_core_utils/test_model_param_helper.py index df01bd636b8..2c45b333817 100644 --- a/tests/test_litellm/litellm_core_utils/test_model_param_helper.py +++ b/tests/test_litellm/litellm_core_utils/test_model_param_helper.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.model_param_helper import ModelParamHelper diff --git a/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py b/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py index 293d6268eba..be7aadd4cfa 100644 --- a/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py +++ b/tests/test_litellm/litellm_core_utils/test_provider_specific_headers.py @@ -112,3 +112,37 @@ class TestProviderSpecificHeaderUtils: provider_specific_header, None ) assert result == {} + + def test_get_provider_specific_headers_scopes_each_entry_independently(self): + """Entries in a list each carry their own provider scope.""" + scoped_headers: list[ProviderSpecificHeader] = [ + { + "custom_llm_provider": "anthropic,bedrock,vertex_ai", + "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}, + }, + { + "custom_llm_provider": "anthropic", + "extra_headers": {"authorization": "Bearer sk-ant-oat01-fake-token"}, + }, + ] + + assert ProviderSpecificHeaderUtils.get_provider_specific_headers( + scoped_headers, "anthropic" + ) == { + "anthropic-beta": "context-1m-2025-08-07", + "authorization": "Bearer sk-ant-oat01-fake-token", + } + assert ProviderSpecificHeaderUtils.get_provider_specific_headers( + scoped_headers, "bedrock" + ) == {"anthropic-beta": "context-1m-2025-08-07"} + assert ( + ProviderSpecificHeaderUtils.get_provider_specific_headers( + scoped_headers, "openai" + ) + == {} + ) + + def test_get_provider_specific_headers_empty_list(self): + """An empty list of scoped entries contributes nothing.""" + result = ProviderSpecificHeaderUtils.get_provider_specific_headers([], "anthropic") + assert result == {} diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_errors.py b/tests/test_litellm/litellm_core_utils/test_realtime_errors.py index 263d1654f65..494d16b0b9b 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_errors.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_errors.py @@ -1,8 +1,5 @@ import json -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.realtime_errors import ( WEBSOCKET_CLOSE_REASON_MAX_BYTES, diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index ccf353b1b6c..61b63e2b917 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -8,7 +6,6 @@ from websockets.exceptions import ConnectionClosed import litellm -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import ( @@ -1326,7 +1323,7 @@ async def test_log_messages_includes_tools_in_model_call_details(): @pytest.mark.asyncio -async def test_realtime_guardrail_blocks_prompt_injection(): +async def test_realtime_guardrail_blocks_prompt_injection(monkeypatch: pytest.MonkeyPatch): """ Test that when a transcription event containing prompt injection arrives from the backend, a registered guardrail blocks it — sending a warning to the client @@ -1350,7 +1347,7 @@ async def test_realtime_guardrail_blocks_prompt_injection(): event_hook=GuardrailEventHooks.realtime_input_transcription, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) # --- client websocket mock --- client_ws = MagicMock() @@ -1405,11 +1402,10 @@ async def test_realtime_guardrail_blocks_prompt_injection(): f"Expected guardrail_violation error type, got: {error_events[0]}" ) - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_guardrail_allows_clean_transcript(): +async def test_realtime_guardrail_allows_clean_transcript(monkeypatch: pytest.MonkeyPatch): """ Test that a clean transcript passes through the guardrail and triggers response.create to the backend. @@ -1430,7 +1426,7 @@ async def test_realtime_guardrail_allows_clean_transcript(): event_hook=GuardrailEventHooks.realtime_input_transcription, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1463,11 +1459,10 @@ async def test_realtime_guardrail_allows_clean_transcript(): response_creates = [e for e in sent_to_backend if e.get("type") == "response.create"] assert len(response_creates) == 1, f"Clean transcript should trigger response.create, got: {sent_to_backend}" - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_text_input_guardrail_blocks_and_returns_error(): +async def test_realtime_text_input_guardrail_blocks_and_returns_error(monkeypatch: pytest.MonkeyPatch): """ Test that when conversation.item.create arrives with text that triggers a guardrail, the proxy blocks it (doesn't forward to backend) and returns an error event directly @@ -1495,7 +1490,7 @@ async def test_realtime_text_input_guardrail_blocks_and_returns_error(): event_hook=GuardrailEventHooks.pre_call, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1558,11 +1553,10 @@ async def test_realtime_text_input_guardrail_blocks_and_returns_error(): ] assert len(original_items) == 0, f"Blocked item should not be forwarded to backend, got: {original_items}" - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_function_call_output_guardrail_blocks_and_returns_error(): +async def test_realtime_function_call_output_guardrail_blocks_and_returns_error(monkeypatch: pytest.MonkeyPatch): """ Test that a client-supplied function_call_output whose content triggers a guardrail is blocked: it is not forwarded to the backend, and an error @@ -1590,7 +1584,7 @@ async def test_realtime_function_call_output_guardrail_blocks_and_returns_error( event_hook=GuardrailEventHooks.pre_call, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1648,11 +1642,10 @@ async def test_realtime_function_call_output_guardrail_blocks_and_returns_error( assert sanitized_item["call_id"] == "call_123" assert "test@example.com" not in sanitized_item["output"] - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_function_call_output_guardrail_allows_clean_output(): +async def test_realtime_function_call_output_guardrail_allows_clean_output(monkeypatch: pytest.MonkeyPatch): """ Test that a clean function_call_output passes through and reaches the backend when guardrails are configured. @@ -1670,7 +1663,7 @@ async def test_realtime_function_call_output_guardrail_allows_clean_output(): event_hook=GuardrailEventHooks.pre_call, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1714,11 +1707,10 @@ async def test_realtime_function_call_output_guardrail_allows_clean_output(): ] assert len(forwarded) == 1, f"Clean function_call_output should be forwarded, got: {forwarded}" - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_text_input_guardrail_uses_pre_call_mode(): +async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pytest.MonkeyPatch): """ Test that _has_realtime_guardrails returns True for a guardrail configured with pre_call mode (not just realtime_input_transcription). @@ -1736,7 +1728,7 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(): event_hook=GuardrailEventHooks.pre_call, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() backend_ws = MagicMock() @@ -1751,11 +1743,10 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(): "pre_call-only guardrail must not disable server_vad auto-response" ) - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_session_created_injects_session_update_for_audio_guardrail(): +async def test_realtime_session_created_injects_session_update_for_audio_guardrail(monkeypatch: pytest.MonkeyPatch): """ Test that when an audio transcription guardrail is configured, a session.created event from the backend triggers a session.update injection (create_response: false) @@ -1775,7 +1766,7 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra event_hook=GuardrailEventHooks.realtime_input_transcription, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1809,11 +1800,12 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra "GA session.update must nest turn_detection under audio.input" ) - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_realtime_session_created_does_not_inject_session_update_for_pre_call_only(): +async def test_realtime_session_created_does_not_inject_session_update_for_pre_call_only( + monkeypatch: pytest.MonkeyPatch, +): """ pre_call-only guardrails must not inject create_response:false on realtime sessions — that breaks server_vad for audio-only voice agents (e.g. Model Armor). @@ -1831,7 +1823,7 @@ async def test_realtime_session_created_does_not_inject_session_update_for_pre_c event_hook=GuardrailEventHooks.pre_call, default_on=True, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1853,11 +1845,10 @@ async def test_realtime_session_created_does_not_inject_session_update_for_pre_c session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] assert len(session_updates) == 0, f"pre_call-only guardrail must not inject session.update, got: {sent_to_backend}" - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(): +async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monkeypatch: pytest.MonkeyPatch): """Model Armor-style pre_call + post_call must not gate audio VAD.""" import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -1867,18 +1858,22 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(): async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): return inputs - litellm.callbacks = [ - ModelArmorStyleGuardrail( - guardrail_name="model_armor_all_pre_call", - event_hook=GuardrailEventHooks.pre_call, - default_on=False, - ), - ModelArmorStyleGuardrail( - guardrail_name="model_armor_all_post_call", - event_hook=GuardrailEventHooks.post_call, - default_on=False, - ), - ] + monkeypatch.setattr( + litellm, + "callbacks", + [ + ModelArmorStyleGuardrail( + guardrail_name="model_armor_all_pre_call", + event_hook=GuardrailEventHooks.pre_call, + default_on=False, + ), + ModelArmorStyleGuardrail( + guardrail_name="model_armor_all_post_call", + event_hook=GuardrailEventHooks.post_call, + default_on=False, + ), + ], + ) client_ws = MagicMock() backend_ws = MagicMock() @@ -1900,11 +1895,10 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(): assert streaming._has_realtime_guardrails() is True assert streaming._has_audio_transcription_guardrails() is False - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_end_session_after_n_fails_closes_connection(): +async def test_end_session_after_n_fails_closes_connection(monkeypatch: pytest.MonkeyPatch): """ Test that end_session_after_n_fails=2 closes the backend websocket after the second guardrail violation in a session. @@ -1923,7 +1917,7 @@ async def test_end_session_after_n_fails_closes_connection(): default_on=True, end_session_after_n_fails=2, ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1948,11 +1942,10 @@ async def test_end_session_after_n_fails_closes_connection(): assert backend_ws.close.called, "Expected backend_ws.close() to be called after 2 violations" assert streaming._violation_count == 2 - litellm.callbacks = [] # cleanup @pytest.mark.asyncio -async def test_on_violation_end_session_closes_on_first_fail(): +async def test_on_violation_end_session_closes_on_first_fail(monkeypatch: pytest.MonkeyPatch): """ Test that on_violation='end_session' closes the session immediately on the first violation, regardless of end_session_after_n_fails. @@ -1971,7 +1964,7 @@ async def test_on_violation_end_session_closes_on_first_fail(): default_on=True, on_violation="end_session", ) - litellm.callbacks = [guardrail] + monkeypatch.setattr(litellm, "callbacks", [guardrail]) client_ws = MagicMock() client_ws.send_text = AsyncMock() @@ -1995,7 +1988,6 @@ async def test_on_violation_end_session_closes_on_first_fail(): assert backend_ws.close.called, "Expected session to close immediately with on_violation=end_session" assert streaming._violation_count == 1 - litellm.callbacks = [] # cleanup @pytest.mark.asyncio @@ -2898,53 +2890,47 @@ def _transcription_guardrail(): ) -def test_setup_folds_in_auto_response_disable_when_transcription_guardrail_active(): +def test_setup_folds_in_auto_response_disable_when_transcription_guardrail_active(monkeypatch: pytest.MonkeyPatch): """Gemini rejects a second setup, so a transcription guardrail's auto-response disable must be folded into the one-and-only setup; otherwise the model auto-responds and the guardrail is bypassed.""" import litellm - litellm.callbacks = [_transcription_guardrail()] - try: - streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock()) - setup = json.dumps( - { - "setup": { - "model": "models/gemini-3.1-flash-live-preview", - "generationConfig": {"responseModalities": ["AUDIO"]}, - "inputAudioTranscription": {}, - } + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock()) + setup = json.dumps( + { + "setup": { + "model": "models/gemini-3.1-flash-live-preview", + "generationConfig": {"responseModalities": ["AUDIO"]}, + "inputAudioTranscription": {}, } - ) - out = json.loads(streaming._maybe_inject_guardrail_auto_response_disable(setup)) - aad = out["setup"]["realtimeInputConfig"]["automaticActivityDetection"] - assert aad["disabled"] is True - finally: - litellm.callbacks = [] + } + ) + out = json.loads(streaming._maybe_inject_guardrail_auto_response_disable(setup)) + aad = out["setup"]["realtimeInputConfig"]["automaticActivityDetection"] + assert aad["disabled"] is True -def test_setup_unchanged_without_transcription_guardrail(): +def test_setup_unchanged_without_transcription_guardrail(monkeypatch: pytest.MonkeyPatch): import litellm - litellm.callbacks = [] + monkeypatch.setattr(litellm, "callbacks", []) streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock()) setup = json.dumps({"setup": {"model": "x", "generationConfig": {"responseModalities": ["AUDIO"]}}}) out = streaming._maybe_inject_guardrail_auto_response_disable(setup) assert json.loads(out) == json.loads(setup) -def test_non_bidi_setup_left_untouched_for_followup_capable_providers(): +def test_non_bidi_setup_left_untouched_for_followup_capable_providers(monkeypatch: pytest.MonkeyPatch): """OpenAI realtime accepts a follow-up session.update, so a non-bidi message (no top-level 'setup' key) must be left untouched even with a guardrail on.""" import litellm - litellm.callbacks = [_transcription_guardrail()] - try: - streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock()) - msg = json.dumps({"type": "session.update", "session": {"instructions": "hi"}}) - assert streaming._maybe_inject_guardrail_auto_response_disable(msg) == msg - finally: - litellm.callbacks = [] + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock()) + msg = json.dumps({"type": "session.update", "session": {"instructions": "hi"}}) + assert streaming._maybe_inject_guardrail_auto_response_disable(msg) == msg @pytest.mark.asyncio diff --git a/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py b/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py index ad24105588d..30385ba758d 100644 --- a/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py +++ b/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py @@ -1,12 +1,7 @@ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index ba8540f81e3..f6b8a93c472 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -2,13 +2,10 @@ Unit tests for SensitiveDataMasker - List Preservation """ -import os -import sys import pytest # Add the parent directory to the system path -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_cursor.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_cursor.py index 453c7490d98..3d9971034ae 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_cursor.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_cursor.py @@ -19,12 +19,9 @@ to 0 when the only update we saw was the cursor, allowing the text-based fallback to estimate from the real completion text. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from litellm.types.utils import ( diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py index 4e28d5ba7d2..75508917a1e 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py @@ -17,12 +17,9 @@ response and assert: raising ``AttributeError``. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm import completion_cost, stream_chunk_builder from litellm.types.utils import ( diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 0f21cce476b..44e77506b3f 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,12 +1,7 @@ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm import ChatCompletionUsageBlock, stream_chunk_builder from litellm.types.utils import GenericStreamingChunk diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index fbdfcac1adc..b5e33a4e421 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1,14 +1,9 @@ import json -import os -import sys import time from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import asyncio import traceback from typing import Optional diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index ee3e7719d52..a2590dbca2d 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1,8 +1,6 @@ #### What this tests #### # This tests litellm.token_counter.token_counter() function import importlib -import os -import sys import time import traceback from unittest.mock import MagicMock @@ -10,9 +8,6 @@ from unittest.mock import MagicMock import pytest import tiktoken -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, patch import litellm diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter_tool.py b/tests/test_litellm/litellm_core_utils/test_token_counter_tool.py index e8836bab2b9..9f8c1070a47 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter_tool.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter_tool.py @@ -1,13 +1,8 @@ #### What this tests #### # This tests litellm.token_counter() function -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path # Use the same token_counter as the main test. from tests.test_litellm.litellm_core_utils.test_token_counter import token_counter diff --git a/tests/test_litellm/litellm_core_utils/test_tool_search_spend_logging.py b/tests/test_litellm/litellm_core_utils/test_tool_search_spend_logging.py index 813b4a5701f..3d6c7e6b8d8 100644 --- a/tests/test_litellm/litellm_core_utils/test_tool_search_spend_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_tool_search_spend_logging.py @@ -26,10 +26,7 @@ These tests exercise the real public entry points (not the private ``_count_content_list`` helper) so the whole chain is covered end to end. """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm import stream_chunk_builder diff --git a/tests/test_litellm/litellm_core_utils/test_url_utils.py b/tests/test_litellm/litellm_core_utils/test_url_utils.py index 751b548adcd..aaaa43a0dc4 100644 --- a/tests/test_litellm/litellm_core_utils/test_url_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_url_utils.py @@ -100,12 +100,12 @@ class TestEncodeUrlPathSegment: @pytest.mark.parametrize("value", ["", ".", "..", None]) def test_rejects_empty_and_dot_segments(self, value): - with pytest.raises(ValueError, match="resource_id (is required|cannot be a dot path segment)"): + with pytest.raises(ValueError, match=r"resource_id (is required|cannot be a dot path segment)"): encode_url_path_segment(value, field_name="resource_id") @pytest.mark.parametrize("value", ["../model", "model/../other", "/model"]) def test_rejects_dot_segments_in_multi_segment_paths(self, value): - with pytest.raises(ValueError, match="model (is required|cannot be a dot path segment)"): + with pytest.raises(ValueError, match=r"model (is required|cannot be a dot path segment)"): encode_url_path_segments(value, field_name="model") diff --git a/tests/test_litellm/litellm_core_utils/test_xai_oauth_routing.py b/tests/test_litellm/litellm_core_utils/test_xai_oauth_routing.py index 03790b220eb..ca25ee80c23 100644 --- a/tests/test_litellm/litellm_core_utils/test_xai_oauth_routing.py +++ b/tests/test_litellm/litellm_core_utils/test_xai_oauth_routing.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm import LlmProviders diff --git a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py b/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py index dd74379a883..8d6c61b890c 100644 --- a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py +++ b/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py @@ -1,9 +1,7 @@ import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" diff --git a/tests/test_litellm/llms/amazon_nova/chat/test_amazon_nova_chat_completion.py b/tests/test_litellm/llms/amazon_nova/chat/test_amazon_nova_chat_completion.py index d7f464e4052..ecdd1b36333 100644 --- a/tests/test_litellm/llms/amazon_nova/chat/test_amazon_nova_chat_completion.py +++ b/tests/test_litellm/llms/amazon_nova/chat/test_amazon_nova_chat_completion.py @@ -1,9 +1,7 @@ import os -import sys import pytest # Ensure the project root is on the import path -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm import completion from litellm.types.utils import ModelResponse, Usage, Choices, Message diff --git a/tests/test_litellm/llms/anthropic/batches/test_handler.py b/tests/test_litellm/llms/anthropic/batches/test_handler.py index 0a472d86257..6fde6350127 100644 --- a/tests/test_litellm/llms/anthropic/batches/test_handler.py +++ b/tests/test_litellm/llms/anthropic/batches/test_handler.py @@ -14,14 +14,11 @@ asyncio.run) is exercised directly, mirroring the dispatch-contract discipline i tests/test_litellm/batches/test_main.py. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler from litellm.types.utils import LiteLLMBatch diff --git a/tests/test_litellm/llms/anthropic/batches/test_transformation.py b/tests/test_litellm/llms/anthropic/batches/test_transformation.py index 1635abcefd8..eacd2c9d03b 100644 --- a/tests/test_litellm/llms/anthropic/batches/test_transformation.py +++ b/tests/test_litellm/llms/anthropic/batches/test_transformation.py @@ -14,15 +14,12 @@ otherwise read process env / secret managers - mocking them keeps the URL/header assertions deterministic without touching production transform logic. """ -import os -import sys import time from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig from litellm.types.utils import LiteLLMBatch, LlmProviders diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index b219dcba491..2b392456763 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,16 +6,11 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json -import os -import sys from typing import Any, Literal, Optional from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../../..") -) # Adds the parent directory to the system path from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.anthropic.chat.guardrail_translation.handler import ( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 43f27cc85f9..4f340ee0f3f 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch import litellm diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index b216e8eef6d..e4dacc308dc 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,12 +1,9 @@ -import os -import sys from typing import Any, cast import pytest import litellm -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.litellm_core_utils.prompt_templates.common_utils import ( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py index 5b7f2a60f68..f48d51dbe1e 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py @@ -66,5 +66,5 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero {"custom_llm_provider": "openai"}, thinking={"type": "enabled", "budget_tokens": 1024}, ) - assert completion_kwargs["model"] == "responses/openai/gpt-5.6-luna" + assert completion_kwargs["model"] == "openai/responses/gpt-5.6-luna" assert completion_kwargs["prompt_cache_key"] == "session-abc" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py index 076d4392f05..5c53a8fc317 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py @@ -1,13 +1,10 @@ """Compaction block SSE events from AnthropicStreamWrapper (compact_20260112 polyfill).""" -import os -import sys from typing import List from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index bd02c61752e..f64ffb6d233 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -21,14 +21,11 @@ into an open ``thinking`` block, crashing Anthropic SDK clients (Claude Code) with "Content block is not a text block". """ -import os -import sys from typing import List, Optional from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py index bd39e420607..29e9279731d 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py @@ -8,14 +8,11 @@ Without the fix, the AnthropicStreamWrapper silently dropped these arguments, causing tool_use blocks to arrive with empty input {}. """ -import os -import sys from typing import List from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py index b9bda07336f..db8aae6702f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py @@ -3,14 +3,11 @@ Tests for AgenticAnthropicStreamingIterator and SSE rebuild helpers. """ import json -import os -import sys from typing import Any, Dict, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 91f5023496a..570ce152714 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,12 +1,10 @@ import json import os -import sys import httpx import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../../..")) from unittest.mock import AsyncMock, MagicMock, patch @@ -217,7 +215,10 @@ async def _async_return(value): def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): """ - Test that litellm.completion is called when a custom LLM provider is given + Test that litellm.completion is called when a custom LLM provider is given. + + Provider resolution now happens exactly once, inside litellm.completion itself + (BerriAI/litellm#37716), so the handler passes the original unresolved model through. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, @@ -241,7 +242,7 @@ def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provide # Verify that the custom provider was passed through call_kwargs = mock_completion.call_args.kwargs assert call_kwargs["custom_llm_provider"] == "my-custom-llm" - assert call_kwargs["model"] == "my-custom-llm/my-custom-model" + assert call_kwargs["model"] == "my-custom-model" assert call_kwargs["api_key"] == "test-api-key" @@ -525,7 +526,7 @@ class TestThinkingSummaryPreservation: finally: litellm.reasoning_auto_summary = original - def test_summary_added_when_env_var_set(self): + def test_summary_added_when_env_var_set(self, monkeypatch): """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is added.""" import litellm from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( @@ -535,7 +536,7 @@ class TestThinkingSummaryPreservation: original = litellm.reasoning_auto_summary try: litellm.reasoning_auto_summary = False - os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") completion_kwargs = { "model": "responses/gpt-5.2", "custom_llm_provider": "openai", @@ -997,3 +998,108 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys and info.get("supports_mid_conversation_system") is not True ] assert missing == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, expected_wire_model, expected_url", + [ + ( + "perplexity/perplexity/kimi-k3", + "perplexity/kimi-k3", + "https://api.perplexity.ai/v1/responses", + ), + ( + "perplexity/perplexity/sonar", + "perplexity/sonar", + "https://api.perplexity.ai/v1/responses", + ), + ("perplexity/sonar", "sonar", "https://api.perplexity.ai/chat/completions"), + ], +) +async def test_messages_strips_provider_prefix_exactly_once( + requested_model, expected_wire_model, expected_url +): + """ + BerriAI/litellm#37716: only the leading provider segment may be stripped on the way upstream. + + A multi-segment id such as perplexity/perplexity/kimi-k3 must reach the provider as + perplexity/kimi-k3, matching what /v1/chat/completions and /v1/responses already send. + + The endpoint is asserted alongside the body because perplexity/perplexity/sonar is a + Responses-only deployment whose bare id perplexity/sonar is an ordinary chat model, so + stripping the prefix must not also move the request onto chat/completions. + + The subject is the outbound request, so the transport is cut at the wire rather than + stubbed with a response body: these ids take different bridges (chat completions + versus the Responses API) and would otherwise need different response shapes. + """ + captured = {} + + async def fake_send(self, request, **kwargs): + captured["body"] = json.loads(request.content) + captured["url"] = str(request.url) + raise httpx.ConnectError("cut at the wire", request=request) + + with ( + patch.object(httpx.AsyncClient, "send", fake_send), + pytest.raises(litellm.exceptions.InternalServerError), + ): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model=requested_model, + api_key="test-api-key", + ) + + assert captured["body"]["model"] == expected_wire_model + assert captured["url"] == expected_url + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, expected_reported_model", + [ + ("perplexity/perplexity/kimi-k3", "perplexity/kimi-k3"), + ("perplexity/sonar", "sonar"), + ], +) +async def test_messages_streaming_reports_provider_local_model(requested_model, expected_reported_model): + """ + BerriAI/litellm#37716: the wire keeps every segment, so ``message_start`` must still + report the id the provider itself knows rather than the caller's prefixed deployment id. + """ + + class _EmptyStream: + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + with patch("litellm.acompletion", new=AsyncMock(return_value=_EmptyStream())): + stream = await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model=requested_model, + api_key="test-api-key", + stream=True, + ) + first_event = await stream.__anext__() + + assert json.loads(first_event.decode().split("data: ", 1)[1])["message"]["model"] == expected_reported_model + + +def test_messages_sync_streaming_reports_provider_local_model(): + """Same guarantee as the async bridge, at the sync call site.""" + with patch("litellm.completion", new=MagicMock(return_value=iter(()))): + stream = litellm.anthropic.messages.create( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="perplexity/perplexity/kimi-k3", + api_key="test-api-key", + stream=True, + ) + first_event = next(iter(stream)) + + assert json.loads(first_event.decode().split("data: ", 1)[1])["message"]["model"] == "perplexity/kimi-k3" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py index eadc0da2f1f..a0d1f9de6ec 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py @@ -12,13 +12,10 @@ The wrapper should properly handle this by: - Properly managing content_block_stop/start events for subsequent content """ -import os -import sys from typing import List import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py index f3cb2956aeb..b6914809263 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py @@ -1,10 +1,7 @@ -import os -import sys from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py index 6d7cd2f88be..137286a18c4 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py @@ -1,9 +1,6 @@ -import os -import sys from typing import List -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py index 07c0012b04d..f478bbb9b50 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py @@ -8,12 +8,10 @@ modes (type="enabled" or type="adaptive"). """ import os -import sys import pytest from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py index f0252e13336..dc2e107928f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py @@ -6,6 +6,8 @@ Regression tests for the /v1/messages request-parse fast paths: while resolving the (static) type hints only once per process. """ +import pytest + import litellm from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( AnthropicMessagesRequestUtils, @@ -88,3 +90,61 @@ def test_drop_params_keeps_speed_for_supporting_model(): litellm.drop_params = original assert result == {"speed": "fast"} + + +def test_drop_params_strips_sampling_params_for_unsupported_model(monkeypatch): + # claude-opus-4-7 has supports_sampling_params: false in the model map; the + # API 400s on these rather than ignoring them. + monkeypatch.setattr(litellm, "drop_params", False) + result = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"temperature": 0.3, "top_p": 0.9, "top_k": 40, "stream": True}, + model="claude-opus-4-7", + drop_params=True, + ) + + assert result == {"stream": True} + + +def test_drop_params_strips_sampling_params_for_provider_prefixed_model(monkeypatch): + # Vertex-routed ids must resolve the same capability flag. + monkeypatch.setattr(litellm, "drop_params", False) + result = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"temperature": 0.3, "top_p": 0.9, "top_k": 40}, + model="vertex_ai/claude-opus-4-7", + drop_params=True, + ) + + assert result == {} + + +def test_sampling_params_kept_for_supporting_model(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + result = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"temperature": 0.3, "top_p": 0.9, "top_k": 40}, + model="claude-sonnet-4-6", + drop_params=True, + ) + + assert result == {"temperature": 0.3, "top_p": 0.9, "top_k": 40} + + +def test_temperature_1_kept_for_unsupported_model(monkeypatch): + # temperature=1 is the one value these models still accept. + monkeypatch.setattr(litellm, "drop_params", False) + result = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"temperature": 1}, + model="claude-opus-4-7", + drop_params=True, + ) + + assert result == {"temperature": 1} + + +def test_sampling_param_raises_clean_400_without_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.utils.UnsupportedParamsError, match="does not support temperature"): + AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params={"temperature": 0.3}, + model="claude-opus-4-7", + drop_params=False, + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py index 3fe1b6b0e38..fe0bcfa4f30 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py @@ -1,11 +1,8 @@ import asyncio -import os -import sys from typing import Any, AsyncIterator, Dict, List import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.caching.caching import Cache, LiteLLMCacheType diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py index 63fed907c3c..bebdbe9f512 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py @@ -1,10 +1,7 @@ -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py index 5c1cd88835f..f33bb3dda8b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py @@ -1,12 +1,9 @@ import asyncio import json -import os -import sys from datetime import datetime import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 7ef3077f9d7..589dc64f9b9 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -1,9 +1,15 @@ +import json import os import sys +from unittest.mock import AsyncMock, patch + +import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +import litellm from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler import ( + LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, ) @@ -43,3 +49,36 @@ def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): ) assert "user" not in responses_kwargs assert "prompt_cache_key" not in responses_kwargs + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model, expected_reported_model", + [ + ("openai/gpt-5.6-luna", "gpt-5.6-luna"), + ("perplexity/perplexity/kimi-k3", "perplexity/kimi-k3"), + ], +) +async def test_streaming_message_start_reports_the_provider_local_model(requested_model, expected_reported_model): + """ + BerriAI/litellm#37716 sends the caller's unresolved id down this bridge so the provider + resolves it once. ``message_start`` is a reporting field rather than a wire value, so it + keeps naming the model as the provider knows it, with only the leading provider segment gone. + """ + + async def empty_stream(): + return + yield + + with patch.object(litellm, "aresponses", AsyncMock(return_value=empty_stream())): + sse = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model=requested_model, + stream=True, + custom_llm_provider=requested_model.split("/")[0], + ) + events = [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in sse] + + message_start = next(e for e in events if e["type"] == "message_start") + assert message_start["message"]["model"] == expected_reported_model diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 03cbfbb8609..964f4b9f68b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -5,13 +5,11 @@ Tests for LiteLLMAnthropicToResponsesAPIAdapter import json import os -import sys from typing import Any, Dict, List from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../../../..")) from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, @@ -845,14 +843,14 @@ class TestTranslateThinkingToReasoning: finally: litellm.reasoning_auto_summary = original - def test_summary_added_when_env_var_set(self): + def test_summary_added_when_env_var_set(self, monkeypatch): """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is included.""" import litellm original = litellm.reasoning_auto_summary try: litellm.reasoning_auto_summary = False - os.environ["LITELLM_REASONING_AUTO_SUMMARY"] = "true" + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") result = _ADAPTER.translate_thinking_to_reasoning( { "type": "enabled", diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 25739a978d0..c27362bf49f 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -554,7 +554,7 @@ class TestProxyOAuthHeaderForwarding: def test_add_provider_specific_headers_forwards_oauth(self): """add_provider_specific_headers_to_request should forward OAuth Authorization - as a ProviderSpecificHeader scoped to Anthropic-compatible providers.""" + as a ProviderSpecificHeader scoped to Anthropic and nothing else.""" from litellm.proxy.litellm_pre_call_utils import ( add_provider_specific_headers_to_request, ) @@ -569,9 +569,7 @@ class TestProxyOAuthHeaderForwarding: assert "provider_specific_header" in data psh = data["provider_specific_header"] - assert "anthropic" in psh["custom_llm_provider"] - assert "bedrock" in psh["custom_llm_provider"] - assert "vertex_ai" in psh["custom_llm_provider"] + assert psh["custom_llm_provider"] == "anthropic" assert psh["extra_headers"]["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" def test_add_provider_specific_headers_ignores_non_oauth(self): @@ -593,7 +591,10 @@ class TestProxyOAuthHeaderForwarding: def test_add_provider_specific_headers_combines_anthropic_and_oauth(self): """When both anthropic-beta and OAuth Authorization are present, both - should be included in the ProviderSpecificHeader.""" + reach Anthropic.""" + from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, + ) from litellm.proxy.litellm_pre_call_utils import ( add_provider_specific_headers_to_request, ) @@ -608,9 +609,12 @@ class TestProxyOAuthHeaderForwarding: add_provider_specific_headers_to_request(data=data, headers=headers) assert "provider_specific_header" in data - psh = data["provider_specific_header"] - assert psh["extra_headers"]["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" - assert psh["extra_headers"]["anthropic-beta"] == "oauth-2025-04-20" + anthropic_headers = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=data["provider_specific_header"], + custom_llm_provider="anthropic", + ) + assert anthropic_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" + assert anthropic_headers["anthropic-beta"] == "oauth-2025-04-20" def test_clean_headers_forwards_x_api_key_when_authenticated_with_litellm_key(self): """clean_headers should forward x-api-key when user authenticated with x-litellm-api-key and forward_llm_provider_auth_headers=True.""" @@ -929,7 +933,7 @@ class TestValidateEnvironmentAuthToken: config = AnthropicModelInfo() with mock_patch.dict("os.environ", {}, clear=True): with pytest.raises( - Exception, match="ANTHROPIC_API_KEY.*ANTHROPIC_AUTH_TOKEN" + Exception, match=r"ANTHROPIC_API_KEY.*ANTHROPIC_AUTH_TOKEN" ): config.validate_environment( headers={}, diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py index 889809140f8..ddac561f337 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_files_and_batches.py b/tests/test_litellm/llms/anthropic/test_anthropic_files_and_batches.py index fecc34694d5..2728ba03ae4 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_files_and_batches.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_files_and_batches.py @@ -8,11 +8,8 @@ Tests for: """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch -sys.path.insert(0, os.path.abspath("../../../../")) import httpx import pytest diff --git a/tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py b/tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py index 97b8ab92a8e..69738118d7a 100644 --- a/tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py +++ b/tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py @@ -3,10 +3,7 @@ Test that Azure AI Anthropic models have cache pricing configured. Verifies the fix for issue #19532. """ -import sys -import os -sys.path.insert(0, os.path.abspath("../../../../../")) import litellm from litellm import get_model_info diff --git a/tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py b/tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py index 70fef0162e6..5c88ae17679 100644 --- a/tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py +++ b/tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py @@ -5,12 +5,9 @@ being either a ``dict`` or a ``ServerToolUse`` pydantic instance. See https://github.com/BerriAI/litellm/issues/26153. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.llms.anthropic.cost_calculation import ( _get_web_search_requests, diff --git a/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py b/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py index bc26268ee92..c925bd7de45 100644 --- a/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py +++ b/tests/test_litellm/llms/apiserpent/test_apiserpent_search.py @@ -239,8 +239,8 @@ class TestAPISerpentSearchIntegration: return mock_response @pytest.mark.asyncio - async def test_asearch_quick_default(self): - os.environ["APISERPENT_API_KEY"] = "test-api-key" + async def test_asearch_quick_default(self, monkeypatch): + monkeypatch.setenv("APISERPENT_API_KEY", "test-api-key") with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", new_callable=AsyncMock, @@ -269,8 +269,8 @@ class TestAPISerpentSearchIntegration: assert response.results[0].title == "Test Result" @pytest.mark.asyncio - async def test_asearch_deep(self): - os.environ["APISERPENT_API_KEY"] = "test-api-key" + async def test_asearch_deep(self, monkeypatch): + monkeypatch.setenv("APISERPENT_API_KEY", "test-api-key") with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", new_callable=AsyncMock, diff --git a/tests/test_litellm/llms/azure/batches/test_handler.py b/tests/test_litellm/llms/azure/batches/test_handler.py index f2332a7de7c..27876405781 100644 --- a/tests/test_litellm/llms/azure/batches/test_handler.py +++ b/tests/test_litellm/llms/azure/batches/test_handler.py @@ -21,13 +21,10 @@ runs for real. from __future__ import annotations import asyncio -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from openai import AsyncOpenAI, OpenAI # noqa: E402 diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py index 31c76c42599..fc7e94a77ba 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py @@ -1,15 +1,10 @@ import json -import os -import sys import traceback from typing import Callable, Optional from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.azure.chat.o_series_transformation import AzureOpenAIO1Config diff --git a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py index 857ed9d22a6..560fee17328 100644 --- a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py +++ b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py @@ -1,15 +1,10 @@ import json -import os -import sys import traceback from typing import Callable, Optional from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.azure.azure import AzureChatCompletion from litellm.llms.azure.image_generation.http_utils import ( diff --git a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py index 529a7453d74..29b74c2ee4a 100644 --- a/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py +++ b/tests/test_litellm/llms/azure/passthrough/test_azure_passthrough_transformation.py @@ -1,11 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock import httpx -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig from litellm.types.utils import ModelResponse diff --git a/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py b/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py index 4638bc4df0f..c14a1cfdda3 100644 --- a/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py @@ -1,14 +1,10 @@ import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path @pytest.mark.asyncio diff --git a/tests/test_litellm/llms/azure/response/test_azure_transformation.py b/tests/test_litellm/llms/azure/response/test_azure_transformation.py index 24ae563fb76..da44394d11d 100644 --- a/tests/test_litellm/llms/azure/response/test_azure_transformation.py +++ b/tests/test_litellm/llms/azure/response/test_azure_transformation.py @@ -1,13 +1,8 @@ -import os -import sys from copy import deepcopy from unittest.mock import patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index 99826c14069..f2c852e9509 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -1,15 +1,11 @@ import json import os -import sys import traceback from typing import Callable, Optional from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token from litellm.secret_managers.get_azure_ad_token_provider import ( @@ -812,7 +808,6 @@ async def test_azure_client_reuse(function_name, is_async, args): """ Test that multiple Azure API calls reuse the same Azure OpenAI client """ - litellm.set_verbose = True # Determine which client class to mock based on whether the test is async client_path = ( diff --git a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py index b172c401e2f..16560c7a1fa 100644 --- a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py +++ b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.exceptions import ContentPolicyViolationError diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index 900372f3e54..0fd9a381a5a 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.azure_ai.azure_model_router.transformation import ( AzureModelRouterConfig, ) diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py index d66798a5725..4b317cff975 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py @@ -4,12 +4,7 @@ Tests for Azure AI Anthropic CountTokens transformation. Verifies that the CountTokens API uses the correct authentication headers. """ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( diff --git a/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py b/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py index da1041f3d60..667552dcf60 100644 --- a/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py +++ b/tests/test_litellm/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.azure_ai.image_edit.transformation import ( AzureFoundryFluxImageEditConfig, diff --git a/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py b/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py index d5256be02d7..b948e46093a 100644 --- a/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py +++ b/tests/test_litellm/llms/azure_ai/image_edit/test_mai_image_edit_transformation.py @@ -1,12 +1,9 @@ import io -import os -import sys from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.llms.azure_ai.image_edit import ( AzureFoundryMAIImageEditConfig, diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py index f7ad333293c..2a44e77ce09 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -1,11 +1,9 @@ import os -import sys from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) import litellm from litellm.llms.azure.azure import AzureChatCompletion @@ -40,8 +38,8 @@ class TestAzureMAIImageGeneration: assert not AzureFoundryMAIImageGenerationConfig.is_mai_model("flux.2-pro") assert not AzureFoundryMAIImageGenerationConfig.is_mai_model("MAI-DS-R1") - def test_mai_flash_and_2e_model_pricing_in_cost_map(self): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + def test_mai_flash_and_2e_model_pricing_in_cost_map(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") flash_info = litellm.get_model_info( @@ -328,8 +326,8 @@ class TestAzureMAIImageGeneration: assert image_response.usage.total_tokens == 1046 assert image_response.size == "1792x1024" - def test_mai_image_cost_calculator_token_based(self): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + def test_mai_image_cost_calculator_token_based(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "azure_ai/MAI-Image-2.5" model_info = litellm.get_model_info(model=model, custom_llm_provider="azure_ai") @@ -360,8 +358,8 @@ class TestAzureMAIImageGeneration: ) assert round(cost, 10) == round(expected_cost, 10) - def test_mai_image_cost_calculator_falls_back_to_flat_image_pricing(self): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + def test_mai_image_cost_calculator_falls_back_to_flat_image_pricing(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "azure_ai/MAI-Image-2.5" model_info = litellm.get_model_info(model=model, custom_llm_provider="azure_ai") diff --git a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py index 602cbf68f3f..ab497d06ca7 100644 --- a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py +++ b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.azure_ai.rerank.transformation import AzureAIRerankConfig diff --git a/tests/test_litellm/llms/base_llm/batches/base_batches_config_test.py b/tests/test_litellm/llms/base_llm/batches/base_batches_config_test.py index fd526c55de4..5195dd8ba44 100644 --- a/tests/test_litellm/llms/base_llm/batches/base_batches_config_test.py +++ b/tests/test_litellm/llms/base_llm/batches/base_batches_config_test.py @@ -19,14 +19,11 @@ transformation is a standalone class with a different shape) cannot use this and keep fully standalone tests. """ -import os -import sys from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.types.utils import LiteLLMBatch, LlmProviders diff --git a/tests/test_litellm/llms/base_llm/batches/test_transformation.py b/tests/test_litellm/llms/base_llm/batches/test_transformation.py index cfb9f278f80..d84c820228f 100644 --- a/tests/test_litellm/llms/base_llm/batches/test_transformation.py +++ b/tests/test_litellm/llms/base_llm/batches/test_transformation.py @@ -18,12 +18,9 @@ filter, dropping the staticmethod/classmethod filter, or widening the prefix filter to all single-underscore names) makes a test fail. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.types.utils import LlmProviders diff --git a/tests/test_litellm/llms/bedrock/batches/test_batch_metadata_sanitization.py b/tests/test_litellm/llms/bedrock/batches/test_batch_metadata_sanitization.py index 8de47331614..a34e4f5d5c9 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_batch_metadata_sanitization.py +++ b/tests/test_litellm/llms/bedrock/batches/test_batch_metadata_sanitization.py @@ -9,12 +9,9 @@ when constructing LiteLLMBatch. This test suite verifies the sanitization layer prevents that. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig diff --git a/tests/test_litellm/llms/bedrock/batches/test_handler.py b/tests/test_litellm/llms/bedrock/batches/test_handler.py index 1436ad2f383..d2dc89a7492 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_handler.py +++ b/tests/test_litellm/llms/bedrock/batches/test_handler.py @@ -8,14 +8,11 @@ the tests don't hit AWS. from __future__ import annotations -import os -import sys from datetime import datetime, timezone from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.bedrock.batches.handler import ( # noqa: E402 BedrockBatchesHandler, diff --git a/tests/test_litellm/llms/bedrock/batches/test_transformation.py b/tests/test_litellm/llms/bedrock/batches/test_transformation.py index 01420eb10df..87f9c506857 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_transformation.py +++ b/tests/test_litellm/llms/bedrock/batches/test_transformation.py @@ -14,14 +14,11 @@ URL/ARN handling, and the error class. AWS auth/sigv4 is the only external seam we mock; everything else runs for real. """ -import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig from litellm.types.utils import LiteLLMBatch, LlmProviders diff --git a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py index e5a2ea9b28f..ed8aab8d3d0 100644 --- a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py @@ -9,13 +9,10 @@ Tests: """ import json -import os -import sys import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) from unittest.mock import MagicMock, Mock, patch diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py index 5f5a6512eac..4db786668b8 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen2_transformation.py @@ -1,14 +1,11 @@ import asyncio import json -import os -import sys from unittest.mock import Mock import pytest # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( AmazonQwen2Config, diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen3_transformation.py index fea210b6c47..e011b1fca2b 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_amazon_qwen3_transformation.py @@ -1,14 +1,11 @@ import asyncio import json -import os -import sys from unittest.mock import Mock import pytest # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( AmazonQwen3Config, diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py index 5fefae7e411..aba51689094 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_base_invoke_transformation.py @@ -1,12 +1,7 @@ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 4c4c0e17a38..cea299280f8 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -1,14 +1,11 @@ import asyncio import json -import os -import sys from unittest.mock import patch import pytest # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 30843e8160b..604f3414775 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1,14 +1,10 @@ import asyncio import json import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch import litellm @@ -678,10 +674,10 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): assert fields["tools"][0]["type"] == "computer_20250124" -def test_parallel_tool_calls_config_kept_for_sonnet_5(): +def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: config = AmazonConverseConfig() @@ -708,7 +704,7 @@ def test_parallel_tool_calls_config_kept_for_sonnet_5(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_parallel_tool_calls_config_dropped_for_ttl_only_model( @@ -3025,7 +3021,7 @@ def test_request_metadata_key_constraints(): long_key = "a" * 257 invalid_metadata = {long_key: "value"} - with pytest.raises(Exception, match="(?i)key length|256 characters"): + with pytest.raises(Exception, match=r"(?i)key length|256 characters"): config.transform_request( model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, @@ -3037,7 +3033,7 @@ def test_request_metadata_key_constraints(): # Test empty key invalid_metadata = {"": "value"} - with pytest.raises(Exception, match="(?i)key length|empty"): + with pytest.raises(Exception, match=r"(?i)key length|empty"): config.transform_request( model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, @@ -3057,7 +3053,7 @@ def test_request_metadata_value_constraints(): long_value = "a" * 257 invalid_metadata = {"key": long_value} - with pytest.raises(Exception, match="(?i)value length|256 characters"): + with pytest.raises(Exception, match=r"(?i)value length|256 characters"): config.transform_request( model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, @@ -3575,7 +3571,7 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): litellm.modify_params = original_modify_params -def test_supports_native_structured_outputs(): +def test_supports_native_structured_outputs(monkeypatch): """Test model detection for native structured outputs support. Support is driven by the ``supports_native_structured_output`` flag in the @@ -3583,7 +3579,7 @@ def test_supports_native_structured_outputs(): """ old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: config = AmazonConverseConfig() @@ -3645,7 +3641,7 @@ def test_supports_native_structured_outputs(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_create_output_config_for_response_format(): @@ -3683,11 +3679,11 @@ def test_create_output_config_for_response_format(): assert parsed_schema == expected -def test_translate_response_format_native_output_config(): +def test_translate_response_format_native_output_config(monkeypatch): """For supported models, _translate_response_format_param should produce outputConfig.""" old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: config = AmazonConverseConfig() @@ -3743,7 +3739,7 @@ def test_translate_response_format_native_output_config(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_translate_response_format_fallback_tool_call(): @@ -3778,11 +3774,11 @@ def test_translate_response_format_fallback_tool_call(): assert result["json_mode"] is True -def test_native_structured_output_no_fake_stream(): +def test_native_structured_output_no_fake_stream(monkeypatch): """When using native structured outputs with streaming, fake_stream should NOT be set.""" old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: config = AmazonConverseConfig() @@ -3828,7 +3824,7 @@ def test_native_structured_output_no_fake_stream(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_transform_request_with_output_config(): @@ -4116,7 +4112,7 @@ def test_add_additional_properties_definitions(): ) -def test_json_object_no_schema_skips_tool_injection(): +def test_json_object_no_schema_skips_tool_injection(monkeypatch): """response_format: {type: json_object} with no schema should NOT inject the synthetic json_tool_call tool. @@ -4126,7 +4122,7 @@ def test_json_object_no_schema_skips_tool_injection(): the model respond naturally with the JSON the caller asked for.""" old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: config = AmazonConverseConfig() @@ -4152,7 +4148,7 @@ def test_json_object_no_schema_skips_tool_injection(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_output_config_applies_additional_properties(): @@ -4805,7 +4801,7 @@ def test_cache_control_injection_tool_config_not_added_without_injection_point() assert all("cachePoint" not in tool for tool in tools) -def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(): +def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(monkeypatch): """ Regression test: cache_control_injection_points with location=tool_config must honor the requested `control.ttl`, mirroring the message/system @@ -4819,7 +4815,7 @@ def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(): """ old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: config = AmazonConverseConfig() @@ -4858,10 +4854,10 @@ def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(): if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) -def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacking_own_pricing(): +def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacking_own_pricing(monkeypatch): """ Regression test: a regional pricing entry that omits `cache_creation_input_token_cost_above_1hr` (e.g. `jp.anthropic.claude-opus-4-7`) @@ -4870,7 +4866,7 @@ def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacki """ old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") try: assert "cache_creation_input_token_cost_above_1hr" not in litellm.model_cost["jp.anthropic.claude-opus-4-7"] @@ -4911,7 +4907,7 @@ def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacki if old_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model(): diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation_nova_2.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation_nova_2.py index bac7aa08a04..58058a2e1d4 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation_nova_2.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation_nova_2.py @@ -8,12 +8,7 @@ Reference: https://docs.aws.amazon.com/nova/latest/nova2-userguide/using-convers """ import pytest -import sys -import os -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import httpx import litellm diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index e8964910c69..e2892a6ccee 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -1,13 +1,8 @@ -import os -import sys from unittest.mock import AsyncMock, MagicMock import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.bedrock.chat.invoke_handler import ( diff --git a/tests/test_litellm/llms/bedrock/chat/test_service_tier.py b/tests/test_litellm/llms/bedrock/chat/test_service_tier.py index a625aae23df..ce9dc4d745e 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_service_tier.py +++ b/tests/test_litellm/llms/bedrock/chat/test_service_tier.py @@ -3,14 +3,9 @@ Tests for Bedrock Converse API serviceTier support. """ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.types.llms.bedrock import ServiceTierBlock diff --git a/tests/test_litellm/llms/bedrock/chat/test_writer_palmyra.py b/tests/test_litellm/llms/bedrock/chat/test_writer_palmyra.py index 9bc6724867f..4acfa3f637f 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_writer_palmyra.py +++ b/tests/test_litellm/llms/bedrock/chat/test_writer_palmyra.py @@ -2,14 +2,9 @@ Tests for Writer Palmyra X5 and X4 models on Bedrock Converse. """ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.common_utils import BedrockModelInfo diff --git a/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py b/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py index 6812f40829a..b357c5ac126 100644 --- a/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py +++ b/tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py @@ -1,11 +1,6 @@ import base64 import json -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.count_tokens.transformation import ( DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS, BedrockCountTokensConfig, diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py index 8b6034d1133..74a55cc1ef2 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.base import HiddenParams @@ -153,7 +148,6 @@ class TestBedrockAsyncInvokeEmbedding: def test_async_invoke_twelvelabs_embedding_with_mock(self): """Test async invoke embedding with mocked HTTP calls.""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/async_invoke/twelvelabs.marengo-embed-2-7-v1:0" @@ -193,7 +187,6 @@ class TestBedrockAsyncInvokeEmbedding: @pytest.mark.asyncio async def test_async_invoke_twelvelabs_embedding_async_with_mock(self): """Test async invoke embedding with async calls.""" - litellm.set_verbose = True client = AsyncHTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/async_invoke/twelvelabs.marengo-embed-2-7-v1:0" diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index 9955851132c..114e473be98 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -1,13 +1,9 @@ import json import os -import sys from unittest.mock import Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -50,7 +46,6 @@ test_image_base64 = "data:image/png,test_image_base64_data" ) def test_bedrock_embedding_with_api_key_bearer_token(model, input_type, embed_response): """Test embedding functionality with bearer token authentication""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" @@ -98,7 +93,6 @@ def test_bedrock_embedding_with_env_variable_bearer_token( model, input_type, embed_response ): """Test embedding functionality with bearer token from environment variable""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "env-bearer-token-12345" @@ -130,7 +124,6 @@ def test_bedrock_embedding_with_env_variable_bearer_token( @pytest.mark.asyncio async def test_async_bedrock_embedding_with_bearer_token(): """Test async embedding functionality with bearer token authentication""" - litellm.set_verbose = True client = AsyncHTTPHandler() test_api_key = "async-bearer-token-12345" model = "bedrock/amazon.titan-embed-text-v1" @@ -160,7 +153,6 @@ async def test_async_bedrock_embedding_with_bearer_token(): def test_bedrock_embedding_with_sigv4(): """Test embedding falls back to SigV4 auth when no bearer token is provided""" - litellm.set_verbose = True model = "bedrock/amazon.titan-embed-text-v1" with patch( @@ -182,7 +174,6 @@ def test_bedrock_embedding_with_sigv4(): def test_bedrock_titan_v2_encoding_format_float(): """Test amazon.titan-embed-text-v2:0 with encoding_format=float parameter""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/amazon.titan-embed-text-v2:0" @@ -220,7 +211,6 @@ def test_bedrock_titan_v2_encoding_format_float(): def test_bedrock_titan_v2_encoding_format_base64(): """Test amazon.titan-embed-text-v2:0 with encoding_format=base64 parameter (maps to binary)""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/amazon.titan-embed-text-v2:0" @@ -260,7 +250,6 @@ def test_bedrock_titan_v2_encoding_format_base64(): def test_twelvelabs_input_type_parameter_mapping(): """Test that input_type parameter is correctly mapped to inputType for TwelveLabs models""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/twelvelabs.marengo-embed-2-7-v1:0" @@ -300,7 +289,6 @@ def test_twelvelabs_input_type_parameter_mapping(): def test_twelvelabs_input_type_parameter_mapping_async_invoke(): """Test that input_type parameter is correctly mapped to inputType for TwelveLabs async invoke models""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/async_invoke/twelvelabs.marengo-embed-2-7-v1:0" @@ -343,7 +331,6 @@ def test_twelvelabs_input_type_parameter_mapping_async_invoke(): def test_twelvelabs_missing_input_type_error(): """Test that missing input_type parameter defaults to 'text' for TwelveLabs models""" - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" @@ -422,7 +409,6 @@ def test_bedrock_embedding_header_forwarding(model, embed_response): Relevant Issue: https://github.com/BerriAI/litellm/pull/16042 """ - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" @@ -489,7 +475,6 @@ def test_bedrock_embedding_extra_headers_and_headers_merge(): This ensures that headers from kwargs (forwarded by proxy) and extra_headers (passed explicitly) are both included in the final headers sent to the provider. """ - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/amazon.titan-embed-text-v1" @@ -557,7 +542,6 @@ def test_bedrock_cohere_v4_embedding_response_parsing(): Test parsing of Bedrock Cohere v4 embedding response which returns a dictionary of embeddings keyed by type (e.g. 'float', 'int8') instead of a direct list. """ - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/cohere.embed-v4:0" @@ -617,7 +601,6 @@ def test_bedrock_embedding_custom_headers_with_iam_role_and_custom_api_base(): Relevant Issue: Custom headers not forwarded with IAM roles + custom api_base """ - litellm.set_verbose = True client = HTTPHandler() # Simulate IAM role credentials with session token @@ -734,7 +717,6 @@ async def test_bedrock_embedding_custom_headers_with_iam_role_and_custom_api_bas This is the async version of the test above, verifying the fix works for both sync and async embedding calls. """ - litellm.set_verbose = True client = AsyncHTTPHandler() # Simulate IAM role credentials with session token @@ -977,7 +959,6 @@ def test_bedrock_cohere_embedding_types_wrapped_as_list( Malformed input request: #/embedding_types: expected type: JSONArray, found: String when `encoding_format` is passed as a string. """ - litellm.set_verbose = True client = HTTPHandler() model = "bedrock/cohere.embed-multilingual-v3" diff --git a/tests/test_litellm/llms/bedrock/embed/test_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_embedding.py index 261448842f4..a6cf54a7870 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_embedding.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import patch import pytest diff --git a/tests/test_litellm/llms/bedrock/image/test_amazon_stability3_transformation.py b/tests/test_litellm/llms/bedrock/image/test_amazon_stability3_transformation.py index a758202d74f..dbde8565e13 100644 --- a/tests/test_litellm/llms/bedrock/image/test_amazon_stability3_transformation.py +++ b/tests/test_litellm/llms/bedrock/image/test_amazon_stability3_transformation.py @@ -1,13 +1,8 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch from litellm.llms.bedrock.image_generation.amazon_stability3_transformation import ( diff --git a/tests/test_litellm/llms/bedrock/image/test_bedrock_image_bearer_token.py b/tests/test_litellm/llms/bedrock/image/test_bedrock_image_bearer_token.py index 41ac030ff07..7c36b2aa75f 100644 --- a/tests/test_litellm/llms/bedrock/image/test_bedrock_image_bearer_token.py +++ b/tests/test_litellm/llms/bedrock/image/test_bedrock_image_bearer_token.py @@ -1,12 +1,8 @@ import json import os -import sys from unittest.mock import Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler @@ -18,7 +14,6 @@ mock_image_response = {"images": ["base64_encoded_image_data"], "error": None} class TestBedrockImageGeneration: def test_image_generation_with_api_key_bearer_token(self): """Test image generation with bearer token authentication""" - litellm.set_verbose = True test_api_key = "test-bearer-token-12345" model = "bedrock/stability.sd3-large-v1:0" prompt = "A cute baby sea otter" @@ -53,7 +48,6 @@ class TestBedrockImageGeneration: def test_image_generation_with_env_variable_bearer_token(self, monkeypatch): """Test image generation with bearer token from environment variable""" - litellm.set_verbose = True test_api_key = "env-bearer-token-12345" model = "bedrock/stability.sd3-large-v1:0" prompt = "A cute baby sea otter" @@ -90,7 +84,6 @@ class TestBedrockImageGeneration: @pytest.mark.asyncio async def test_async_image_generation_with_bearer_token(self): """Test async image generation with bearer token authentication""" - litellm.set_verbose = True test_api_key = "async-bearer-token-12345" model = "bedrock/stability.sd3-large-v1:0" prompt = "A cute baby sea otter" @@ -125,7 +118,6 @@ class TestBedrockImageGeneration: def test_image_generation_with_sigv4(self): """Test image generation falls back to SigV4 auth when no bearer token is provided""" - litellm.set_verbose = True model = "bedrock/stability.sd3-large-v1:0" prompt = "A cute baby sea otter" diff --git a/tests/test_litellm/llms/bedrock/invoke_agent/test_bedrock_agent_transformation.py b/tests/test_litellm/llms/bedrock/invoke_agent/test_bedrock_agent_transformation.py index 9e526e47784..3eb85449985 100644 --- a/tests/test_litellm/llms/bedrock/invoke_agent/test_bedrock_agent_transformation.py +++ b/tests/test_litellm/llms/bedrock/invoke_agent/test_bedrock_agent_transformation.py @@ -1,13 +1,8 @@ import base64 -import os -import sys from unittest.mock import patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.chat.invoke_agent.transformation import ( AmazonInvokeAgentConfig, diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 604388ce91a..d3c28302bf9 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -2,7 +2,6 @@ import asyncio import copy import json import os -import sys from datetime import datetime from types import SimpleNamespace from unittest.mock import Mock @@ -11,7 +10,6 @@ import pytest # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.bedrock.common_utils import ( diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 1c90b7c8c87..b005d77ac8b 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,10 +1,5 @@ -import os -import sys from unittest.mock import patch -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index ffe21b91ab2..9efcee192b1 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,5 +1,4 @@ import json -import os import sys import types from types import SimpleNamespace @@ -7,7 +6,6 @@ from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.handler import BedrockRealtime diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index aa002b6e302..ae6b1febd6b 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -1,11 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path import base64 diff --git a/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py b/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py index 17443ca899e..b2a2046b131 100644 --- a/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py +++ b/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py @@ -6,15 +6,10 @@ forward_client_headers_to_llm_api were not being passed to Bedrock rerank provid """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.bedrock.base_aws_llm import Boto3CredentialsInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -66,7 +61,6 @@ def test_bedrock_rerank_header_forwarding_sync(model): This test verifies the fix for the issue where headers configured via forward_client_headers_to_llm_api were not being passed to Bedrock rerank provider. """ - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" @@ -160,7 +154,6 @@ async def test_bedrock_rerank_header_forwarding_async(model): This test verifies the fix for the issue where headers configured via forward_client_headers_to_llm_api were not being passed to Bedrock rerank provider. """ - litellm.set_verbose = True client = AsyncHTTPHandler() test_api_key = "test-bearer-token-12345" @@ -332,7 +325,6 @@ def test_bedrock_rerank_extra_headers_and_headers_merge(): This ensures that headers from kwargs (forwarded by proxy) and extra_headers (passed explicitly) are both included in the final headers sent to the provider. """ - litellm.set_verbose = True client = HTTPHandler() test_api_key = "test-bearer-token-12345" model = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" diff --git a/tests/test_litellm/llms/bedrock/rerank/transformation.py b/tests/test_litellm/llms/bedrock/rerank/transformation.py index 870a7cb1f1e..b45042d1f6a 100644 --- a/tests/test_litellm/llms/bedrock/rerank/transformation.py +++ b/tests/test_litellm/llms/bedrock/rerank/transformation.py @@ -1,13 +1,8 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch from litellm import rerank diff --git a/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py b/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py index 950336c7ad0..20bf65ee385 100644 --- a/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py +++ b/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py @@ -60,9 +60,9 @@ class TestAgentCoreSearch: """ @pytest.mark.asyncio - async def test_agentcore_search_request_payload(self): + async def test_agentcore_search_request_payload(self, monkeypatch): """Validates the MCP tools/call payload and SigV4 signing without real AWS calls.""" - os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", GATEWAY_URL) mock_response = _make_mock_response(_mcp_response_body()) @@ -321,11 +321,11 @@ class TestAgentCoreSearch: assert headers["Authorization"] == "Bearer test-jwt-token" assert signed_body == json.dumps(request_data).encode() - def test_sign_request_uses_bearer_token_from_env(self): + def test_sign_request_uses_bearer_token_from_env(self, monkeypatch): """Server token is attached when the request targets the configured gateway host.""" config = AgentCoreSearchConfig() - os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" - os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + monkeypatch.setenv("AGENTCORE_GATEWAY_TOKEN", "env-jwt-token") + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", GATEWAY_URL) try: headers, _ = config.sign_request( headers={}, @@ -338,11 +338,11 @@ class TestAgentCoreSearch: os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) os.environ.pop("AGENTCORE_GATEWAY_URL", None) - def test_sign_request_refuses_server_token_to_untrusted_host(self): + def test_sign_request_refuses_server_token_to_untrusted_host(self, monkeypatch): """Server-managed token must not be sent to a caller-chosen api_base.""" config = AgentCoreSearchConfig() - os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" - os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + monkeypatch.setenv("AGENTCORE_GATEWAY_TOKEN", "env-jwt-token") + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", GATEWAY_URL) try: with pytest.raises(ValueError, match="Refusing to send"): config.sign_request( @@ -355,11 +355,11 @@ class TestAgentCoreSearch: os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) os.environ.pop("AGENTCORE_GATEWAY_URL", None) - def test_sign_request_uses_env_token_for_gateway_api_base_without_gateway_url(self): + def test_sign_request_uses_env_token_for_gateway_api_base_without_gateway_url(self, monkeypatch): """api_base pointing at a real gateway is a trusted destination for the env token, so operators configuring api_base in yaml don't also need AGENTCORE_GATEWAY_URL.""" config = AgentCoreSearchConfig() - os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" + monkeypatch.setenv("AGENTCORE_GATEWAY_TOKEN", "env-jwt-token") os.environ.pop("AGENTCORE_GATEWAY_URL", None) try: headers, _ = config.sign_request( @@ -380,12 +380,12 @@ class TestAgentCoreSearch: "https://attacker.example.com/gw.gateway.bedrock-agentcore.us-east-1.amazonaws.com/mcp", ], ) - def test_sign_request_refuses_sigv4_to_untrusted_host(self, untrusted_api_base): + def test_sign_request_refuses_sigv4_to_untrusted_host(self, untrusted_api_base, monkeypatch): """A SigV4 signature carries the proxy's credential scope and session token, so it must never be sent to a host that is not the operator's gateway.""" config = AgentCoreSearchConfig() os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) - os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", GATEWAY_URL) try: with patch.object( AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM @@ -410,12 +410,12 @@ class TestAgentCoreSearch: "http://internal-gateway.corp/mcp", ], ) - def test_sign_request_refuses_server_token_over_plaintext_http(self, plaintext_api_base): + def test_sign_request_refuses_server_token_over_plaintext_http(self, plaintext_api_base, monkeypatch): """A trusted hostname over plain http would expose the bearer token to network observers, so credentials only ride https (or localhost).""" config = AgentCoreSearchConfig() - os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" - os.environ["AGENTCORE_GATEWAY_URL"] = plaintext_api_base + monkeypatch.setenv("AGENTCORE_GATEWAY_TOKEN", "env-jwt-token") + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", plaintext_api_base) try: with pytest.raises(ValueError, match="plaintext"): config.sign_request( @@ -446,11 +446,11 @@ class TestAgentCoreSearch: ) mock_base_sign.assert_not_called() - def test_sign_request_allows_plain_http_for_localhost(self): + def test_sign_request_allows_plain_http_for_localhost(self, monkeypatch): """Local development against an MCP stub on 127.0.0.1 keeps working.""" config = AgentCoreSearchConfig() - os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" - os.environ["AGENTCORE_GATEWAY_URL"] = "http://127.0.0.1:8931/mcp" + monkeypatch.setenv("AGENTCORE_GATEWAY_TOKEN", "env-jwt-token") + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", "http://127.0.0.1:8931/mcp") try: headers, _ = config.sign_request( headers={}, @@ -483,11 +483,11 @@ class TestAgentCoreSearch: # AWS_BEARER_TOKEN_BEDROCK env fallback. assert mock_base_sign.call_args.kwargs["api_key"] == "" - def test_sign_request_custom_hostname_requires_region(self): + def test_sign_request_custom_hostname_requires_region(self, monkeypatch): """Custom hostname + empty AWS config chain → clear error, no guessed region.""" config = AgentCoreSearchConfig() custom_url = "https://gateway.internal.example.com/mcp" - os.environ["AGENTCORE_GATEWAY_URL"] = custom_url + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", custom_url) mock_session = MagicMock() mock_session.region_name = None # nothing configured anywhere @@ -503,11 +503,11 @@ class TestAgentCoreSearch: finally: os.environ.pop("AGENTCORE_GATEWAY_URL", None) - def test_sign_request_custom_hostname_uses_shared_config_region(self): + def test_sign_request_custom_hostname_uses_shared_config_region(self, monkeypatch): """Custom hostname + region from AWS shared config (profile) must be honored.""" config = AgentCoreSearchConfig() custom_url = "https://gateway.internal.example.com/mcp" - os.environ["AGENTCORE_GATEWAY_URL"] = custom_url + monkeypatch.setenv("AGENTCORE_GATEWAY_URL", custom_url) mock_session = MagicMock() mock_session.region_name = "eu-west-1" # e.g. from ~/.aws/config profile diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index b9f8283b78e..50e2b53c2b3 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -1,15 +1,11 @@ import json import os -import sys import threading import time import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from datetime import datetime, timedelta, timezone diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index 83f3d73015d..389bf4a8e40 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.llms.bedrock.common_utils import BedrockModelInfo diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py b/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py index daedbe5052c..75e9a8afcb6 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py @@ -10,13 +10,11 @@ being applied to boto3 clients, causing "certificate verify failed" errors. """ import os -import sys import tempfile from unittest.mock import MagicMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -40,12 +38,12 @@ class TestBedrockSSLVerify: ssl_verify = base_aws._get_ssl_verify() assert ssl_verify is True - def test_base_aws_llm_get_ssl_verify_false(self): + def test_base_aws_llm_get_ssl_verify_false(self, monkeypatch): """Test that _get_ssl_verify returns False when SSL verification is disabled.""" base_aws = BaseAWSLLM() # Set SSL_VERIFY to False via environment - os.environ["SSL_VERIFY"] = "False" + monkeypatch.setenv("SSL_VERIFY", "False") ssl_verify = base_aws._get_ssl_verify() assert ssl_verify is False @@ -53,7 +51,7 @@ class TestBedrockSSLVerify: # Clean up os.environ.pop("SSL_VERIFY", None) - def test_base_aws_llm_get_ssl_verify_custom_ca_bundle(self): + def test_base_aws_llm_get_ssl_verify_custom_ca_bundle(self, monkeypatch): """Test that _get_ssl_verify returns custom CA bundle path when SSL_CERT_FILE is set.""" base_aws = BaseAWSLLM() @@ -66,7 +64,7 @@ class TestBedrockSSLVerify: try: # Set SSL_CERT_FILE environment variable - os.environ["SSL_CERT_FILE"] = ca_bundle_path + monkeypatch.setenv("SSL_CERT_FILE", ca_bundle_path) os.environ.pop("SSL_VERIFY", None) litellm.ssl_verify = True @@ -327,7 +325,7 @@ class TestBedrockSSLVerify: os.environ.pop("SSL_CERT_FILE", None) os.unlink(ca_bundle_path) - def test_ssl_verify_priority_env_over_litellm_config(self): + def test_ssl_verify_priority_env_over_litellm_config(self, monkeypatch): """Test that SSL_VERIFY environment variable takes priority over litellm.ssl_verify.""" base_aws = BaseAWSLLM() @@ -335,7 +333,7 @@ class TestBedrockSSLVerify: litellm.ssl_verify = True # Set SSL_VERIFY environment variable to False - os.environ["SSL_VERIFY"] = "False" + monkeypatch.setenv("SSL_VERIFY", "False") try: ssl_verify = base_aws._get_ssl_verify() @@ -345,7 +343,7 @@ class TestBedrockSSLVerify: os.environ.pop("SSL_VERIFY", None) litellm.ssl_verify = True - def test_ssl_cert_file_priority_over_default(self): + def test_ssl_cert_file_priority_over_default(self, monkeypatch): """Test that SSL_CERT_FILE takes priority when ssl_verify is True.""" base_aws = BaseAWSLLM() @@ -358,7 +356,7 @@ class TestBedrockSSLVerify: try: # Set SSL_CERT_FILE environment variable - os.environ["SSL_CERT_FILE"] = ca_bundle_path + monkeypatch.setenv("SSL_CERT_FILE", ca_bundle_path) os.environ.pop("SSL_VERIFY", None) litellm.ssl_verify = True diff --git a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py index 3a27f3ed002..dbd31c7e81b 100644 --- a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py +++ b/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py @@ -1,13 +1,129 @@ """Test Bedrock cross-region inference profile model mapping""" -import os -import sys +import json +from functools import lru_cache +from pathlib import Path +from typing import NamedTuple -sys.path.insert(0, os.path.abspath("../../../..")) +import pytest + +import litellm +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.utils import _get_model_info_helper from litellm.cost_calculator import completion_cost -from litellm.types.utils import ModelResponse, Usage, Choices, Message +from litellm.types.utils import ( + Choices, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) + + +@pytest.fixture +def local_model_cost_map(monkeypatch): + """Resolve models against this checkout's cost map instead of the network-fetched + ``main`` copy, which lags this branch until merge.""" + original_converse_models = set(litellm.bedrock_converse_models) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + try: + litellm.bedrock_converse_models.update( + key + for key, value in litellm.model_cost.items() + if isinstance(value, dict) + and value.get("litellm_provider") == "bedrock_converse" + ) + yield + finally: + litellm.bedrock_converse_models.clear() + litellm.bedrock_converse_models.update(original_converse_models) + litellm.get_model_info.cache_clear() + + +class GptProfile(NamedTuple): + model_id: str + input_cost: float + input_cost_above_272k: float + cache_write: float + cache_write_above_272k: float + cache_read: float + cache_read_above_272k: float + output_cost: float + output_cost_above_272k: float + + +GPT_5_6_PROFILES = [ + GptProfile( + model_id="us.openai.gpt-5.6-sol", + input_cost=5.5e-06, input_cost_above_272k=1.1e-05, + cache_write=6.875e-06, cache_write_above_272k=1.375e-05, + cache_read=5.5e-07, cache_read_above_272k=1.1e-06, + output_cost=3.3e-05, output_cost_above_272k=4.95e-05, + ), + GptProfile( + model_id="global.openai.gpt-5.6-sol", + input_cost=5e-06, input_cost_above_272k=1e-05, + cache_write=6.25e-06, cache_write_above_272k=1.25e-05, + cache_read=5e-07, cache_read_above_272k=1e-06, + output_cost=3e-05, output_cost_above_272k=4.5e-05, + ), + GptProfile( + model_id="us.openai.gpt-5.6-terra", + input_cost=2.2e-06, input_cost_above_272k=4.4e-06, + cache_write=2.75e-06, cache_write_above_272k=5.5e-06, + cache_read=2.2e-07, cache_read_above_272k=4.4e-07, + output_cost=1.32e-05, output_cost_above_272k=1.98e-05, + ), + GptProfile( + model_id="global.openai.gpt-5.6-terra", + input_cost=2e-06, input_cost_above_272k=4e-06, + cache_write=2.5e-06, cache_write_above_272k=5e-06, + cache_read=2e-07, cache_read_above_272k=4e-07, + output_cost=1.2e-05, output_cost_above_272k=1.8e-05, + ), + GptProfile( + model_id="us.openai.gpt-5.6-luna", + input_cost=2.2e-07, input_cost_above_272k=4.4e-07, + cache_write=2.75e-07, cache_write_above_272k=5.5e-07, + cache_read=2.2e-08, cache_read_above_272k=4.4e-08, + output_cost=1.32e-06, output_cost_above_272k=1.98e-06, + ), + GptProfile( + model_id="global.openai.gpt-5.6-luna", + input_cost=2e-07, input_cost_above_272k=4e-07, + cache_write=2.5e-07, cache_write_above_272k=5e-07, + cache_read=2e-08, cache_read_above_272k=4e-08, + output_cost=1.2e-06, output_cost_above_272k=1.8e-06, + ), +] + + +@lru_cache(maxsize=1) +def _packaged_cost_map(): + """The map litellm actually resolves against, for fields ModelInfoBase drops.""" + path = Path(litellm.__file__).parent / "model_prices_and_context_window_backup.json" + return json.loads(path.read_text()) + + +def _bedrock_response(model, usage): + return ModelResponse( + id="test", + created=1234567890, + model=model, + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="OK", role="assistant"), + ) + ], + usage=usage, + ) def test_bedrock_cross_region_inference_profile_mapping(): @@ -52,3 +168,140 @@ def test_proxy_cost_calculation_scenario(): ) expected_cost = (100 * 8e-07) + (50 * 4e-06) assert cost == expected_cost + + +@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id) +def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_map): + """GPT-5.6 is served by Converse on bedrock-runtime, never by Invoke.""" + assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse" + + +@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id) +def test_bedrock_gpt_5_6_published_rates(profile, local_model_cost_map): + """Geo and Global profiles carry their own published rates, per context tier.""" + model_info = _get_model_info_helper( + model=f"bedrock/{profile.model_id}", custom_llm_provider="bedrock" + ) + + assert model_info["litellm_provider"] == "bedrock_converse" + assert model_info["mode"] == "chat" + assert model_info["max_input_tokens"] == 1000000 + assert model_info["input_cost_per_token"] == profile.input_cost + assert ( + model_info["input_cost_per_token_above_272k_tokens"] + == profile.input_cost_above_272k + ) + assert model_info["output_cost_per_token"] == profile.output_cost + assert ( + model_info["output_cost_per_token_above_272k_tokens"] + == profile.output_cost_above_272k + ) + assert model_info["cache_creation_input_token_cost"] == profile.cache_write + assert ( + model_info["cache_creation_input_token_cost_above_272k_tokens"] + == profile.cache_write_above_272k + ) + assert model_info["cache_read_input_token_cost"] == profile.cache_read + assert ( + model_info["cache_read_input_token_cost_above_272k_tokens"] + == profile.cache_read_above_272k + ) + + +def test_bedrock_gpt_5_6_above_272k_tier_applies_to_cost(local_model_cost_map): + """A prompt over 272K tokens is billed at the long-context rate, not the base rate.""" + response = _bedrock_response( + "bedrock/us.openai.gpt-5.6-sol", + Usage(prompt_tokens=300000, completion_tokens=1000, total_tokens=301000), + ) + + cost = completion_cost( + completion_response=response, + model="bedrock/us.openai.gpt-5.6-sol", + custom_llm_provider="bedrock", + ) + + assert cost == pytest.approx((300000 * 1.1e-05) + (1000 * 4.95e-05), rel=1e-9) + + +def test_bedrock_gpt_5_6_bills_cache_read_tokens(local_model_cost_map): + """Bedrock caches long prefixes implicitly and reports them, so a cache-read turn + must be billed at the cache rate rather than dropped to zero.""" + usage = Usage( + prompt_tokens=15611, + completion_tokens=5, + total_tokens=15616, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=15609), + ) + response = _bedrock_response("bedrock/us.openai.gpt-5.6-sol", usage) + + cost = completion_cost( + completion_response=response, + model="bedrock/us.openai.gpt-5.6-sol", + custom_llm_provider="bedrock", + ) + + expected = (2 * 5.5e-06) + (15609 * 5.5e-07) + (5 * 3.3e-05) + assert cost == pytest.approx(expected, rel=1e-9) + # Without cache_read_input_token_cost the cached prefix bills at zero. + assert cost > (15611 * 5.5e-06) * 0.1 + + +def test_bedrock_gpt_5_6_bills_cache_write_tokens(local_model_cost_map): + """The write side of the same cache cycle is billed at the 30m cache-write rate.""" + usage = Usage( + prompt_tokens=15611, + completion_tokens=5, + total_tokens=15616, + cache_creation_input_tokens=15609, + ) + response = _bedrock_response("bedrock/us.openai.gpt-5.6-sol", usage) + + cost = completion_cost( + completion_response=response, + model="bedrock/us.openai.gpt-5.6-sol", + custom_llm_provider="bedrock", + ) + + expected = (2 * 5.5e-06) + (15609 * 6.875e-06) + (5 * 3.3e-05) + assert cost == pytest.approx(expected, rel=1e-9) + + +@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id) +def test_bedrock_gpt_5_6_advertises_only_converse_supported_features( + profile, local_model_cost_map +): + model_info = _get_model_info_helper( + model=f"bedrock/{profile.model_id}", custom_llm_provider="bedrock" + ) + + assert model_info["supports_function_calling"] is True + assert model_info["supports_tool_choice"] is True + assert model_info["supports_vision"] is True + + # Bedrock rejects an explicit cachePoint block for these models, so the flag that + # offers caller-driven caching stays off even though the cache rates are declared. + assert not model_info.get("supports_prompt_caching") + + # ModelInfoBase drops these two, so they are read from the map litellm resolves. + raw = _packaged_cost_map()[profile.model_id] + assert raw["supported_modalities"] == ["text", "image"] + assert raw["supported_output_modalities"] == ["text"] + # No bedrock_converse entry declares supported_endpoints; these models are reachable + # on chat completions and on the Responses API without it. + assert "supported_endpoints" not in raw + + +@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id) +def test_bedrock_gpt_5_6_offers_tools_but_not_reasoning(profile, local_model_cost_map): + """Converse rejects the Anthropic-shaped thinking block LiteLLM emits for + reasoning_effort, so neither reasoning param may be offered yet, while the tool + params these models do accept must be.""" + supported = AmazonConverseConfig().get_supported_openai_params( + model=f"bedrock/{profile.model_id}" + ) + + assert "tools" in supported + assert "tool_choice" in supported + assert "reasoning_effort" not in supported + assert "thinking" not in supported diff --git a/tests/test_litellm/llms/bedrock/test_request_metadata.py b/tests/test_litellm/llms/bedrock/test_request_metadata.py index ad14db5c85f..5a14bbea9f2 100644 --- a/tests/test_litellm/llms/bedrock/test_request_metadata.py +++ b/tests/test_litellm/llms/bedrock/test_request_metadata.py @@ -1,11 +1,8 @@ import asyncio import json -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -36,13 +33,6 @@ ALL_FIELDS = [ IDENTITY = {"user_api_key_alias": "prod-key", "user_api_key_team_alias": "platform"} -@pytest.fixture(autouse=True) -def reset_setting(): - previous = litellm.bedrock_request_metadata_fields - yield - litellm.bedrock_request_metadata_fields = previous - - def litellm_params(metadata_key, **metadata): return {metadata_key: dict(metadata)} @@ -73,8 +63,8 @@ CONVERSE_DRIVERS = [converse_body, converse_body_async] @pytest.mark.parametrize("setting", [None, []]) -def test_feature_off_by_default_leaves_body_and_headers_untouched(setting): - litellm.bedrock_request_metadata_fields = setting +def test_feature_off_by_default_leaves_body_and_headers_untouched(setting, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", setting) params = litellm_params("metadata", spend_logs_metadata={"team": "x"}, **IDENTITY) assert "requestMetadata" not in converse_body(params) @@ -88,18 +78,18 @@ def test_feature_off_by_default_leaves_body_and_headers_untouched(setting): @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) -def test_resolver_reads_both_metadata_variable_names(metadata_key): +def test_resolver_reads_both_metadata_variable_names(metadata_key, monkeypatch: pytest.MonkeyPatch): """`/v1/chat/completions` populates `metadata`; the LITELLM_METADATA_ROUTES populate `litellm_metadata`. Reading only one silently forwards nothing on the other route.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) params = litellm_params(metadata_key, spend_logs_metadata={"cost_center": "cc-1"}, **IDENTITY) assert converse_body(params)["requestMetadata"] == {**IDENTITY, "cost_center": "cc-1"} @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) -def test_invoke_messages_header_reads_both_metadata_variable_names(metadata_key): - litellm.bedrock_request_metadata_fields = ALL_FIELDS +def test_invoke_messages_header_reads_both_metadata_variable_names(metadata_key, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) params = litellm_params(metadata_key, **IDENTITY) headers, _ = AmazonAnthropicClaudeMessagesConfig().validate_anthropic_messages_environment( @@ -112,10 +102,15 @@ def test_invoke_messages_header_reads_both_metadata_variable_names(metadata_key) @pytest.mark.parametrize("reverse_client_keys", [False, True]) @pytest.mark.parametrize("field_order", [ALL_FIELDS, list(reversed(ALL_FIELDS))]) @pytest.mark.parametrize("client_source", ["spend_logs_metadata", "requestMetadata"]) -def test_identity_survives_a_caller_filling_every_slot(reverse_client_keys, field_order, client_source): +def test_identity_survives_a_caller_filling_every_slot( + reverse_client_keys, + field_order, + client_source, + monkeypatch: pytest.MonkeyPatch, +): """A caller sending 16 keys of its own must not evict the identity the feature exists to produce. Driven over every input ordering so the invariant is not an accident of one.""" - litellm.bedrock_request_metadata_fields = field_order + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", field_order) client_keys = [f"client_{index:02d}" for index in range(BEDROCK_REQUEST_METADATA_MAX_PAIRS)] client_pairs = {key: "v" for key in (reversed(client_keys) if reverse_client_keys else client_keys)} if client_source == "spend_logs_metadata": @@ -141,11 +136,14 @@ def test_identity_survives_a_caller_filling_every_slot(reverse_client_keys, fiel ["user_api_key_alias", "user_api_key_team_alias", "spend_logs_metadata", "user_api_key_team_alias"], ], ) -def test_a_field_repeated_in_the_allow_list_does_not_consume_a_client_slot(field_order): +def test_a_field_repeated_in_the_allow_list_does_not_consume_a_client_slot( + field_order, + monkeypatch: pytest.MonkeyPatch, +): """An operator repeating a field in YAML must not inflate the reserved count and shrink the client budget. Asserts the client keys that should have fitted actually reach the wire, since asserting only that identity survives passes with or without the deduplication.""" - litellm.bedrock_request_metadata_fields = field_order + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", field_order) client_keys = [f"client_{index:02d}" for index in range(BEDROCK_REQUEST_METADATA_MAX_PAIRS - 1)] params = litellm_params("metadata", spend_logs_metadata={key: "v" for key in client_keys}, **IDENTITY) @@ -162,11 +160,15 @@ def test_a_field_repeated_in_the_allow_list_does_not_consume_a_client_slot(field "forged_key", ["user_api_key_team_alias", "user_api_key_org_alias", "user_api_key_hash"], ) -def test_caller_cannot_forge_or_shadow_a_reserved_identity_key(forged_key, client_source): +def test_caller_cannot_forge_or_shadow_a_reserved_identity_key( + forged_key, + client_source, + monkeypatch: pytest.MonkeyPatch, +): """`user_api_key_org_alias` and `user_api_key_hash` are names the proxy does not set here, so an exact-key reservation would let the forged value through under a name that reads as proxy-authoritative in the AWS billing record.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) forged = {forged_key: "attacker-controlled"} if client_source == "spend_logs_metadata": params, optional_params = litellm_params("metadata", spend_logs_metadata=forged, **IDENTITY), {} @@ -179,10 +181,10 @@ def test_caller_cannot_forge_or_shadow_a_reserved_identity_key(forged_key, clien assert "attacker-controlled" not in resolved.values() -def test_identity_violating_the_character_class_is_dropped_and_the_request_succeeds(): +def test_identity_violating_the_character_class_is_dropped_and_the_request_succeeds(monkeypatch: pytest.MonkeyPatch): """A team alias with an apostrophe must not turn a working request into a 400 the moment an operator flips the setting on.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) params = litellm_params( "metadata", user_api_key_alias="prod-key", @@ -196,8 +198,8 @@ def test_identity_violating_the_character_class_is_dropped_and_the_request_succe assert body["messages"] -def test_caller_supplied_violation_still_raises_bad_request(): - litellm.bedrock_request_metadata_fields = ALL_FIELDS +def test_caller_supplied_violation_still_raises_bad_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) with pytest.raises(litellm.exceptions.BadRequestError): converse_body( @@ -206,34 +208,34 @@ def test_caller_supplied_violation_still_raises_bad_request(): ) -def test_non_string_and_absent_identity_values_are_dropped(): - litellm.bedrock_request_metadata_fields = ALL_FIELDS + ["user_api_key_spend"] +def test_non_string_and_absent_identity_values_are_dropped(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS + ["user_api_key_spend"]) params = litellm_params("metadata", user_api_key_alias="prod-key", user_api_key_spend=1.25) assert converse_body(params)["requestMetadata"] == {"user_api_key_alias": "prod-key"} -def test_email_is_separately_opt_in(): +def test_email_is_separately_opt_in(monkeypatch: pytest.MonkeyPatch): """PII crossing into CloudTrail only when the operator names the field.""" identity_with_email = {**IDENTITY, "user_api_key_user_email": "owner@example.com"} - litellm.bedrock_request_metadata_fields = ["user_api_key_alias", "user_api_key_team_alias"] + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ["user_api_key_alias", "user_api_key_team_alias"]) assert ( "user_api_key_user_email" not in converse_body(litellm_params("metadata", **identity_with_email))["requestMetadata"] ) - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) assert converse_body(litellm_params("metadata", **identity_with_email))["requestMetadata"] == identity_with_email -def test_resolver_returns_none_when_nothing_survives(): - litellm.bedrock_request_metadata_fields = ALL_FIELDS +def test_resolver_returns_none_when_nothing_survives(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) assert resolve_bedrock_request_metadata(litellm_params=None) is None assert resolve_bedrock_request_metadata(litellm_params={"metadata": {"unrelated": "x"}}) is None -def test_invoke_header_is_json_encoded_and_signed(): - litellm.bedrock_request_metadata_fields = ALL_FIELDS +def test_invoke_header_is_json_encoded_and_signed(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) params = litellm_params("metadata", spend_logs_metadata={"cost_center": "cc-1"}, **IDENTITY) headers = AmazonInvokeConfig().validate_environment( @@ -250,10 +252,10 @@ def test_invoke_header_is_json_encoded_and_signed(): assert "anthropic-version" not in signed -def test_a_caller_supplied_guardrail_header_still_wins(): +def test_a_caller_supplied_guardrail_header_still_wins(monkeypatch: pytest.MonkeyPatch): """The no-displace rule is deliberate for the guardrail headers and must survive the request-metadata header becoming proxy-owned.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) headers = AmazonInvokeConfig().validate_environment( headers={"X-Amzn-Bedrock-GuardrailIdentifier": "caller-set"}, @@ -318,10 +320,10 @@ def metadata_header_values(headers): return [value for name, value in headers.items() if name.lower() == BEDROCK_REQUEST_METADATA_HEADER.lower()] -def test_converse_still_sets_the_bearer_authorization_header(): +def test_converse_still_sets_the_bearer_authorization_header(monkeypatch: pytest.MonkeyPatch): """Converse owns the metadata header now, and that must not disturb the api_key path its validate_environment existed for. Closing the forgery hole cannot break authentication.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) headers = AmazonConverseConfig().validate_environment( headers={}, @@ -341,11 +343,11 @@ def test_converse_still_sets_the_bearer_authorization_header(): "caller_header_name", [BEDROCK_REQUEST_METADATA_HEADER, BEDROCK_REQUEST_METADATA_HEADER.lower(), "x-AMZN-bedrock-Request-METADATA"], ) -def test_a_caller_cannot_forge_the_request_metadata_header(driver, caller_header_name): +def test_a_caller_cannot_forge_the_request_metadata_header(driver, caller_header_name, monkeypatch: pytest.MonkeyPatch): """`extra_headers` puts caller-supplied names into the same dict the proxy merges into, so a deferring merge would sign the caller's forged identity into the AWS billing record. Every spelling must lose, or a second variant is left for the transport to choose between.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) headers = driver({caller_header_name: FORGED}, litellm_params("metadata", **IDENTITY)) @@ -355,11 +357,11 @@ def test_a_caller_cannot_forge_the_request_metadata_header(driver, caller_header @pytest.mark.parametrize("driver", HEADER_DRIVERS) -def test_a_caller_cannot_forge_the_header_when_the_resolver_yields_nothing(driver): +def test_a_caller_cannot_forge_the_header_when_the_resolver_yields_nothing(driver, monkeypatch: pytest.MonkeyPatch): """Forwarding enabled but nothing resolvable, which a caller can arrange by supplying values that all fail Bedrock's rules. Owned-but-empty must mean no header on the wire, never a fallback to the caller's.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) unresolvable = litellm_params("metadata", user_api_key_alias="O'Brien's key", user_api_key_team_alias="x" * 300) headers = driver({BEDROCK_REQUEST_METADATA_HEADER: FORGED}, unresolvable) @@ -373,11 +375,15 @@ def test_a_caller_cannot_forge_the_header_when_the_resolver_yields_nothing(drive "forged_key", ["user_api_key_team_alias", "user_api_key_org_alias", "user_api_key_hash"], ) -def test_a_caller_cannot_keep_reserved_body_keys_when_the_resolver_yields_nothing(forged_key, driver): +def test_a_caller_cannot_keep_reserved_body_keys_when_the_resolver_yields_nothing( + forged_key, + driver, + monkeypatch: pytest.MonkeyPatch, +): """The Converse body has the same fail-open shape as the header: with forwarding on and nothing resolvable, leaving the caller's `requestMetadata` in place would keep their reserved-prefix keys on the wire. Owned-but-empty must remove the field outright.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) body = driver(litellm_params("metadata"), {"requestMetadata": {forged_key: "FORGED"}}) @@ -386,10 +392,10 @@ def test_a_caller_cannot_keep_reserved_body_keys_when_the_resolver_yields_nothin @pytest.mark.parametrize("driver", CONVERSE_DRIVERS) -def test_benign_caller_body_metadata_still_survives_when_no_identity_resolves(driver): +def test_benign_caller_body_metadata_still_survives_when_no_identity_resolves(driver, monkeypatch: pytest.MonkeyPatch): """Removing the field must be scoped to the reserved keys being the only thing left, not a blanket drop of the caller's own attribution pairs.""" - litellm.bedrock_request_metadata_fields = ALL_FIELDS + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ALL_FIELDS) body = driver( litellm_params("metadata"), @@ -400,10 +406,10 @@ def test_benign_caller_body_metadata_still_survives_when_no_identity_resolves(dr @pytest.mark.parametrize("driver", CONVERSE_DRIVERS) -def test_caller_body_metadata_is_left_alone_when_forwarding_is_off(driver): +def test_caller_body_metadata_is_left_alone_when_forwarding_is_off(driver, monkeypatch: pytest.MonkeyPatch): """With the feature off the proxy does not own the field, so the pre-existing pass-through behaviour for a caller-supplied `requestMetadata` must be unchanged.""" - litellm.bedrock_request_metadata_fields = None + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", None) caller_supplied = {"user_api_key_team_alias": "caller-set", "cost_center": "cc-9"} body = driver(litellm_params("metadata", **IDENTITY), {"requestMetadata": caller_supplied}) @@ -412,10 +418,10 @@ def test_caller_body_metadata_is_left_alone_when_forwarding_is_off(driver): @pytest.mark.parametrize("driver", HEADER_DRIVERS) -def test_a_caller_header_is_left_alone_when_forwarding_is_off(driver): +def test_a_caller_header_is_left_alone_when_forwarding_is_off(driver, monkeypatch: pytest.MonkeyPatch): """The proxy only claims the name when the operator turned forwarding on; with the feature off this is an ordinary passthrough header and stripping it would be a regression.""" - litellm.bedrock_request_metadata_fields = None + monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", None) headers = driver({BEDROCK_REQUEST_METADATA_HEADER: FORGED}, litellm_params("metadata", **IDENTITY)) diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 28c8e5c7ed6..9e05d48a18f 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -8,10 +8,7 @@ gate, the URL construction for both paths, and the shared Bearer auth. """ import copy -import os -import sys -sys.path.insert(0, os.path.abspath("../../../../..")) import pytest from botocore.exceptions import ( diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 07910b0b56f..cd775abf136 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -6,11 +6,8 @@ API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.ht """ import json -import os -import sys from unittest.mock import patch -sys.path.insert(0, os.path.abspath("../../../../..")) import httpx import pytest diff --git a/tests/test_litellm/llms/black_forest_labs/image_edit/test_bfl_image_edit_transformation.py b/tests/test_litellm/llms/black_forest_labs/image_edit/test_bfl_image_edit_transformation.py index 17decaf8257..ec243b7058d 100644 --- a/tests/test_litellm/llms/black_forest_labs/image_edit/test_bfl_image_edit_transformation.py +++ b/tests/test_litellm/llms/black_forest_labs/image_edit/test_bfl_image_edit_transformation.py @@ -7,8 +7,6 @@ since polling logic was moved to the handler. import base64 import json -import os -import sys import time from io import BytesIO from typing import Dict, List @@ -17,9 +15,6 @@ from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.black_forest_labs.image_edit.transformation import ( BlackForestLabsImageEditConfig, diff --git a/tests/test_litellm/llms/black_forest_labs/image_generation/test_bfl_image_generation_transformation.py b/tests/test_litellm/llms/black_forest_labs/image_generation/test_bfl_image_generation_transformation.py index 153df5305a7..d6e2c4a3e06 100644 --- a/tests/test_litellm/llms/black_forest_labs/image_generation/test_bfl_image_generation_transformation.py +++ b/tests/test_litellm/llms/black_forest_labs/image_generation/test_bfl_image_generation_transformation.py @@ -6,16 +6,11 @@ since polling logic was moved to the handler. """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.black_forest_labs.image_generation.transformation import ( BlackForestLabsImageGenerationConfig, diff --git a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py index 94b8c51dd52..440304aeac1 100644 --- a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py +++ b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py @@ -1,10 +1,7 @@ -import os -import sys import pytest import json # Adds the parent directory to the system path -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.bytez.chat.transformation import BytezChatConfig, API_BASE, version diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index 6f8a2788c38..ca79c8d7025 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock import httpx @@ -12,9 +10,6 @@ from litellm.llms.bedrock.chat.converse_handler import make_sync_call from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path def test_encode_model_id_with_inference_profile(): diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 90a1c24bada..8e0415d50de 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -5,14 +5,11 @@ Source: litellm/llms/chatgpt/responses/transformation.py """ import json -import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.openai.common_utils import OpenAIError from litellm.types.router import GenericLiteLLMParams diff --git a/tests/test_litellm/llms/cohere/chat/test_cohere_transformation.py b/tests/test_litellm/llms/cohere/chat/test_cohere_transformation.py index c208f4c5489..61334b6ff63 100644 --- a/tests/test_litellm/llms/cohere/chat/test_cohere_transformation.py +++ b/tests/test_litellm/llms/cohere/chat/test_cohere_transformation.py @@ -1,10 +1,5 @@ -import os -import sys from unittest.mock import MagicMock -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.cohere.chat.transformation import CohereChatConfig diff --git a/tests/test_litellm/llms/cohere/embed/test_v1_transformation.py b/tests/test_litellm/llms/cohere/embed/test_v1_transformation.py index 77b500a7e8c..66129b64a2c 100644 --- a/tests/test_litellm/llms/cohere/embed/test_v1_transformation.py +++ b/tests/test_litellm/llms/cohere/embed/test_v1_transformation.py @@ -1,10 +1,5 @@ -import os -import sys from unittest.mock import MagicMock -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig from litellm.types.utils import EmbeddingResponse diff --git a/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py b/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py index 46c37e6af6c..cd3ac57c7e8 100644 --- a/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py +++ b/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py @@ -2,12 +2,9 @@ Unit tests for Cohere Rerank Guardrail Translation Handler """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms import get_guardrail_translation_mapping diff --git a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py b/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py index 0b3348c1b7f..7a69b676667 100644 --- a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py +++ b/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py @@ -5,13 +5,9 @@ Tests the CometAPIChatConfig class methods using mocks """ import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.cometapi.chat.transformation import ( CometAPIChatCompletionStreamingHandler, @@ -187,7 +183,6 @@ def test_cometapi_integration(): Integration test - requires real API key Run with: pytest -k test_cometapi_integration -s """ - import os from litellm import completion # Try to get API key from multiple environment variables @@ -221,7 +216,6 @@ def test_cometapi_streaming_integration(): Integration test for streaming - requires real API key Run with: pytest -k test_cometapi_streaming_integration -s """ - import os from litellm import completion # Try to get API key from multiple environment variables @@ -285,7 +279,6 @@ def test_cometapi_with_custom_base_url(): """ Test CometAPI with custom base URL """ - import os from litellm import completion api_key = ( diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 0a05126919a..34a6d37663b 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -105,14 +105,14 @@ def test_crusoe_provider_detection_by_prefix(): assert model == "meta-llama/Llama-3.3-70B-Instruct" -def test_crusoe_model_list_populated(): +def test_crusoe_model_list_populated(monkeypatch): """Test Crusoe models are present in model_prices_and_context_window.json""" import litellm original_model_cost = litellm.model_cost original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") expected = [ @@ -132,4 +132,4 @@ def test_crusoe_model_list_populated(): if original_env is None: os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", original_env) diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_handler.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_handler.py index 789c88d66f8..763647aa463 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_handler.py @@ -1,13 +1,8 @@ -import os -import sys from unittest.mock import AsyncMock, Mock, patch import aiohttp import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 2dc7fbfd62a..4c92c52d556 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -1,7 +1,5 @@ import asyncio import concurrent.futures -import os -import sys import aiohttp import aiohttp.client_exceptions @@ -9,9 +7,6 @@ import aiohttp.http_exceptions import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.llms.custom_httpx.aiohttp_transport import ( AiohttpResponseStream, diff --git a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py index bd9db87a765..32c555f205a 100644 --- a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py +++ b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py @@ -7,14 +7,11 @@ Covers: - _raise_masked_sync_error and _raise_masked_async_error """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index fa1c7308c6f..f7f89cd1d8d 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -4,7 +4,6 @@ import io import os import pathlib import ssl -import sys import threading import weakref from unittest.mock import MagicMock, patch @@ -14,9 +13,6 @@ import httpx import pytest from aiohttp import ClientSession, TCPConnector -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport from litellm.llms.custom_httpx.http_handler import ( @@ -131,79 +127,62 @@ def test_sync_post_streaming_status_error_should_not_wait_forever_for_body( @pytest.mark.asyncio async def test_ssl_security_level(monkeypatch): # Ensure aiohttp transport is enabled for this test - original_disable = litellm.disable_aiohttp_transport - litellm.disable_aiohttp_transport = False + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) - try: - with patch.dict(os.environ, clear=True): - # Set environment variable for SSL security level - monkeypatch.setenv("SSL_SECURITY_LEVEL", "DEFAULT@SECLEVEL=1") + with patch.dict(os.environ, clear=True): + # Set environment variable for SSL security level + monkeypatch.setenv("SSL_SECURITY_LEVEL", "DEFAULT@SECLEVEL=1") - # Create async client with SSL verification disabled to isolate SSL context testing - client = AsyncHTTPHandler() + # Create async client with SSL verification disabled to isolate SSL context testing + client = AsyncHTTPHandler() - try: - # Get the transport (should be LiteLLMAiohttpTransport) - transport = client.client._transport - assert isinstance(transport, LiteLLMAiohttpTransport) + try: + # Get the transport (should be LiteLLMAiohttpTransport) + transport = client.client._transport + assert isinstance(transport, LiteLLMAiohttpTransport) - # Get the aiohttp ClientSession - client_session = transport._get_valid_client_session() + # Get the aiohttp ClientSession + client_session = transport._get_valid_client_session() - # Get the connector from the session - connector = client_session.connector - assert isinstance(connector, TCPConnector) + # Get the connector from the session + connector = client_session.connector + assert isinstance(connector, TCPConnector) - # Get the SSL context from the connector - ssl_context = connector._ssl + # Get the SSL context from the connector + ssl_context = connector._ssl - # Verify that the SSL context exists and has the correct cipher string - assert isinstance(ssl_context, ssl.SSLContext) - finally: - await client.close() - finally: - # Restore original setting - litellm.disable_aiohttp_transport = original_disable + # Verify that the SSL context exists and has the correct cipher string + assert isinstance(ssl_context, ssl.SSLContext) + finally: + await client.close() @pytest.mark.asyncio -async def test_force_ipv4_transport(): +async def test_force_ipv4_transport(monkeypatch: pytest.MonkeyPatch): """Test transport creation with force_ipv4 enabled""" - original_force_ipv4 = litellm.force_ipv4 - original_disable = litellm.disable_aiohttp_transport - litellm.force_ipv4 = True - litellm.disable_aiohttp_transport = True + monkeypatch.setattr(litellm, "force_ipv4", True) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - try: - transport = AsyncHTTPHandler._create_async_transport() + transport = AsyncHTTPHandler._create_async_transport() - # Should get an AsyncHTTPTransport (no real HTTP call — avoids CI hangs) - assert isinstance(transport, httpx.AsyncHTTPTransport) - finally: - litellm.force_ipv4 = original_force_ipv4 - litellm.disable_aiohttp_transport = original_disable + # Should get an AsyncHTTPTransport (no real HTTP call — avoids CI hangs) + assert isinstance(transport, httpx.AsyncHTTPTransport) @pytest.mark.asyncio -async def test_aiohttp_disabled_transport(): +async def test_aiohttp_disabled_transport(monkeypatch: pytest.MonkeyPatch): """Test transport creation with aiohttp disabled""" - original_disable = litellm.disable_aiohttp_transport - original_force_ipv4 = litellm.force_ipv4 - litellm.disable_aiohttp_transport = True - litellm.force_ipv4 = False + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", False) - try: - transport = AsyncHTTPHandler._create_async_transport() + transport = AsyncHTTPHandler._create_async_transport() - # Should get None when both aiohttp is disabled and force_ipv4 is False - assert transport is None - finally: - litellm.disable_aiohttp_transport = original_disable - litellm.force_ipv4 = original_force_ipv4 + # Should get None when both aiohttp is disabled and force_ipv4 is False + assert transport is None @pytest.mark.asyncio -async def test_ssl_verification_with_aiohttp_transport(): +async def test_ssl_verification_with_aiohttp_transport(monkeypatch: pytest.MonkeyPatch): """ Test aiohttp respects ssl_verify=False @@ -213,38 +192,33 @@ async def test_ssl_verification_with_aiohttp_transport(): import aiohttp # Ensure aiohttp transport is enabled for this test - original_disable = litellm.disable_aiohttp_transport - litellm.disable_aiohttp_transport = False + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + + litellm_async_client = AsyncHTTPHandler(ssl_verify=False) try: - litellm_async_client = AsyncHTTPHandler(ssl_verify=False) + transport = litellm_async_client.client._transport + assert isinstance(transport, LiteLLMAiohttpTransport) + transport_connector = transport._get_valid_client_session().connector + assert isinstance(transport_connector, TCPConnector) + aiohttp_session = aiohttp.ClientSession( + connector=aiohttp.TCPConnector(ssl=False) + ) try: - transport = litellm_async_client.client._transport - assert isinstance(transport, LiteLLMAiohttpTransport) - transport_connector = transport._get_valid_client_session().connector - assert isinstance(transport_connector, TCPConnector) + aiohttp_connector = aiohttp_session.connector + assert isinstance(aiohttp_connector, aiohttp.TCPConnector) - aiohttp_session = aiohttp.ClientSession( - connector=aiohttp.TCPConnector(ssl=False) - ) - try: - aiohttp_connector = aiohttp_session.connector - assert isinstance(aiohttp_connector, aiohttp.TCPConnector) - - # assert both litellm transport and aiohttp session have ssl_verify=False - assert transport_connector._ssl == aiohttp_connector._ssl - finally: - await aiohttp_session.close() + # assert both litellm transport and aiohttp session have ssl_verify=False + assert transport_connector._ssl == aiohttp_connector._ssl finally: - await litellm_async_client.close() + await aiohttp_session.close() finally: - # Restore original setting - litellm.disable_aiohttp_transport = original_disable + await litellm_async_client.close() @pytest.mark.asyncio -async def test_ssl_verification_with_shared_session(): +async def test_ssl_verification_with_shared_session(monkeypatch: pytest.MonkeyPatch): """ Test that ssl_verify=False is respected even with shared sessions. @@ -257,67 +231,55 @@ async def test_ssl_verification_with_shared_session(): import aiohttp # Ensure aiohttp transport is enabled for this test - original_disable = litellm.disable_aiohttp_transport - litellm.disable_aiohttp_transport = False + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + + shared_session = aiohttp.ClientSession() try: - # Create a shared session (simulating what happens in production) - shared_session = aiohttp.ClientSession() + # Create transport with shared session and ssl_verify=False + transport = AsyncHTTPHandler._create_aiohttp_transport( + ssl_verify=False, + shared_session=shared_session, + ) - try: - # Create transport with shared session and ssl_verify=False - transport = AsyncHTTPHandler._create_aiohttp_transport( - ssl_verify=False, - shared_session=shared_session, - ) + # Verify the transport uses the shared session + assert transport.client is shared_session - # Verify the transport uses the shared session - assert transport.client is shared_session - - # Verify the SSL setting is stored in the transport for per-request use - assert transport._ssl_verify is False - finally: - await shared_session.close() + # Verify the SSL setting is stored in the transport for per-request use + assert transport._ssl_verify is False finally: - # Restore original setting - litellm.disable_aiohttp_transport = original_disable + await shared_session.close() @pytest.mark.asyncio -async def test_ssl_context_with_shared_session(): +async def test_ssl_context_with_shared_session(monkeypatch: pytest.MonkeyPatch): """ Test that ssl_context is respected even with shared sessions. """ import aiohttp # Ensure aiohttp transport is enabled for this test - original_disable = litellm.disable_aiohttp_transport - litellm.disable_aiohttp_transport = False + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + + custom_ssl_context = ssl.create_default_context() + + # Create a shared session + shared_session = aiohttp.ClientSession() try: - # Create a custom SSL context - custom_ssl_context = ssl.create_default_context() + # Create transport with shared session and custom ssl_context + transport = AsyncHTTPHandler._create_aiohttp_transport( + ssl_context=custom_ssl_context, + shared_session=shared_session, + ) - # Create a shared session - shared_session = aiohttp.ClientSession() + # Verify the transport uses the shared session + assert transport.client is shared_session - try: - # Create transport with shared session and custom ssl_context - transport = AsyncHTTPHandler._create_aiohttp_transport( - ssl_context=custom_ssl_context, - shared_session=shared_session, - ) - - # Verify the transport uses the shared session - assert transport.client is shared_session - - # Verify the SSL context is stored in the transport for per-request use - assert transport._ssl_verify is custom_ssl_context - finally: - await shared_session.close() + # Verify the SSL context is stored in the transport for per-request use + assert transport._ssl_verify is custom_ssl_context finally: - # Restore original setting - litellm.disable_aiohttp_transport = original_disable + await shared_session.close() def test_get_ssl_configuration(): @@ -563,26 +525,22 @@ def test_ssl_ecdh_curve( if env_curve: monkeypatch.setenv("SSL_ECDH_CURVE", env_curve) - original_value = litellm.ssl_ecdh_curve - try: - litellm.ssl_ecdh_curve = litellm_curve + monkeypatch.setattr(litellm, "ssl_ecdh_curve", litellm_curve) - # Create a real SSL context and patch set_ecdh_curve on it - # We need a real SSLContext instance (not a MagicMock) because _create_ssl_context - # calls methods like set_ciphers() and minimum_version that require a real context. - # We patch set_ecdh_curve specifically to verify it's called with the correct curve. - real_ssl_context = ssl.create_default_context() - with patch("ssl.create_default_context", return_value=real_ssl_context): - with patch.object(real_ssl_context, "set_ecdh_curve") as mock_set_curve: - ssl_context = get_ssl_configuration() + # Create a real SSL context and patch set_ecdh_curve on it + # We need a real SSLContext instance (not a MagicMock) because _create_ssl_context + # calls methods like set_ciphers() and minimum_version that require a real context. + # We patch set_ecdh_curve specifically to verify it's called with the correct curve. + real_ssl_context = ssl.create_default_context() + with patch("ssl.create_default_context", return_value=real_ssl_context): + with patch.object(real_ssl_context, "set_ecdh_curve") as mock_set_curve: + ssl_context = get_ssl_configuration() - if should_call: - mock_set_curve.assert_called_once_with(expected_curve) - else: - mock_set_curve.assert_not_called() - assert isinstance(ssl_context, ssl.SSLContext) - finally: - litellm.ssl_ecdh_curve = original_value + if should_call: + mock_set_curve.assert_called_once_with(expected_curve) + else: + mock_set_curve.assert_not_called() + assert isinstance(ssl_context, ssl.SSLContext) def test_default_user_agent_is_litellm_version(monkeypatch): @@ -753,46 +711,38 @@ class TestDefaultCachedClientTimeoutHonorsRequestTimeout: no per-model timeout (e.g. Bedrock) hung for 600s. """ - @pytest.fixture - def restore_request_timeout(self): - original_value = litellm.request_timeout - original_flag = litellm.request_timeout_explicitly_set - try: - yield - finally: - litellm.request_timeout = original_value - litellm.request_timeout_explicitly_set = original_flag - - def test_default_when_request_timeout_unset(self, restore_request_timeout): + def test_default_when_request_timeout_unset(self, monkeypatch: pytest.MonkeyPatch): from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TIMEOUT, _default_cached_client_timeout, ) - litellm.request_timeout = litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS - litellm.request_timeout_explicitly_set = False + monkeypatch.setattr( + litellm, "request_timeout", litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS + ) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False) assert _default_cached_client_timeout() is _DEFAULT_TIMEOUT - def test_uses_explicit_request_timeout(self, restore_request_timeout): + def test_uses_explicit_request_timeout(self, monkeypatch: pytest.MonkeyPatch): from litellm.llms.custom_httpx.http_handler import ( _default_cached_client_timeout, ) - litellm.request_timeout = 300 - litellm.request_timeout_explicitly_set = True + monkeypatch.setattr(litellm, "request_timeout", 300) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", True) resolved = _default_cached_client_timeout() assert resolved.read == 300.0 assert resolved.connect == 5.0 def test_cached_async_client_built_with_explicit_request_timeout( - self, restore_request_timeout + self, monkeypatch: pytest.MonkeyPatch ): from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.utils import LlmProviders - litellm.request_timeout = 300 - litellm.request_timeout_explicitly_set = True + monkeypatch.setattr(litellm, "request_timeout", 300) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", True) litellm.in_memory_llm_clients_cache = LLMClientCache() client = get_async_httpx_client(llm_provider=LlmProviders.BEDROCK) assert client.timeout.read == 300.0 diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index c87abbd8bc4..9faa77d6dce 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -1,15 +1,12 @@ import asyncio import json import logging -import os -import sys import time from unittest.mock import AsyncMock, Mock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path import litellm from litellm._logging import verbose_logger from litellm.integrations.code_interpreter_interception.handler import ( @@ -24,7 +21,10 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import ( BaseLLMHTTPHandler, + _collect_ws_project_quota_callbacks, _google_genai_streaming_hidden_params, + _has_pre_call_deployment_hook, + _rust_responses_websocket_enabled, ) from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams @@ -2445,3 +2445,82 @@ async def test_generic_http_handler_async_streaming_forwards_provider_response_h collected = [chunk async for chunk in response] assert "".join([chunk.choices[0].delta.content or "" for chunk in collected]) == "hi" + + +@pytest.mark.parametrize( + "custom_llm_provider, litellm_params, expected", + [ + ("openai", GenericLiteLLMParams(rust=True), True), + ("openai", GenericLiteLLMParams(), False), + ("openai", GenericLiteLLMParams(rust=False), False), + ("azure", GenericLiteLLMParams(rust=True), False), + ("hosted_vllm", GenericLiteLLMParams(rust=True), False), + (None, GenericLiteLLMParams(rust=True), False), + ], +) +def test_the_rust_responses_websocket_needs_both_openai_and_the_rust_flag( + custom_llm_provider, litellm_params, expected +): + assert _rust_responses_websocket_enabled(custom_llm_provider, litellm_params) is expected + + +def test_a_plain_callback_does_not_advertise_a_pre_call_deployment_hook(monkeypatch): + from litellm.integrations.custom_logger import CustomLogger + + class _PlainLogger(CustomLogger): + pass + + logging_obj = Mock() + logging_obj.dynamic_success_callbacks = [] + + monkeypatch.setattr(litellm, "callbacks", []) + assert _has_pre_call_deployment_hook(logging_obj) is False + + monkeypatch.setattr(litellm, "callbacks", [_PlainLogger()]) + assert _has_pre_call_deployment_hook(logging_obj) is False + + +def test_a_callback_that_overrides_the_deployment_hook_is_detected(monkeypatch): + from litellm.integrations.custom_logger import CustomLogger + + class _DeploymentHookLogger(CustomLogger): + async def async_pre_call_deployment_hook(self, kwargs, call_type): + return None + + class _InheritsTheHook(_DeploymentHookLogger): + pass + + logging_obj = Mock() + logging_obj.dynamic_success_callbacks = [] + + monkeypatch.setattr(litellm, "callbacks", [_DeploymentHookLogger()]) + assert _has_pre_call_deployment_hook(logging_obj) is True + + monkeypatch.setattr(litellm, "callbacks", [_InheritsTheHook()]) + assert _has_pre_call_deployment_hook(logging_obj) is True + + monkeypatch.setattr(litellm, "callbacks", []) + logging_obj.dynamic_success_callbacks = [_DeploymentHookLogger()] + assert _has_pre_call_deployment_hook(logging_obj) is True + + +def test_only_callbacks_that_can_charge_a_frame_are_collected_for_ws_quota(monkeypatch): + from litellm.integrations.custom_logger import CustomLogger + + class _PlainLogger(CustomLogger): + pass + + class _QuotaLogger(CustomLogger): + async def enforce_project_io_token_quota_for_frame(self, *args, **kwargs): + return None + + class _NotCallableAttribute: + enforce_project_io_token_quota_for_frame = "not a method" + + plain, quota, decoy = _PlainLogger(), _QuotaLogger(), _NotCallableAttribute() + + monkeypatch.setattr(litellm, "callbacks", [plain, decoy]) + assert _collect_ws_project_quota_callbacks() == () + + monkeypatch.setattr(litellm, "callbacks", [plain, quota, decoy]) + assert _collect_ws_project_quota_callbacks() == (quota,) diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_chat_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_chat_transformation.py index 8dbc197d4b5..d2a90baf6b2 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_chat_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_chat_transformation.py @@ -5,12 +5,7 @@ These tests validate the DashScopeConfig class which extends OpenAIGPTConfig. DashScope is an OpenAI-compatible provider with minor customizations. """ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.types.llms.openai import AllMessageValues import pytest diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index 510776ddfdf..8dc4620dd1b 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -10,12 +10,10 @@ Tests the cost calculation for Dashscope models including: import math import os -import sys import pytest # Add the project root to Python path -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.llms.dashscope.cost_calculator import ( diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_embedding_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_embedding_transformation.py index 5e4d0177e8d..1b6eea0e4c8 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_embedding_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_embedding_transformation.py @@ -3,14 +3,11 @@ Unit tests for DashScope embedding transformation. """ import json -import os -import sys from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.dashscope.common_utils import DashScopeError from litellm.llms.dashscope.embed.transformation import ( diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index 0e8d58b6530..936de812bc6 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -3,14 +3,11 @@ Unit tests for DashScope rerank transformation. """ import json -import os -import sys from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.dashscope.common_utils import DashScopeError from litellm.llms.dashscope.rerank.transformation import ( diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 165046a2298..41fb2589655 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -1,11 +1,8 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch from litellm.llms.databricks.chat.transformation import ( diff --git a/tests/test_litellm/llms/databricks/responses/test_databricks_responses_transformation.py b/tests/test_litellm/llms/databricks/responses/test_databricks_responses_transformation.py index b4a368be81f..4420506bf91 100644 --- a/tests/test_litellm/llms/databricks/responses/test_databricks_responses_transformation.py +++ b/tests/test_litellm/llms/databricks/responses/test_databricks_responses_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import patch import litellm diff --git a/tests/test_litellm/llms/databricks/test_databricks_common_utils.py b/tests/test_litellm/llms/databricks/test_databricks_common_utils.py index 7f7ec8e9000..ee50ffdabdc 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_common_utils.py +++ b/tests/test_litellm/llms/databricks/test_databricks_common_utils.py @@ -1,13 +1,8 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch from litellm.llms.databricks.common_utils import DatabricksBase diff --git a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py b/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py index 139990021b4..86fdd89acf6 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py +++ b/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py @@ -23,15 +23,11 @@ These tests align with Databricks Partner Architecture best practices: """ import json -import os import sys import pytest from unittest.mock import MagicMock, patch, Mock -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.llms.databricks.common_utils import DatabricksBase, DatabricksException diff --git a/tests/test_litellm/llms/datarobot/chat/test_datarobot_chat_transformation.py b/tests/test_litellm/llms/datarobot/chat/test_datarobot_chat_transformation.py index 3f772b263fd..153d37d549c 100644 --- a/tests/test_litellm/llms/datarobot/chat/test_datarobot_chat_transformation.py +++ b/tests/test_litellm/llms/datarobot/chat/test_datarobot_chat_transformation.py @@ -83,8 +83,8 @@ class TestDataRobotConfig: == api_base ) - def test_resolve_api_base_with_environment_variable(self, handler): - os.environ["DATAROBOT_ENDPOINT"] = "https://env.datarobot.com" + def test_resolve_api_base_with_environment_variable(self, handler, monkeypatch): + monkeypatch.setenv("DATAROBOT_ENDPOINT", "https://env.datarobot.com") assert ( handler._resolve_api_base(None) == "https://env.datarobot.com/api/v2/genai/llmgw/chat/completions/" @@ -101,7 +101,7 @@ class TestDataRobotConfig: def test_resolve_api_key(self, api_key, expected_api_key, handler): assert handler._resolve_api_key(api_key) == expected_api_key - def test_resolve_api_key_with_environment_variable(self, handler): - os.environ["DATAROBOT_API_TOKEN"] = "env_key" + def test_resolve_api_key_with_environment_variable(self, handler, monkeypatch): + monkeypatch.setenv("DATAROBOT_API_TOKEN", "env_key") assert handler._resolve_api_key(None) == "env_key" del os.environ["DATAROBOT_API_TOKEN"] diff --git a/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index d59ab975ef2..7c1f5256deb 100644 --- a/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,14 +1,10 @@ import io import os import pathlib -import sys from unittest.mock import MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.base_llm.audio_transcription.transformation import ( diff --git a/tests/test_litellm/llms/deepgram/test_deepgram_mock_transcription.py b/tests/test_litellm/llms/deepgram/test_deepgram_mock_transcription.py index 1f209004be8..e12b3982b13 100644 --- a/tests/test_litellm/llms/deepgram/test_deepgram_mock_transcription.py +++ b/tests/test_litellm/llms/deepgram/test_deepgram_mock_transcription.py @@ -1,15 +1,10 @@ import io import json -import os -import sys from typing import Any from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.types.utils import TranscriptionResponse diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py index a5eb836e71d..0865a14fbd9 100644 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py +++ b/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py @@ -1,24 +1,22 @@ import asyncio import json import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest # Add litellm to path -sys.path.insert(0, os.path.abspath("../../../..")) import litellm -def test_deepseek_supported_openai_params(): +def test_deepseek_supported_openai_params(monkeypatch): """ Test "reasoning_effort" is an openai param supported for the DeepSeek model on deepinfra """ from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig # Ensure we're using the local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") supported_openai_params = DeepInfraConfig().get_supported_openai_params( diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank.py index a1e47f815e7..02161b38fb6 100644 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank.py +++ b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank.py @@ -4,14 +4,11 @@ Tests for DeepInfra rerank functionality following repository patterns. import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest # Add litellm to path -sys.path.insert(0, os.path.abspath("../../../..")) import litellm diff --git a/tests/test_litellm/llms/docker_model_runner/test_docker_model_runner_chat_transformation.py b/tests/test_litellm/llms/docker_model_runner/test_docker_model_runner_chat_transformation.py index 0b4a2a5de8c..8ab24c16f14 100644 --- a/tests/test_litellm/llms/docker_model_runner/test_docker_model_runner_chat_transformation.py +++ b/tests/test_litellm/llms/docker_model_runner/test_docker_model_runner_chat_transformation.py @@ -5,10 +5,7 @@ This test validates that the DockerModelRunnerChatConfig correctly transforms requests to the proper URL, headers, and body format. """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../../..")) import json from typing import cast diff --git a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py b/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py index c0f74eff51b..f26a6aeafda 100644 --- a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py +++ b/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py @@ -1,9 +1,7 @@ import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" diff --git a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py b/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py index bf40abd7016..560eaf4f06b 100644 --- a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py +++ b/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py @@ -5,14 +5,9 @@ These tests validate the FeatherlessAIConfig class which extends OpenAIGPTConfig Featherless AI is an OpenAI-compatible provider with a few customizations. """ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.featherless_ai.chat.transformation import FeatherlessAIConfig diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index b46ba081f6f..e728fc4bc40 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,15 +1,10 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest import litellm -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm import get_model_info, supports_reasoning, supports_vision from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig diff --git a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py b/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py index 996f1fd975b..e1b88a9c78e 100644 --- a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.fireworks_ai.completion.transformation import ( FireworksAITextCompletionConfig, diff --git a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py b/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py index 4f76a39684a..e5a77aa8d41 100644 --- a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py @@ -1,13 +1,8 @@ -import os -import sys import pytest import litellm -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.fireworks_ai.completion.transformation import ( FireworksAITextCompletionConfig, diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py index b52c910d5a6..16226a3ce74 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.llms.fireworks_ai.common_utils import resolve_fireworks_resource_name diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index 3297750fa6e..f1664dabf48 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.fireworks_ai.cost_calculator import cost_per_token diff --git a/tests/test_litellm/llms/gdc/chat/test_gdc_chat_transformation.py b/tests/test_litellm/llms/gdc/chat/test_gdc_chat_transformation.py index d106cf7ea21..3153c12aa94 100644 --- a/tests/test_litellm/llms/gdc/chat/test_gdc_chat_transformation.py +++ b/tests/test_litellm/llms/gdc/chat/test_gdc_chat_transformation.py @@ -1,11 +1,8 @@ -import os -import sys from unittest.mock import MagicMock, patch import pytest # Adds the parent directory to the system path -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.gdc.chat.transformation import GDCGeminiConfig diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index 02004d5c8a8..deb148a07c0 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1,12 +1,9 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path import litellm from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig diff --git a/tests/test_litellm/llms/gemini/test_cost_calculator.py b/tests/test_litellm/llms/gemini/test_cost_calculator.py index 6917092966b..fc8d71afaa9 100644 --- a/tests/test_litellm/llms/gemini/test_cost_calculator.py +++ b/tests/test_litellm/llms/gemini/test_cost_calculator.py @@ -81,8 +81,8 @@ def test_no_usage_details(): assert cost == 0.0 -def test_gemini_image_edit_cost_prefers_token_usage_metadata(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_gemini_image_edit_cost_prefers_token_usage_metadata(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -120,8 +120,8 @@ def test_gemini_image_edit_cost_prefers_token_usage_metadata(): assert cost != flat_image_cost -def test_gemini_image_edit_cost_uses_output_token_details(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_gemini_image_edit_cost_uses_output_token_details(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -176,8 +176,8 @@ def test_gemini_image_edit_cost_uses_output_token_details(): assert cost != all_output_as_image_cost -def test_gemini_image_generation_cost_uses_output_token_details(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_gemini_image_generation_cost_uses_output_token_details(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -232,8 +232,8 @@ def test_gemini_image_generation_cost_uses_output_token_details(): assert cost != all_output_as_image_cost -def test_gemini_image_edit_cost_falls_back_to_flat_image_pricing(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_gemini_image_edit_cost_falls_back_to_flat_image_pricing(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -264,8 +264,8 @@ def _image_response_with_web_search(web_search_requests): return ImageResponse(data=[ImageObject(b64_json="img1")], usage=usage) -def test_gemini_image_generation_cost_adds_web_search_grounding(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_gemini_image_generation_cost_adds_web_search_grounding(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") @@ -286,8 +286,8 @@ def test_gemini_image_generation_cost_adds_web_search_grounding(): assert round(grounded - ungrounded, 10) == round(expected_web_search_cost, 10) -def test_gemini_image_generation_cost_no_web_search_when_absent(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_gemini_image_generation_cost_no_web_search_when_absent(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini/gemini-3-pro-image-preview" diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index 98f3ac0f4e5..4893825373a 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -2,14 +2,9 @@ Test Gemini TTS (Text-to-Speech) functionality """ -import os -import sys import pytest from unittest.mock import patch, MagicMock -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig diff --git a/tests/test_litellm/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py b/tests/test_litellm/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py index 90cf5a17398..f43e2e4d1cb 100644 --- a/tests/test_litellm/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py +++ b/tests/test_litellm/llms/github_copilot/embedding/test_github_copilot_embedding_transformation.py @@ -1,10 +1,7 @@ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.exceptions import AuthenticationError from litellm.llms.github_copilot.embedding.transformation import ( diff --git a/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py b/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py index 8ed84b3ed8d..8039e744f46 100644 --- a/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py +++ b/tests/test_litellm/llms/github_copilot/messages/test_github_copilot_messages_transformation.py @@ -1,10 +1,7 @@ -import os -import sys from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.exceptions import AuthenticationError from litellm.llms.github_copilot.common_utils import GetAPIKeyError diff --git a/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py b/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py index 174efceb499..c761d084da8 100644 --- a/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py +++ b/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py @@ -7,11 +7,8 @@ transformations for the Responses API. Source: litellm/llms/github_copilot/responses/transformation.py """ -import sys -import os from unittest.mock import patch, MagicMock -sys.path.insert(0, os.path.abspath("../../../../..")) import pytest import litellm diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index 51cffd5e51a..f1f1978b06f 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -1,14 +1,11 @@ import asyncio import json -import os -import sys from datetime import datetime, timedelta from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock, mock_open, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import httpx from respx import MockRouter diff --git a/tests/test_litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py b/tests/test_litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py index 6eabf2472ea..6586f970b80 100644 --- a/tests/test_litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py +++ b/tests/test_litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py @@ -1,10 +1,5 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.gradient_ai.chat.transformation import ( GradientAIConfig, diff --git a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 3ddb67b9f8d..e316cd14dd4 100644 --- a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,11 +1,6 @@ import json -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, diff --git a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_ssl_verify.py b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_ssl_verify.py index 8f98b3ca8f1..2364468efe1 100644 --- a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_ssl_verify.py +++ b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_ssl_verify.py @@ -8,15 +8,10 @@ Issue: ssl_verify parameter was being ignored because hosted_vllm fell through to the OpenAI catch-all path in main.py, which doesn't pass ssl_verify to the HTTP client. """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_ssl_verify.py b/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_ssl_verify.py index bb911814c23..de94da49384 100644 --- a/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_ssl_verify.py +++ b/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_ssl_verify.py @@ -8,15 +8,10 @@ Issue: ssl_verify parameter was being ignored because hosted_vllm fell through to the openai_like catch-all path in main.py, which doesn't pass ssl_verify to the HTTP client. """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py b/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py index 93c518599d6..34be3e12abd 100644 --- a/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py @@ -6,15 +6,10 @@ especially ensuring that encoding_format is not included when not provided. """ import json -import os -import sys from unittest.mock import Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.hosted_vllm.embedding.transformation import ( diff --git a/tests/test_litellm/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/test_litellm/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index eb578b86af0..e81bf0c4f1f 100644 --- a/tests/test_litellm/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/test_litellm/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -8,15 +8,10 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para """ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.hosted_vllm.responses.transformation import ( diff --git a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py index c907e3249d1..f1226311b5e 100644 --- a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py +++ b/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py @@ -1,11 +1,6 @@ import json -import os -import sys from unittest.mock import patch, MagicMock, AsyncMock -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm import pytest diff --git a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py index 0750fb9e405..cff3c6be940 100644 --- a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py +++ b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py @@ -231,10 +231,10 @@ def test_inception_in_provider_lists(): assert "https://api.inceptionlabs.ai/v1" in litellm.openai_compatible_endpoints -def test_inception_model_configuration(): +def test_inception_model_configuration(monkeypatch): from litellm import get_model_info - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") litellm.inception_models = set() litellm.add_known_models() @@ -251,8 +251,8 @@ def test_inception_model_configuration(): assert info.get("supports_response_schema") is True -def test_inception_model_list_populated(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_inception_model_list_populated(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") litellm.inception_models = set() litellm.add_known_models() diff --git a/tests/test_litellm/llms/inception/test_inception_completion_transformation.py b/tests/test_litellm/llms/inception/test_inception_completion_transformation.py index 9b7c8dd3742..62688a13c35 100644 --- a/tests/test_litellm/llms/inception/test_inception_completion_transformation.py +++ b/tests/test_litellm/llms/inception/test_inception_completion_transformation.py @@ -143,10 +143,10 @@ async def test_inception_fim_async(): assert r.choices[0].text == "a + b" -def test_inception_fim_model_configuration(): +def test_inception_fim_model_configuration(monkeypatch): from litellm import get_model_info - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") litellm.text_completion_inception_models = set() litellm.add_known_models() diff --git a/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py b/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py index 9e761817ef5..38355f32da1 100644 --- a/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py +++ b/tests/test_litellm/llms/jina_ai/embedding/test_jina_embedding_transformation.py @@ -1,10 +1,7 @@ -import os -import sys from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) # Adds the parent directory to the system path from litellm.llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig diff --git a/tests/test_litellm/llms/lemonade/test_lemonade.py b/tests/test_litellm/llms/lemonade/test_lemonade.py index cb70e7794a8..fa0d9d279a7 100644 --- a/tests/test_litellm/llms/lemonade/test_lemonade.py +++ b/tests/test_litellm/llms/lemonade/test_lemonade.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch import litellm diff --git a/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py b/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py index 43ce030323b..d6f76a16a9a 100644 --- a/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py +++ b/tests/test_litellm/llms/manus/responses/test_manus_responses_transformation.py @@ -7,10 +7,7 @@ transformations for the Responses API. Source: litellm/llms/manus/responses/transformation.py """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.manus.responses.transformation import ManusResponsesAPIConfig from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams diff --git a/tests/test_litellm/llms/meta_llama/test_meta_llama_chat_transformation.py b/tests/test_litellm/llms/meta_llama/test_meta_llama_chat_transformation.py index 7b974aba35c..15995d873c2 100644 --- a/tests/test_litellm/llms/meta_llama/test_meta_llama_chat_transformation.py +++ b/tests/test_litellm/llms/meta_llama/test_meta_llama_chat_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.meta_llama.chat.transformation import LlamaAPIConfig diff --git a/tests/test_litellm/llms/minimax/chat/test_transformation.py b/tests/test_litellm/llms/minimax/chat/test_transformation.py index 286498830c5..9d51b556500 100644 --- a/tests/test_litellm/llms/minimax/chat/test_transformation.py +++ b/tests/test_litellm/llms/minimax/chat/test_transformation.py @@ -3,14 +3,10 @@ Test MiniMax OpenAI-compatible API support """ import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../") -) # Adds the parent directory to the system path import litellm from litellm import completion diff --git a/tests/test_litellm/llms/minimax/messages/test_transformation.py b/tests/test_litellm/llms/minimax/messages/test_transformation.py index 6e4b0428bb9..01d32221fe5 100644 --- a/tests/test_litellm/llms/minimax/messages/test_transformation.py +++ b/tests/test_litellm/llms/minimax/messages/test_transformation.py @@ -3,14 +3,10 @@ Test MiniMax Anthropic-compatible API support """ import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../") -) # Adds the parent directory to the system path import litellm from litellm import completion diff --git a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py index 55c5d05cdc0..15694d9f218 100644 --- a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py @@ -1,5 +1,3 @@ -import os -import sys from typing import List, cast from unittest.mock import MagicMock, patch @@ -8,9 +6,6 @@ import pytest from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.types.llms.openai import AllMessageValues -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from litellm.llms.mistral.chat.transformation import ( MistralChatResponseIterator, diff --git a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py index 2767deae176..6fe39798f4f 100644 --- a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py +++ b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py @@ -7,11 +7,7 @@ ModelScope is an OpenAI-compatible provider with minor customizations. import json import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from unittest.mock import patch diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py index fbcec3d4d2e..2ffe7c3e686 100644 --- a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -5,15 +5,10 @@ These tests validate the ModelScopeImageGenerationConfig class which handles transformation between OpenAI-compatible format and ModelScope API format. """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.modelscope.image_generation.transformation import ( ModelScopeImageGenerationConfig, diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py index 417dd4a767c..50f476eaaaa 100644 --- a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -5,11 +5,8 @@ These tests validate the MoonshotChatConfig class which extends OpenAIGPTConfig. Moonshot AI is an OpenAI-compatible provider with minor customizations. """ -import os -import sys from unittest.mock import patch -sys.path.insert(0, os.path.abspath("../../../../..")) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py b/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py index cb15dd3fa3e..6d77e81b767 100644 --- a/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py +++ b/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py @@ -5,12 +5,7 @@ These tests validate the NebiusConfig class which extends OpenAIGPTConfig. Nebius AI Studio is an OpenAI-compatible provider with minor customizations. """ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py b/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py index ade5e4176e8..3f2a3f77c41 100644 --- a/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py +++ b/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py @@ -5,16 +5,11 @@ These tests validate the NovitaConfig class which extends OpenAIGPTConfig. Novita AI is an OpenAI-compatible provider with a few customizations. """ -import os -import sys from typing import Dict, List, Optional from unittest.mock import patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.novita.chat.transformation import NovitaConfig diff --git a/tests/test_litellm/llms/nscale/chat/test_nscale_chat_transformation.py b/tests/test_litellm/llms/nscale/chat/test_nscale_chat_transformation.py index 4fcd79ae2a1..415ce9ce9c9 100644 --- a/tests/test_litellm/llms/nscale/chat/test_nscale_chat_transformation.py +++ b/tests/test_litellm/llms/nscale/chat/test_nscale_chat_transformation.py @@ -1,10 +1,6 @@ import os -import sys from unittest.mock import patch -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.nscale.chat.transformation import NscaleConfig diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py index 0e355b91ca8..63a53c2c97b 100644 --- a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py +++ b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py @@ -15,7 +15,6 @@ import numpy as np import pytest import soundfile as sf -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.nvidia_riva.audio_transcription.audio_utils import ( resample_to_riva_pcm, diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py index 341a0e77ce0..7ecc0b47d9f 100644 --- a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py +++ b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py @@ -9,8 +9,6 @@ is aggregated. import asyncio import io -import os -import sys from types import SimpleNamespace from unittest.mock import MagicMock @@ -18,7 +16,6 @@ import numpy as np import pytest import soundfile as sf -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.nvidia_riva.audio_transcription import handler as handler_mod from litellm.llms.nvidia_riva.audio_transcription.handler import ( diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py index c4cca8490bf..38489328e30 100644 --- a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py +++ b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py @@ -5,12 +5,9 @@ These tests do not require ``nvidia-riva-client`` or any audio libs to be installed; the transformation layer is intentionally pure-Python on dicts. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 5aa96a66d2d..86c534c73c2 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -1,6 +1,4 @@ import datetime -import os -import sys import httpx import pytest import json @@ -8,7 +6,6 @@ import json import litellm # Adds the parent directory to the system path -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm import ModelResponse from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS diff --git a/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py b/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py index a4a5f111513..0a47852d085 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py @@ -106,7 +106,7 @@ class TestGenericToolCallErrors: ) def test_non_string_id_raises(self): - with pytest.raises(OCIError, match="id.*must be a string"): + with pytest.raises(OCIError, match=r"id.*must be a string"): adapt_messages_to_generic_oci_standard_tool_call( "assistant", [ @@ -126,7 +126,7 @@ class TestGenericToolCallErrors: ) def test_non_string_function_name_raises(self): - with pytest.raises(OCIError, match="function.name.*must be a string"): + with pytest.raises(OCIError, match=r"function\.name.*must be a string"): adapt_messages_to_generic_oci_standard_tool_call( "assistant", [ @@ -139,7 +139,7 @@ class TestGenericToolCallErrors: ) def test_non_string_arguments_raises(self): - with pytest.raises(OCIError, match="arguments.*must be a JSON string"): + with pytest.raises(OCIError, match=r"arguments.*must be a JSON string"): adapt_messages_to_generic_oci_standard_tool_call( "assistant", [ diff --git a/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py index acad5da93e2..002def9196d 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py @@ -9,10 +9,7 @@ Issue: OCI API returns tool calls with incomplete structures during streaming Error: ValidationError: 1 validation error for OCIStreamChunk message.toolCalls.0.arguments Field required """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.oci.chat.generic import handle_generic_stream_chunk from litellm.types.utils import ModelResponseStream diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py b/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py index 30f49bea344..363c0b46809 100644 --- a/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py +++ b/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py @@ -5,15 +5,12 @@ These tests exercise the transformation layer only — no real OCI calls are mad """ import json -import os -import sys from typing import Any from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.oci.common_utils import OCIError from litellm.llms.oci.embed.transformation import OCI_EMBED_BATCH_LIMIT, OCIEmbedConfig diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py index 61c13ad62a1..46a91520ab0 100644 --- a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -1,12 +1,10 @@ import json import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig from litellm.types.utils import EmbeddingResponse diff --git a/tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py b/tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py index f6151497e1c..525788c158a 100644 --- a/tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py +++ b/tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py @@ -2,13 +2,10 @@ Unit tests for OCR Guardrail Translation Handler """ -import os import re -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms import get_guardrail_translation_mapping diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index dd59cdcac1c..acd69b94d02 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -1,14 +1,9 @@ import json -import os -import sys from litellm._uuid import uuid from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.ollama.completion.transformation import ( OllamaConfig, diff --git a/tests/test_litellm/llms/ollama/test_ollama_model_info.py b/tests/test_litellm/llms/ollama/test_ollama_model_info.py index 8d46151ecce..053d4da035f 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_model_info.py +++ b/tests/test_litellm/llms/ollama/test_ollama_model_info.py @@ -3,15 +3,11 @@ import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path """ Unit tests for OllamaModelInfo.get_models functionality. """ # Ensure a dummy httpx module is available for import in tests -import sys import types # Provide a dummy httpx module for import in get_models diff --git a/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py b/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py index 91ebb2bd9d4..395a4fb5715 100644 --- a/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py +++ b/tests/test_litellm/llms/oobabooga/chat/test_oobabooga.py @@ -1,8 +1,5 @@ -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 2e75f29b1c5..a29e0be4655 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -6,15 +6,10 @@ with guardrail transformations, including tool calls. """ import json -import os -import sys from typing import Any, Literal, Optional import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../../..") -) # Adds the parent directory to the system path from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.openai.chat.guardrail_translation.handler import ( diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 101c5363bf7..f4c38f8f797 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,12 +2,9 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY diff --git a/tests/test_litellm/llms/openai/completion/test_completion_handler.py b/tests/test_litellm/llms/openai/completion/test_completion_handler.py index c6af96fa375..329956605ab 100644 --- a/tests/test_litellm/llms/openai/completion/test_completion_handler.py +++ b/tests/test_litellm/llms/openai/completion/test_completion_handler.py @@ -5,14 +5,11 @@ text completion path. Regression tests for https://github.com/BerriAI/litellm/issues/27410 """ -import os -import sys import pytest import respx from httpx import Response -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm import atext_completion, text_completion diff --git a/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py b/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py index 257db89d073..c96fbf34fe1 100644 --- a/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py @@ -2,14 +2,11 @@ Unit tests for OpenAI Text Completion Guardrail Translation Handler """ -import os -import sys from typing import List, Optional, Tuple from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms import get_guardrail_translation_mapping diff --git a/tests/test_litellm/llms/openai/completion/test_text_completion_token_ids.py b/tests/test_litellm/llms/openai/completion/test_text_completion_token_ids.py index 9c612af3898..35faeeb268a 100644 --- a/tests/test_litellm/llms/openai/completion/test_text_completion_token_ids.py +++ b/tests/test_litellm/llms/openai/completion/test_text_completion_token_ids.py @@ -3,14 +3,11 @@ Unit tests for text_completion with token IDs (list of integers) as prompt. Tests the fix for https://github.com/BerriAI/litellm/issues/17118 """ -import os -import sys import pytest import respx from httpx import Response -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm import text_completion diff --git a/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py b/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py index cfccd6f3bbe..0d699b1ec95 100644 --- a/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py @@ -2,13 +2,10 @@ Unit tests for OpenAI Image Generation Guardrail Translation Handler """ -import os -import sys from typing import List, Optional, Tuple import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms import get_guardrail_translation_mapping diff --git a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py index 33db9d33c1c..06871edb773 100644 --- a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py +++ b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py @@ -6,13 +6,10 @@ litellm.aimage_generation() are forwarded to the OpenAI API client as extra_headers in the images.generate() call. """ -import os -import sys from unittest.mock import MagicMock, AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.openai.openai import OpenAIChatCompletion diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index 2633e76b0f3..4221954d787 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -8,9 +6,6 @@ import pytest from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py index 62fc3a8d0aa..54f206d098d 100644 --- a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py @@ -5,14 +5,11 @@ Tests for the Realtime transcription_sessions surface used by gpt-realtime-whisp - BaseLLMHTTPHandler.async_realtime_transcription_session_handler targeting """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py index 195fba69010..e1cc6a92927 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.openai.responses.count_tokens.transformation import ( OpenAICountTokensConfig, ) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 4c45eaac7b9..447175b09a6 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -5,16 +5,11 @@ Tests the handler's ability to process input/output for the Responses API with guardrail transformations. """ -import os -import sys from typing import Any, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path from fastapi import HTTPException from openai.types.responses import ResponseFunctionToolCall diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 13b96dc9943..c03c632363d 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1,14 +1,9 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig diff --git a/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py b/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py index 5b6387cb100..88149d82c52 100644 --- a/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py @@ -2,13 +2,10 @@ Unit tests for OpenAI Text-to-Speech Guardrail Translation Handler """ -import os -import sys from typing import List, Optional, Tuple import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms import get_guardrail_translation_mapping diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index bfd681cc06e..3ae29e411e8 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -1,14 +1,9 @@ -import os -import sys from unittest.mock import MagicMock, call, patch import httpx import openai import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.litellm_core_utils.token_counter import token_counter @@ -86,7 +81,6 @@ async def test_openai_client_reuse(function_name, is_async, args): """ Test that multiple API calls reuse the same OpenAI client """ - litellm.set_verbose = True # Determine which client class to mock based on whether the test is async client_path = ( diff --git a/tests/test_litellm/llms/openai/test_openai_empty_response.py b/tests/test_litellm/llms/openai/test_openai_empty_response.py index 8a0ff237869..26b28f967db 100644 --- a/tests/test_litellm/llms/openai/test_openai_empty_response.py +++ b/tests/test_litellm/llms/openai/test_openai_empty_response.py @@ -2,13 +2,10 @@ Test for issue #17209: Clearer error when LLM endpoint returns empty response """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.llms.openai.openai import OpenAIChatCompletion from litellm.llms.openai.common_utils import OpenAIError diff --git a/tests/test_litellm/llms/openai/test_use_chat_completions_api_no_leak.py b/tests/test_litellm/llms/openai/test_use_chat_completions_api_no_leak.py index 9a266fca81f..7013afc7a5f 100644 --- a/tests/test_litellm/llms/openai/test_use_chat_completions_api_no_leak.py +++ b/tests/test_litellm/llms/openai/test_use_chat_completions_api_no_leak.py @@ -7,11 +7,8 @@ proxy config, it must never be forwarded to the upstream provider's request body. OpenAI/Anthropic reject unknown body params with HTTP 400. """ -import os -import sys from unittest.mock import MagicMock -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.types.utils import all_litellm_params diff --git a/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py b/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py index 307972ff477..269cbc7855d 100644 --- a/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py @@ -2,13 +2,10 @@ Unit tests for OpenAI Audio Transcription Guardrail Translation Handler """ -import os -import sys from typing import List, Optional, Tuple import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms import get_guardrail_translation_mapping diff --git a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py new file mode 100644 index 00000000000..1ce2da65fef --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py @@ -0,0 +1,211 @@ +""" +Tests for SCX.ai provider configuration and integration. +""" + +import litellm + + +class TestSCXAIProviderConfig: + def test_scx_ai_in_provider_list(self): + from litellm import LlmProviders + + assert hasattr(LlmProviders, "SCX_AI") + assert LlmProviders.SCX_AI.value == "scx-ai" + assert "scx-ai" in litellm.provider_list + + def test_scx_ai_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("scx-ai") + + scx = JSONProviderRegistry.get("scx-ai") + assert scx is not None + assert scx.base_url == "https://api.scx.ai/v1" + assert scx.api_key_env == "SCX_API_KEY" + assert scx.param_mappings.get("max_completion_tokens") == "max_tokens" + assert scx.constraints.get("temperature_max") == 1.99 + + def test_scx_ai_in_openai_compatible_providers(self): + from litellm.constants import openai_compatible_providers + + assert "scx-ai" in openai_compatible_providers + + def test_scx_ai_provider_resolution(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="scx-ai/GLM-5.2", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "GLM-5.2" + assert provider == "scx-ai" + assert api_base == "https://api.scx.ai/v1" + + def test_scx_ai_api_base_override(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="scx-ai/GLM-5.2", + custom_llm_provider=None, + api_base="https://custom.scx.ai/v1", + api_key="sk-test", + ) + + assert provider == "scx-ai" + assert api_base == "https://custom.scx.ai/v1" + assert api_key == "sk-test" + + def test_scx_ai_url_autodetection(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="GLM-5.2", + custom_llm_provider=None, + api_base="https://api.scx.ai/v1", + api_key=None, + ) + assert provider == "scx-ai" + assert api_base == "https://api.scx.ai/v1" + + def test_scx_ai_temperature_clamped_to_max(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("scx-ai") + assert provider is not None + config = create_config_class(provider)() + + optional_params = config.map_openai_params( + non_default_params={"temperature": 2.5}, + optional_params={}, + model="GLM-5.2", + drop_params=False, + ) + assert optional_params["temperature"] == 1.99 + + optional_params = config.map_openai_params( + non_default_params={"temperature": 1.7}, + optional_params={}, + model="GLM-5.2", + drop_params=False, + ) + assert optional_params["temperature"] == 1.7 + + optional_params = config.map_openai_params( + non_default_params={"temperature": 0.4}, + optional_params={}, + model="GLM-5.2", + drop_params=False, + ) + assert optional_params["temperature"] == 0.4 + + def test_scx_ai_max_completion_tokens_mapped(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("scx-ai") + assert provider is not None + config = create_config_class(provider)() + + optional_params = config.map_openai_params( + non_default_params={"max_completion_tokens": 256}, + optional_params={}, + model="GLM-5.2", + drop_params=False, + ) + assert optional_params["max_tokens"] == 256 + assert "max_completion_tokens" not in optional_params + + def test_scx_ai_router_config(self): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "scx-chat", + "litellm_params": { + "model": "scx-ai/GLM-5.2", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "scx-chat" + + +class TestSCXAIModelMetadata: + SCX_MODELS = ( + "scx-ai/GLM-5.2", + "scx-ai/Qwen3.8-Max", + ) + VISION_MODELS = ("scx-ai/Qwen3.8-Max",) + + @staticmethod + def _load(path_parts): + import json + from pathlib import Path + + json_path = Path(__file__).parents[4].joinpath(*path_parts) + with open(json_path) as f: + return json.load(f) + + def test_scx_ai_models_registered_with_correct_metadata(self): + model_cost = self._load(("model_prices_and_context_window.json",)) + for model in self.SCX_MODELS: + info = model_cost.get(model) + assert info is not None, f"{model} missing from model_prices_and_context_window.json" + assert info["litellm_provider"] == "scx-ai" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] > 0 + assert info["output_cost_per_token"] > 0 + assert info["supports_function_calling"] is True + assert info["supports_tool_choice"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info.get("supports_vision", False) is (model in self.VISION_MODELS) + + assert info["supports_prompt_caching"] is True + assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"] + + assert info["max_output_tokens"] == 131072 + assert info["max_tokens"] == info["max_output_tokens"] + assert info["max_input_tokens"] >= 1_000_000 + + def test_scx_ai_models_synced_to_backup(self): + model_cost = self._load(("model_prices_and_context_window.json",)) + backup = self._load(("litellm", "model_prices_and_context_window_backup.json")) + for model in self.SCX_MODELS: + assert model in backup, f"{model} missing from backup json" + assert backup[model] == model_cost[model], f"{model} differs between root and backup json" + + +class TestSCXAIDashboardRegistration: + @staticmethod + def _provider_create_fields(): + import json + from pathlib import Path + + import litellm + + path = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + with open(path) as f: + return json.load(f) + + def test_scx_ai_is_selectable_in_the_add_model_form(self): + entries = [e for e in self._provider_create_fields() if e["litellm_provider"] == "scx-ai"] + assert len(entries) == 1, "scx-ai must appear exactly once in provider_create_fields.json" + + entry = entries[0] + assert entry["provider"] == "SCX_AI" + assert entry["provider_display_name"] == "SCX.ai" + assert entry["default_model_placeholder"].startswith("scx-ai/") + + fields = {f["key"]: f for f in entry["credential_fields"]} + assert fields["api_key"]["required"] is True + assert fields["api_key"]["field_type"] == "password" + assert fields["api_base"]["required"] is False diff --git a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py index 8d1129cc5da..b177b80aed1 100644 --- a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,12 +1,7 @@ -import os -import sys import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openrouter.chat.transformation import ( diff --git a/tests/test_litellm/llms/openrouter/image_edit/test_openrouter_image_edit_transformation.py b/tests/test_litellm/llms/openrouter/image_edit/test_openrouter_image_edit_transformation.py index f352c077fc4..5a78560f61b 100644 --- a/tests/test_litellm/llms/openrouter/image_edit/test_openrouter_image_edit_transformation.py +++ b/tests/test_litellm/llms/openrouter/image_edit/test_openrouter_image_edit_transformation.py @@ -1,16 +1,11 @@ import base64 import json -import os -import sys from io import BytesIO from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.openrouter.common_utils import OpenRouterException from litellm.llms.openrouter.image_edit.transformation import ( diff --git a/tests/test_litellm/llms/openrouter/image_generation/test_openrouter_image_gen_transformation.py b/tests/test_litellm/llms/openrouter/image_generation/test_openrouter_image_gen_transformation.py index 52a4fabaed7..e45270fb5e3 100644 --- a/tests/test_litellm/llms/openrouter/image_generation/test_openrouter_image_gen_transformation.py +++ b/tests/test_litellm/llms/openrouter/image_generation/test_openrouter_image_gen_transformation.py @@ -1,14 +1,9 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.openrouter.image_generation.transformation import ( OpenRouterImageGenerationConfig, diff --git a/tests/test_litellm/llms/openrouter/test_openrouter_provider_routing.py b/tests/test_litellm/llms/openrouter/test_openrouter_provider_routing.py index 0815b15c873..d2e4e88e77f 100644 --- a/tests/test_litellm/llms/openrouter/test_openrouter_provider_routing.py +++ b/tests/test_litellm/llms/openrouter/test_openrouter_provider_routing.py @@ -11,12 +11,9 @@ so the correct model ID is sent to the OpenRouter API. See: https://github.com/BerriAI/litellm/issues/16353 """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py index 40d57c76d02..057ab9ede9a 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -3,16 +3,12 @@ Unit tests for OVHCloud AI Endpoints chat integration. """ import os -import sys import pytest from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.utils import get_optional_params -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.ovhcloud.chat.transformation import ( OVHCloudChatCompletionStreamingHandler, @@ -179,7 +175,6 @@ class TestOVHCloudConfig: def test_ovhcloud_integration(): - import os from litellm import completion api_key = os.getenv("OVHCLOUD_API_KEY") @@ -207,7 +202,6 @@ def test_OVHCloud_streaming_integration(): Integration test for streaming - requires real API key Run with: pytest -k test_OVHCloud_streaming_integration -s """ - import os from litellm import completion api_key = os.getenv("OVHCLOUD_API_KEY") @@ -262,7 +256,6 @@ def test_ovhcloud_with_custom_base_url(): """ Test OVHCloud with custom base URL """ - import os from litellm import completion api_key = os.getenv("OVHCLOUD_API_KEY") diff --git a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py index 7be295826e3..8a9ae4dae6d 100644 --- a/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py +++ b/tests/test_litellm/llms/parallel_ai/test_parallel_ai_search.py @@ -2,13 +2,10 @@ Tests for Parallel AI Search API integration (v1 endpoint). """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm diff --git a/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py b/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py index af441313d58..29d185b686d 100644 --- a/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py +++ b/tests/test_litellm/llms/perplexity/chat/test_perplexity_chat_transformation.py @@ -5,14 +5,11 @@ Tests the response transformation to extract citation tokens and search queries from Perplexity API responses. """ -import os -import sys from unittest.mock import Mock import pytest # Add the project root to Python path -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm import ModelResponse from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig diff --git a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py index a3ec81c569c..534176e381a 100644 --- a/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py +++ b/tests/test_litellm/llms/perplexity/responses/test_perplexity_responses_transformation.py @@ -8,13 +8,10 @@ Source: litellm/llms/perplexity/responses/transformation.py """ import json -import os -import sys import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/test_litellm/llms/perplexity/test_perplexity.py b/tests/test_litellm/llms/perplexity/test_perplexity.py index c6fb819e97b..797a56070c6 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) import pytest diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index 16708e062e4..117379c331a 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -8,13 +8,11 @@ search queries, and reasoning tokens. import json import math import os -import sys from unittest.mock import patch import pytest # Add the project root to Python path -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.cost_calculator import completion_cost, cost_per_token diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py index 8691e6a1ee5..990fa7eb464 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py @@ -8,12 +8,10 @@ including integration with the main LiteLLM cost calculator. import json import math import os -import sys import pytest # Add the project root to Python path -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm import ModelResponse diff --git a/tests/test_litellm/llms/publicai/test_publicai_chat_transformation.py b/tests/test_litellm/llms/publicai/test_publicai_chat_transformation.py index 2dabf604b98..487f311b2fc 100644 --- a/tests/test_litellm/llms/publicai/test_publicai_chat_transformation.py +++ b/tests/test_litellm/llms/publicai/test_publicai_chat_transformation.py @@ -5,11 +5,8 @@ These tests validate the PublicAI configuration which is now JSON-based. PublicAI is an OpenAI-compatible provider with minor customizations. """ -import os -import sys from unittest.mock import patch -sys.path.insert(0, os.path.abspath("../../../../..")) import pytest diff --git a/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py b/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py index baf2ab33910..437f53fea1a 100644 --- a/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py +++ b/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py @@ -6,13 +6,11 @@ for RAGFlow's OpenAI-compatible API with custom path structures. """ import os -import sys from unittest.mock import Mock, patch import pytest # Add the project root to Python path -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.ragflow.chat.transformation import RAGFlowConfig diff --git a/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py b/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py index 47811321133..97d65935a1b 100644 --- a/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py +++ b/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py @@ -1,6 +1,4 @@ import json -import os -import sys from io import BufferedReader, BytesIO from typing import Dict, List from unittest.mock import MagicMock, mock_open, patch @@ -8,9 +6,6 @@ from unittest.mock import MagicMock, mock_open, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.recraft.image_edit.transformation import RecraftImageEditConfig from litellm.types.images.main import ImageEditOptionalRequestParams diff --git a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py index ccc72dde7b8..2dfe33b828c 100644 --- a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py +++ b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py @@ -1,15 +1,10 @@ import json -import os -import sys from typing import List, Optional from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.recraft.image_generation.transformation import ( RecraftImageGenerationConfig, diff --git a/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py b/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py index 8871260813d..2e4d68a02da 100644 --- a/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py +++ b/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py @@ -2,10 +2,7 @@ Test RunwayML text-to-speech transformation """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py index f928964dab8..e2dd3bca74f 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py @@ -1,12 +1,9 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.sagemaker.common_utils import AWSEventStreamDecoder from litellm.llms.sagemaker.completion.transformation import SagemakerConfig diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py index c7ffe727d1a..2a14d58a187 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py @@ -7,12 +7,9 @@ matching the behavior of the completion handler. """ import json -import os -import sys from datetime import timezone from unittest.mock import MagicMock, call, patch -sys.path.insert(0, os.path.abspath("../../../../..")) from botocore.credentials import Credentials diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py index 943a3160bb7..3951b17db92 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py @@ -7,14 +7,11 @@ transformation, and model type detection. """ import json -import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm import embedding from litellm.llms.sagemaker.embedding.cohere_transformation import ( diff --git a/tests/test_litellm/llms/test_cache_control_and_reasoning.py b/tests/test_litellm/llms/test_cache_control_and_reasoning.py index 42f754bc093..be1ba1e7dbd 100644 --- a/tests/test_litellm/llms/test_cache_control_and_reasoning.py +++ b/tests/test_litellm/llms/test_cache_control_and_reasoning.py @@ -7,14 +7,9 @@ This test file verifies the fixes for Issue #19923: - Model metadata correctly reflects capabilities """ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.llms.minimax.chat.transformation import MinimaxChatConfig from litellm.llms.openrouter.chat.transformation import OpenrouterConfig diff --git a/tests/test_litellm/llms/vercel_ai_gateway/chat/test_vercel_ai_gateway_transformation.py b/tests/test_litellm/llms/vercel_ai_gateway/chat/test_vercel_ai_gateway_transformation.py index f6ac8af1115..a58942559fd 100644 --- a/tests/test_litellm/llms/vercel_ai_gateway/chat/test_vercel_ai_gateway_transformation.py +++ b/tests/test_litellm/llms/vercel_ai_gateway/chat/test_vercel_ai_gateway_transformation.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.vercel_ai_gateway.chat.transformation import ( VercelAIGatewayConfig, diff --git a/tests/test_litellm/llms/vercel_ai_gateway/embedding/test_vercel_ai_gateway_embedding.py b/tests/test_litellm/llms/vercel_ai_gateway/embedding/test_vercel_ai_gateway_embedding.py index af1e1df92fd..7ce91558f39 100644 --- a/tests/test_litellm/llms/vercel_ai_gateway/embedding/test_vercel_ai_gateway_embedding.py +++ b/tests/test_litellm/llms/vercel_ai_gateway/embedding/test_vercel_ai_gateway_embedding.py @@ -1,13 +1,9 @@ import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.vercel_ai_gateway.embedding.transformation import ( VercelAIGatewayEmbeddingConfig, diff --git a/tests/test_litellm/llms/vertex_ai/agent_engine/test_transformation.py b/tests/test_litellm/llms/vertex_ai/agent_engine/test_transformation.py index af0faee9e21..19616682c59 100644 --- a/tests/test_litellm/llms/vertex_ai/agent_engine/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/agent_engine/test_transformation.py @@ -4,12 +4,9 @@ Tests for Vertex AI Agent Engine transformation. Tests the request transformation and streaming chunk parsing without making real API calls. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( VertexAgentEngineResponseIterator, diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py index 3fa28699f73..3a1922d1021 100644 --- a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py @@ -1,13 +1,11 @@ import base64 import json import os -import sys from urllib.parse import urlparse import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.vertex_ai.audio_transcription.transformation import ( diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_handler.py b/tests/test_litellm/llms/vertex_ai/batches/test_handler.py index 9535bf17411..38fde3caa63 100644 --- a/tests/test_litellm/llms/vertex_ai/batches/test_handler.py +++ b/tests/test_litellm/llms/vertex_ai/batches/test_handler.py @@ -30,14 +30,11 @@ from __future__ import annotations import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.vertex_ai.batches.handler import ( # noqa: E402 VertexAIBatchPrediction, diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py b/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py index 8352ec16389..ccb2d7e310d 100644 --- a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py @@ -13,13 +13,10 @@ There are no real I/O seams here; ``uuid.uuid4`` is the only nondeterministic dependency and is patched where the displayName is asserted. """ -import os -import sys from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402 VertexAIBatchTransformation, diff --git a/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index ad890d0c7ea..f666829d2e8 100644 --- a/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,14 +1,9 @@ -import os -import sys from typing import List from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_handler.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_handler.py index 5e854bbad70..0a44f0a9a74 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_handler.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_handler.py @@ -3,6 +3,7 @@ Test Vertex AI files handler functionality """ import asyncio +import re from types import MappingProxyType import pytest from unittest.mock import AsyncMock, patch @@ -180,7 +181,10 @@ class TestVertexAIFilesHandler: # Should raise ValueError for failed download with pytest.raises( ValueError, - match="Failed to download file from GCS: gs://test-bucket/litellm-vertex-files/uploads/abc-test-file.txt", + match=re.escape( + "Failed to download file from GCS: " + "gs://test-bucket/litellm-vertex-files/uploads/abc-test-file.txt" + ), ): await self.handler.afile_content( file_content_request=file_content_request, diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py index 756923c5df6..fad310fc5c0 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.vertex_ai.gemini import transformation from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py index cd866187166..e54e25cbd18 100644 --- a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py @@ -30,8 +30,8 @@ def _image_response_with_web_search(web_search_requests): return ImageResponse(data=[ImageObject(b64_json="img1")], usage=usage) -def test_vertex_image_generation_cost_adds_web_search_grounding(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_vertex_image_generation_cost_adds_web_search_grounding(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3-pro-image-preview" model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") @@ -55,8 +55,8 @@ def test_vertex_image_generation_cost_adds_web_search_grounding(): assert round(grounded - ungrounded, 10) == round(expected_web_search_cost, 10) -def test_vertex_image_generation_cost_no_web_search_when_absent(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_vertex_image_generation_cost_no_web_search_when_absent(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model = "gemini-3-pro-image-preview" diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py index 8c72bdee525..54607cc5284 100644 --- a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -1,11 +1,9 @@ import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.llms.vertex_ai.image_generation import ( get_vertex_ai_image_generation_config, diff --git a/tests/test_litellm/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py b/tests/test_litellm/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py index 6b605aed0ca..edb6e889814 100644 --- a/tests/test_litellm/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/multimodal_embeddings/test_vertex_ai_multimodal_embedding_transformation.py @@ -1,14 +1,9 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.llms.vertex_ai.multimodal_embeddings.transformation import ( VertexAIMultimodalEmbeddingConfig, diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index f11b00d204d..720c629cbf7 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -10,14 +10,11 @@ Validates: """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest import websockets.exceptions # registers websockets.exceptions on the websockets namespace -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig diff --git a/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py b/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py index d8b299dcf66..7538c070cd0 100644 --- a/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py +++ b/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py @@ -6,11 +6,8 @@ and that the request body is properly formatted. """ import json -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.abspath("../../../..")) import pytest diff --git a/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py b/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py index 26aa85a886e..9e960570036 100644 --- a/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py @@ -5,10 +5,7 @@ This test verifies that the BGE response transformer properly validates and handles different response formats. """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) import pytest diff --git a/tests/test_litellm/llms/vertex_ai/test_gemini_empty_properties.py b/tests/test_litellm/llms/vertex_ai/test_gemini_empty_properties.py index 1a4e4d35ca9..441b598e751 100644 --- a/tests/test_litellm/llms/vertex_ai/test_gemini_empty_properties.py +++ b/tests/test_litellm/llms/vertex_ai/test_gemini_empty_properties.py @@ -1,9 +1,6 @@ """Test for Gemini schema handling with empty properties.""" -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.llms.vertex_ai.common_utils import add_object_type diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex.py b/tests/test_litellm/llms/vertex_ai/test_vertex.py index ec73e5e42be..e3007bac7f3 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex.py @@ -1,7 +1,5 @@ import base64 import json -import os -import sys from dotenv import load_dotenv @@ -12,9 +10,6 @@ import litellm.litellm_core_utils.prompt_templates.factory load_dotenv() from unittest.mock import MagicMock -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import pytest import litellm @@ -33,7 +28,6 @@ def test_completion_pydantic_obj_2(): from litellm.llms.custom_httpx.http_handler import HTTPHandler - litellm.set_verbose = True class CalendarEvent(BaseModel): name: str @@ -259,7 +253,6 @@ def test_vertex_tool_type_field_removal(): def test_function_calling_with_gemini(): from litellm.llms.custom_httpx.http_handler import HTTPHandler - litellm.set_verbose = True client = HTTPHandler() with patch.object(client, "post", new=MagicMock()) as mock_post: try: @@ -310,7 +303,6 @@ def test_function_calling_with_gemini(): def test_multiple_function_call(): - litellm.set_verbose = True from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() @@ -420,7 +412,6 @@ def test_multiple_function_call(): def test_multiple_function_call_changed_text_pos(): - litellm.set_verbose = True from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() @@ -528,7 +519,6 @@ def test_multiple_function_call_changed_text_pos(): def test_function_calling_with_gemini_multiple_results(): - litellm.set_verbose = True from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() @@ -1103,7 +1093,6 @@ def test_logprobs_unit_test(): def test_logprobs(): - litellm.set_verbose = True from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 813264c1feb..cc923f05831 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1,14 +1,9 @@ -import os -import sys from unittest.mock import patch import pytest from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.llms.vertex_ai.common_utils import ( _get_vertex_url, @@ -33,7 +28,7 @@ def test_validate_vertex_location_accepts_valid(location): ["attacker.example/", "evil.com#", "us.attacker.example", "us/../..", "US", "us_central1", "-us", "", None], ) def test_validate_vertex_location_rejects_invalid(location): - with pytest.raises(ValueError, match="vertex_location is required|Invalid vertex_location format"): + with pytest.raises(ValueError, match=r"vertex_location is required|Invalid vertex_location format"): validate_vertex_location(location) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py b/tests/test_litellm/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py index e0eccad80e2..55493d47f3d 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py @@ -3,8 +3,6 @@ Split from test_vertex.py to satisfy CI per-file size limits. """ import asyncio -import os -import sys import time from dotenv import load_dotenv @@ -16,7 +14,6 @@ import pytest import litellm from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.abspath("../..")) from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_image_generation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_image_generation.py index 2a87d84e20f..84444690fa2 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_image_generation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_image_generation.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.vertex_ai.image_generation.image_generation_handler import ( diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py index 18fc239b7c6..29d22e844a5 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py @@ -1,16 +1,11 @@ import asyncio import json -import os -import sys from unittest.mock import MagicMock, call, patch import pytest from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.vertex_ai.vertex_ai_aws_wif import VertexAIAwsWifAuth diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 1e5ae05aa25..05da22a73fd 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,13 +1,8 @@ -import os -import sys from unittest.mock import MagicMock, Mock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.vertex_ai.text_to_speech.transformation import ( diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py index b1aa7f629d5..fa286f6f609 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py @@ -6,15 +6,10 @@ Vertex AI Anthropic models don't support URL sources for images. LiteLLM should convert image URLs to base64 when using Vertex AI Anthropic. """ -import os -import sys from unittest.mock import patch, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index ac2368130d8..552ca98441f 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, ) diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py index 7c61aba4f99..957d7475d91 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py @@ -13,14 +13,10 @@ These tests verify that: import json import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py index f617a8db850..6255394d838 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py @@ -1,13 +1,9 @@ import json import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformation import ( diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py index 242a89d729a..3bca51ec6b3 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py @@ -1,14 +1,9 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path from litellm.llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( VertexAILlama3Config, diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py index 5a86325b7fd..4a11c84a96d 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py @@ -8,14 +8,10 @@ These tests verify that: """ import os -import sys from unittest.mock import MagicMock, patch, AsyncMock import pytest -sys.path.insert( - 0, os.path.abspath("../../../../../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py index 7922331d19f..d42bf7b7a1c 100644 --- a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py +++ b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py @@ -2,15 +2,12 @@ Tests for Volcengine Responses API transformation. """ -import os -import sys from typing import List, Literal, Optional, Union import httpx import pytest from pydantic import BaseModel, Field -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.volcengine.responses.transformation import ( diff --git a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py index 1670dac0e9d..0122bc50695 100644 --- a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py +++ b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py @@ -3,13 +3,10 @@ Integration tests for Volcengine embedding following LiteLLM testing patterns Based on the BaseLLMEmbeddingTest framework """ -import os -import sys from unittest.mock import MagicMock, patch import pytest # Add parent directory to path for imports -sys.path.insert(0, os.path.abspath("../../../../..")) from tests.llm_translation.base_embedding_unit_tests import BaseLLMEmbeddingTest import litellm @@ -31,7 +28,6 @@ class TestVolcEngineEmbedding(BaseLLMEmbeddingTest): @pytest.mark.parametrize("sync_mode", [True, False]) async def test_basic_embedding(self, sync_mode): """Test basic embedding functionality with realistic response""" - litellm.set_verbose = True embedding_call_args = self.get_base_embedding_call_args() # Mock the embedding functions to avoid actual API calls diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index ef7bb0e44f0..a5d1eccebe0 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -5,12 +5,7 @@ These tests validate the WandbInferenceConfig class which extends OpenAIGPTConfi Nebius AI Studio is an OpenAI-compatible provider with minor customizations. """ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index 6ff53287e9d..e269e782061 100644 --- a/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -5,13 +5,10 @@ Validates that litellm.transcription transforms requests correctly for WatsonX. """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.watsonx.audio_transcription.transformation import ( diff --git a/tests/test_litellm/llms/watsonx/embed/test_watsonx_embedding_transformation.py b/tests/test_litellm/llms/watsonx/embed/test_watsonx_embedding_transformation.py index 5c2688620d4..58f6bb23498 100644 --- a/tests/test_litellm/llms/watsonx/embed/test_watsonx_embedding_transformation.py +++ b/tests/test_litellm/llms/watsonx/embed/test_watsonx_embedding_transformation.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../../..")) import pytest diff --git a/tests/test_litellm/llms/watsonx/passthrough/test_watsonx_passthrough_transformation.py b/tests/test_litellm/llms/watsonx/passthrough/test_watsonx_passthrough_transformation.py index d1db04f5215..d8976d19f5c 100644 --- a/tests/test_litellm/llms/watsonx/passthrough/test_watsonx_passthrough_transformation.py +++ b/tests/test_litellm/llms/watsonx/passthrough/test_watsonx_passthrough_transformation.py @@ -5,14 +5,11 @@ Tests the Watsonx-specific passthrough configuration including URL construction, streaming detection, and authentication handling. """ -import os -import sys from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import litellm from litellm.llms.watsonx.passthrough.transformation import WatsonxPassthroughConfig diff --git a/tests/test_litellm/llms/watsonx/test_watsonx.py b/tests/test_litellm/llms/watsonx/test_watsonx.py index 315ffdb45a9..8ac4472b22d 100644 --- a/tests/test_litellm/llms/watsonx/test_watsonx.py +++ b/tests/test_litellm/llms/watsonx/test_watsonx.py @@ -1,10 +1,5 @@ import json -import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from typing import Optional from unittest.mock import Mock, patch diff --git a/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py b/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py index be74dc40eda..ffc48ecfae9 100644 --- a/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py +++ b/tests/test_litellm/llms/watsonx/test_watsonx_common_utils.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import MagicMock, call, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.llms.watsonx.common_utils import generate_iam_token diff --git a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py index 871613c9c9a..befd4c5ffbd 100644 --- a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py @@ -7,11 +7,8 @@ transformations for the Responses API. Source: litellm/llms/xai/responses/transformation.py """ -import os -import sys from unittest.mock import MagicMock -sys.path.insert(0, os.path.abspath("../../../../..")) import pytest diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py index eac5b89e4f3..e5e853ec82f 100644 --- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py +++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py @@ -1,9 +1,4 @@ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py index b3855202ae0..55e28dff81d 100644 --- a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py +++ b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py @@ -4,7 +4,6 @@ Test suite for XAI cost calculation functionality. import math import os -import sys import litellm from litellm.types.utils import ( @@ -13,9 +12,6 @@ from litellm.types.utils import ( Usage, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, diff --git a/tests/test_litellm/llms/xai/test_xai_key_fallback.py b/tests/test_litellm/llms/xai/test_xai_key_fallback.py index ec3eb83309c..092e4951547 100644 --- a/tests/test_litellm/llms/xai/test_xai_key_fallback.py +++ b/tests/test_litellm/llms/xai/test_xai_key_fallback.py @@ -1,10 +1,5 @@ import asyncio -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/llms/xai/xai_responses/test_transformation.py b/tests/test_litellm/llms/xai/xai_responses/test_transformation.py index dc535cf709b..c783918ca06 100644 --- a/tests/test_litellm/llms/xai/xai_responses/test_transformation.py +++ b/tests/test_litellm/llms/xai/xai_responses/test_transformation.py @@ -7,10 +7,7 @@ transformations for the Responses API. Source: litellm/llms/xai/responses/transformation.py """ -import sys -import os -sys.path.insert(0, os.path.abspath("../../../../..")) import pytest from litellm.types.utils import LlmProviders diff --git a/tests/test_litellm/llms/you_com/test_you_com_search.py b/tests/test_litellm/llms/you_com/test_you_com_search.py index eacc495cede..13d1be6062f 100644 --- a/tests/test_litellm/llms/you_com/test_you_com_search.py +++ b/tests/test_litellm/llms/you_com/test_you_com_search.py @@ -2,12 +2,9 @@ Tests for You.com Search API integration. """ -import os -import sys import pytest from unittest.mock import AsyncMock, patch, MagicMock -sys.path.insert(0, os.path.abspath("../..")) import litellm diff --git a/tests/test_litellm/llms/zai/test_zai_provider.py b/tests/test_litellm/llms/zai/test_zai_provider.py index e8374f92a19..38ddac8d510 100644 --- a/tests/test_litellm/llms/zai/test_zai_provider.py +++ b/tests/test_litellm/llms/zai/test_zai_provider.py @@ -13,6 +13,12 @@ from litellm import completion from litellm.cost_calculator import cost_per_token +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + @pytest.fixture def zai_response(): """Mock response from Z.AI API""" @@ -51,12 +57,8 @@ def test_zai_in_provider_lists(): assert "zai" in litellm.provider_list -def test_zai_models_in_model_cost(): +def test_zai_models_in_model_cost(local_model_cost_map): """Test that ZAI models are in the model cost map""" - import os - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") zai_models = [ "zai/glm-4.7", @@ -75,12 +77,8 @@ def test_zai_models_in_model_cost(): assert litellm.model_cost[model]["litellm_provider"] == "zai" -def test_zai_glm46_cost_calculation(): +def test_zai_glm46_cost_calculation(local_model_cost_map): """Test the cost calculation for glm-4.6""" - import os - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") key = "zai/glm-4.6" info = litellm.model_cost[key] @@ -96,12 +94,8 @@ def test_zai_glm46_cost_calculation(): assert math.isclose(completion_cost, 2.2, rel_tol=1e-6) -def test_zai_flash_model_is_free(): +def test_zai_flash_model_is_free(local_model_cost_map): """Test that glm-4.5-flash has zero cost""" - import os - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") key = "zai/glm-4.5-flash" info = litellm.model_cost[key] @@ -110,12 +104,8 @@ def test_zai_flash_model_is_free(): assert info["output_cost_per_token"] == 0 -def test_glm47_supports_reasoning(): +def test_glm47_supports_reasoning(local_model_cost_map): """Test that GLM-4.7 supports reasoning""" - import os - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") key = "zai/glm-4.7" assert key in litellm.model_cost, f"Model {key} not found in model_cost" @@ -124,12 +114,8 @@ def test_glm47_supports_reasoning(): assert info["supports_reasoning"] is True -def test_glm47_cost_calculation(): +def test_glm47_cost_calculation(local_model_cost_map): """Test cost calculation for GLM-4.7""" - import os - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") prompt_cost, completion_cost = cost_per_token( model="zai/glm-4.7", @@ -146,7 +132,7 @@ def test_glm47_cost_calculation(): async def test_zai_completion_call(respx_mock, zai_response, monkeypatch): """Test completion call with zai provider using mocked response""" monkeypatch.setenv("ZAI_API_KEY", "test-api-key") - litellm.disable_aiohttp_transport = True + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond( json=zai_response @@ -172,7 +158,7 @@ async def test_zai_completion_call(respx_mock, zai_response, monkeypatch): def test_zai_sync_completion(respx_mock, zai_response, monkeypatch): """Test synchronous completion call""" monkeypatch.setenv("ZAI_API_KEY", "test-api-key") - litellm.disable_aiohttp_transport = True + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond( json=zai_response diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index e43e4be8bcc..b8f265ad7ea 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -9,9 +7,6 @@ from fastapi.testclient import TestClient from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 418059be835..697c9b018ec 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -1,7 +1,6 @@ import contextlib import json import os -import sys from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -9,7 +8,6 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path from starlette.datastructures import Headers @@ -6240,7 +6238,6 @@ class TestAggregateGatewayDcrChallenge: well_known_root_suffix), so a DCR client behind a sub-path is pointed at a route that exists instead of a 404. Regression: the challenge used to hard-code /mcp and omit the root path the route inserts.""" - import os with ( patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), @@ -6314,6 +6311,48 @@ class TestAggregateGatewayDcrChallenge: www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] assert www_authenticate == f'Bearer resource_metadata="http://testserver{expected_metadata_path}"' + async def test_per_server_challenge_keeps_spelling_under_server_root_path(self): + """On a sub-path deployment the challenge must still advertise the spelling the client + used. ``_original_path`` is a raw request-line path, so under SERVER_ROOT_PATH it reads + ``/litellm/{server}/mcp``; matching that against the root-relative ``/{server}/mcp`` shape + used to fail, silently pointing a legacy-spelling client at the standard-pattern document + whose ``resource`` is ``{base}/mcp/{server}`` rather than the ``{base}/{server}/mcp`` URL it + called, which a strict RFC 9728 section 3 client rejects.""" + + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="gh-id", + name="github", + server_name="github", + url="https://upstream.example/mcp", + transport="http", + auth_type=MCPAuth.oauth2, + ) + for original_path, expected_metadata_path in ( + ("/litellm/mcp/github", "/litellm/.well-known/oauth-protected-resource/litellm/mcp/github"), + ("/litellm/github/mcp", "/litellm/.well-known/oauth-protected-resource/litellm/github/mcp"), + ): + scope = { + **self._scope(path="/mcp/github"), + "root_path": "/litellm", + "_original_path": original_path, + } + with ( + patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), + patch(self._AUTH_PATCH_TARGET, side_effect=self._auth_401()), + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = server + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(scope) + assert exc_info.value.status_code == 401 + www_authenticate = (exc_info.value.headers or {})["WWW-Authenticate"] + assert www_authenticate == f'Bearer resource_metadata="http://testserver{expected_metadata_path}"' + async def test_no_per_server_challenge_for_non_gateway_managed_targets(self): """The per-server challenge fires only for the server set the gateway's keyless flow serves: an OBO server and a multi-server CSV path keep the original admission error @@ -6387,9 +6426,7 @@ class TestAggregateGatewayDcrChallenge: assert _gateway_dcr_challenge_target("/mcp/srv", None, None) == expected, resolved assert _gateway_dcr_challenge_target("/mcp/a,b", None, None) is None assert _gateway_dcr_challenge_target("/mcp", None, None) is None - with patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr: + with patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr: mock_mgr.get_mcp_server_by_name.return_value = _server(MCPAuth.oauth2) assert _gateway_dcr_challenge_target("/mcp/srv", ["other"], None) is None @@ -7030,25 +7067,109 @@ class TestUserSubjectTeamUnion: assert await manager.operator_open_server_ids(admitted) == {"srv-byom"} assert await manager.operator_open_server_ids(scoped_key) == set(), "explicit key scope still suppresses BYOM" - async def test_admitted_admin_is_scoped_to_grants_not_full_registry(self): - """The wrapper's admin short-circuit hands the FULL registry to any admin-role auth before - the grant union or the per-team org ceilings run. A session bearer is a third-party client - credential, not the dashboard: an admin signing in through the connect flow gets their - grants like anyone else. A real admin key keeps the dashboard behavior unchanged.""" + @pytest.mark.parametrize( + "role", ["PROXY_ADMIN", "PROXY_ADMIN_VIEW_ONLY"], ids=["proxy_admin", "proxy_admin_view_only"] + ) + async def test_admitted_admin_gets_registry_like_an_admin_key(self, role): + """Connect-page parity: admin view rides the HUMAN, not the credential. An admitted session + subject with an admin-view role resolves the same full registry an admin KEY does, so the + servers the dashboard shows an admin are the servers their OAuth session serves. Regression + pin for the customer report where an admin's Claude Code session showed zero tools.""" + from litellm.proxy._types import LitellmUserRoles + + manager = self._manager_with(["srv-granted", "srv-secret"]) + admitted = _make_admitted_subject("admin-user") + admitted.user_role = LitellmUserRoles[role] + key_admin = UserAPIKeyAuth(user_id="admin-user", api_key="sk-hash", user_role=LitellmUserRoles[role]) + with patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=["srv-granted"])): + admitted_view = set(await manager.get_allowed_mcp_servers(admitted)) + key_admin_view = set(await manager.get_allowed_mcp_servers(key_admin)) + assert admitted_view == {"srv-granted", "srv-secret"}, "an admitted admin resolves the registry" + assert key_admin_view == admitted_view, "session and key admin views must be identical" + + async def test_admitted_admin_explicit_scope_still_wins(self): + """An admin whose own user row names servers is entitlement-bound whatever their role: the + row binds through the ceiling for an admitted subject (a user row's mcp_servers is the + human's grant list, not a credential scope), so the registry seed must not fire. A KEY + carrying an explicit scope disqualifies directly, empty list included.""" + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles + + manager = self._manager_with(["srv-granted", "srv-secret"]) + admitted = _make_admitted_subject("admin-user", own_servers=["srv-granted"]) + admitted.user_role = LitellmUserRoles.PROXY_ADMIN + with ( + patch.object( + MCPRequestHandler, "_get_allowed_mcp_servers_for_user", AsyncMock(return_value=["srv-granted"]) + ), + patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=["srv-granted"])), + ): + assert set(await manager.get_allowed_mcp_servers(admitted)) == {"srv-granted"} + + scoped_key = UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-hash", + user_role=LitellmUserRoles.PROXY_ADMIN, + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-k", mcp_servers=[]), + ) + with patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=[])): + assert await manager.get_allowed_mcp_servers(scoped_key) == [] + + async def test_admitted_admin_db_default_empty_scope_still_gets_registry(self): + """The admitted subject's object_permission is the user's own row, whose mcp_servers column + is [] by DB default whenever the row exists for any other field: default noise, never an + explicit scope. The registry seed must fire through it, or every admin with a shared + permission row keeps resolving zero servers while their dashboard shows all of them.""" + from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles + + manager = self._manager_with(["srv-granted", "srv-secret"]) + admitted = _make_admitted_subject("admin-user") + admitted.user_role = LitellmUserRoles.PROXY_ADMIN + admitted.object_permission = LiteLLM_ObjectPermissionTable(object_permission_id="op-u", mcp_servers=[]) + with patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=[])): + assert set(await manager.get_allowed_mcp_servers(admitted)) == {"srv-granted", "srv-secret"} + + async def test_non_admin_admitted_subject_never_gets_registry(self): + """The negative control for the registry seed: a plain admitted subject with no admin-view + role resolves only their grant union, however many servers the registry holds.""" + manager = self._manager_with(["srv-granted", "srv-secret"]) + plain = _make_admitted_subject("plain-user") + with patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=["srv-granted"])): + assert set(await manager.get_allowed_mcp_servers(plain)) == {"srv-granted"} + + async def test_admitted_admin_entitlement_ceiling_disables_registry(self): + """An entitlement ceiling, including an UNRESOLVED one, binds the human whatever their role: + the registry seed must not fire on a transient fault, and the grant union answers instead.""" from litellm.proxy._types import LitellmUserRoles manager = self._manager_with(["srv-granted", "srv-secret"]) admitted = _make_admitted_subject("admin-user") admitted.user_role = LitellmUserRoles.PROXY_ADMIN - with patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=["srv-granted"])): - admitted_view = set(await manager.get_allowed_mcp_servers(admitted)) - key_admin_view = set( - await manager.get_allowed_mcp_servers( - UserAPIKeyAuth(user_id="admin-user", api_key="sk-hash", user_role=LitellmUserRoles.PROXY_ADMIN) - ) - ) - assert admitted_view == {"srv-granted"}, "an admitted admin gets their grants, not the registry" - assert key_admin_view == {"srv-granted", "srv-secret"}, "admin KEY behavior must be unchanged" + with ( + patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_user", AsyncMock(return_value=None)), + patch.object(MCPRequestHandler, "get_allowed_mcp_servers", AsyncMock(return_value=["srv-granted"])), + ): + assert set(await manager.get_allowed_mcp_servers(admitted)) == {"srv-granted"} + + async def test_admitted_admin_tools_ride_own_source_on_ungranted_server(self): + """Admin view is an open channel on the tools axis too: the user's OWN source resolves the + tools for a server no grant names, so an admin session's registry-wide servers are invokable + rather than listable-but-uninvokable. A non-admin subject on the same server stays denied. + An admin whose row carries any entitlement never reaches this channel: the ceiling clause + disqualifies the predicate first, so their own tool permissions keep binding on the grants path.""" + from litellm.proxy._types import LitellmUserRoles + + admin = _make_admitted_subject("admin-user") + admin.user_role = LitellmUserRoles.PROXY_ADMIN + plain = _make_admitted_subject("plain-user") + with self._patch(teams_by_id={}, user_teams=[]): + with patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.operator_open_server_ids", + AsyncMock(return_value=set()), + ): + admin_tools = await MCPRequestHandler.get_allowed_tools_for_server("srv-any", admin) + plain_tools = await MCPRequestHandler.get_allowed_tools_for_server("srv-any", plain) + assert admin_tools is None, "admin channel resolves allow-all through the user's own source" + assert plain_tools == [], "a non-admin subject with no granting source stays denied" async def test_admitted_opt_out_via_wrapper_keeps_team_servers(self): """The wrapper's no_mcp_servers early-return is a KEY rule (a scoped credential's opt-out is diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index cac879f3ad2..bcac27a4a14 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -2957,6 +2957,55 @@ def test_validate_trusted_redirect_uri_logs_diagnostic_on_rejection(caplog, monk assert "X-Forwarded-Host" in msg +@pytest.mark.parametrize( + "direct_ip,expect_accepted", + [ + ("10.0.0.7", True), + ("203.0.113.5", False), + ], +) +def test_validate_trusted_redirect_uri_follows_the_xff_trust_gate(direct_ip, expect_accepted, monkeypatch): + try: + from fastapi import HTTPException, Request + + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + validate_trusted_redirect_uri, + ) + except ImportError: + pytest.skip("MCP oauth_utils not available") + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.delenv("MCP_TRUSTED_REDIRECT_ORIGINS", raising=False) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "http://localhost:4000/" + mock_request.client = MagicMock() + mock_request.client.host = direct_ip + + headers = { + "X-Forwarded-Proto": "https", + "X-Forwarded-Host": "proxy.example.com", + } + mock_request.headers.get = lambda name, default=None: headers.get(name, default) + mock_request.headers.__contains__ = lambda self_, name: name in headers + + redirect_uri = "https://proxy.example.com/callback" + general_settings = { + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + } + + with patch("litellm.proxy.proxy_server.general_settings", general_settings, create=True): + if expect_accepted: + validate_trusted_redirect_uri(mock_request, redirect_uri) + return + with pytest.raises(HTTPException) as exc_info: + validate_trusted_redirect_uri(mock_request, redirect_uri) + + assert exc_info.value.status_code == 400 + assert "proxy.example.com" in str(exc_info.value.detail) + + @pytest.mark.parametrize( "bad_value", [ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py index 4b9e7f2258b..5357e0dce9e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_cost_calculator.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import orjson @@ -8,9 +6,6 @@ import pytest from fastapi import Request from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy._experimental.mcp_server.cost_calculator import MCPCostCalculator diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_custom_fields.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_custom_fields.py index 7a096fdc899..333d4c98899 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_custom_fields.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_custom_fields.py @@ -5,13 +5,10 @@ Tests that mcp_info can accept arbitrary custom fields in addition to predefined """ import pytest -import sys -import os from unittest.mock import Mock, patch from typing import Dict, Any # Add the path to find the modules -sys.path.insert(0, os.path.abspath("../../../..")) # Adjust the path as needed from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPAuth diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py index 9a741a3f861..43cf35c152d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py @@ -1,12 +1,8 @@ import json import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path class TestMCPRegistryFile: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py index 3182318caed..5a24ca00c25 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_metadata_preservation.py @@ -5,12 +5,10 @@ This module tests that tool metadata is preserved when creating prefixed tools, which is critical for ChatGPT UI widget rendering. """ -import sys import pytest # Add the parent directory to the path so we can import litellm -sys.path.insert(0, "../../../../../") from mcp.types import Tool as MCPTool diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py index fe583ace897..1c59b7b87e0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py @@ -8,7 +8,6 @@ Covers: """ import asyncio -import sys import time from unittest.mock import AsyncMock, MagicMock, patch @@ -16,7 +15,6 @@ import httpx import pytest from fastapi import HTTPException, Request -sys.path.insert(0, "../../../../../") from litellm.proxy._experimental.mcp_server import discoverable_endpoints diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py index f25d3baea0a..67663448d65 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_cold_start.py @@ -1,10 +1,8 @@ """Unit tests for MCP OAuth passthrough cold-start route behavior.""" -import sys import pytest -sys.path.insert(0, "../../../../../") from litellm.proxy._types import MCPTransport from litellm.types.mcp import MCPAuth diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py index 095ae00fd45..6d66748bf3f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -1,12 +1,10 @@ """Unit tests for MCP OAuth passthrough tool-fetch behavior.""" -import sys from unittest.mock import AsyncMock, MagicMock import httpx import pytest -sys.path.insert(0, "../../../../../") from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index aa6e63b7ca0..82f74cda835 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1,5 +1,6 @@ import asyncio import contextvars +import os from datetime import datetime, timedelta from unittest.mock import AsyncMock, MagicMock, patch @@ -8094,6 +8095,54 @@ class TestPreemptive401ModeAware: assert exc.value.status_code == 401 assert "www-authenticate" in {k.lower() for k in exc.value.headers} + @pytest.mark.asyncio + @pytest.mark.parametrize( + "original_path, expected_as_path", + ( + ("/litellm/mcp/interactive", "/litellm/.well-known/oauth-authorization-server/litellm/mcp/interactive"), + ("/litellm/interactive/mcp", "/litellm/.well-known/oauth-authorization-server/litellm/interactive"), + ), + ) + async def test_gateway_as_metadata_challenge_under_server_root_path(self, original_path, expected_as_path): + """Under SERVER_ROOT_PATH the challenge must keep the spelling the client called and point at + a route the proxy registered, so it has to compare a route-relative path and carry the root suffix.""" + from litellm.proxy._experimental.mcp_server import server as server_module + + server = _make_oauth2_server("interactive", oauth2_flow="authorization_code") + scope = { + **self._scope(server.alias), + "root_path": "/litellm", + "_original_path": original_path, + "headers": [(b"host", b"testserver")], + } + with ( + patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), + patch.object( + server_module.global_mcp_server_manager, + "get_mcp_server_by_name", + return_value=server, + ), + patch.object( + server_module.global_mcp_server_manager, + "has_user_oauth_token", + new_callable=AsyncMock, + return_value=False, + ), + pytest.raises(HTTPException) as exc, + ): + await server_module._raise_preemptive_401_for_unauthenticated_servers( + scope=scope, + mcp_servers=[server.alias], + oauth2_headers=None, + mcp_server_auth_headers=None, + user_api_key_auth=UserAPIKeyAuth(api_key="sk-litellm-virtual-key"), + client_ip=None, + ) + + assert exc.value.status_code == 401 + headers = {k.lower(): v for k, v in (exc.value.headers or {}).items()} + assert headers["www-authenticate"] == f'Bearer authorization_uri="http://testserver{expected_as_path}"' + @pytest.mark.asyncio async def test_gateway_managed_interactive_no_token_challenges_with_authorization_bearer(self): """The bug fix: no stored token, key in Authorization (oauth2_headers diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 5ee8143fb8e..cdea803ebf3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -18,7 +18,6 @@ from litellm.proxy._experimental.mcp_server.exceptions import ( from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListFault # Add the parent directory to the path so we can import litellm -sys.path.insert(0, "../../../../../") import httpx @@ -4846,7 +4845,9 @@ class TestMCPServerManager: @staticmethod def _manager_with_deepwiki_and_huggingface() -> MCPServerManager: manager = MCPServerManager() - deepwiki = MCPServer(server_id="deepwiki-id", name="deepwiki", server_name="deepwiki", transport=MCPTransport.http) + deepwiki = MCPServer( + server_id="deepwiki-id", name="deepwiki", server_name="deepwiki", transport=MCPTransport.http + ) huggingface = MCPServer( server_id="huggingface-id", name="huggingface", server_name="huggingface", transport=MCPTransport.http ) @@ -4867,8 +4868,14 @@ class TestMCPServerManager: with pytest.raises(ValueError, match="Tool hub_repo_search not found"): manager._resolve_mcp_server_for_tool_call("deepwiki", "hub_repo_search") - assert manager._resolve_mcp_server_for_tool_call("deepwiki", "read_wiki_structure") is manager.registry["deepwiki-id"] - assert manager._resolve_mcp_server_for_tool_call("huggingface", "hub_repo_search") is manager.registry["huggingface-id"] + assert ( + manager._resolve_mcp_server_for_tool_call("deepwiki", "read_wiki_structure") + is manager.registry["deepwiki-id"] + ) + assert ( + manager._resolve_mcp_server_for_tool_call("huggingface", "hub_repo_search") + is manager.registry["huggingface-id"] + ) def test_get_mcp_server_from_tool_name_rejects_other_servers_prefix(self): manager = self._manager_with_deepwiki_and_huggingface() @@ -4876,7 +4883,9 @@ class TestMCPServerManager: assert manager._get_mcp_server_from_tool_name("huggingface-read_wiki_structure") is None assert manager._get_mcp_server_from_tool_name("deepwiki-hub_repo_search") is None assert manager._get_mcp_server_from_tool_name("deepwiki-read_wiki_structure") is manager.registry["deepwiki-id"] - assert manager._get_mcp_server_from_tool_name("huggingface-hub_repo_search") is manager.registry["huggingface-id"] + assert ( + manager._get_mcp_server_from_tool_name("huggingface-hub_repo_search") is manager.registry["huggingface-id"] + ) def test_resolve_mcp_server_for_tool_call_shared_bare_name_resolves_via_own_prefixed_spelling(self): manager = MCPServerManager() @@ -10500,6 +10509,39 @@ class TestSessionResourceScopeIntersect: assert MCPServerManager._admitted_session_resource_scope(self._admitted_auth("b")) == "b" + @pytest.mark.asyncio + async def test_admin_registry_seed_still_bounded_by_session_resource_scope(self): + """The admin-view registry seed flows through the same scoped exit as every union: a + session envelope sealed to one server never widens past it, even held by an admin whose + role resolves the whole registry. Pin for the connect-page-parity change; without the + single-exit shape, the old early return would hand a per-server bearer the registry.""" + from unittest.mock import AsyncMock, patch + + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy._types import LitellmUserRoles + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + manager = MCPServerManager() + for sid in ("granted-id", "other-id"): + manager.registry[sid] = MCPServer( + server_id=sid, name=sid, server_name=sid, url="https://example.com/mcp", transport=MCPTransport.http + ) + auth = self._admitted_auth("granted-id") + auth.user_role = LitellmUserRoles.PROXY_ADMIN + with ( + patch.object(MCPServerManager, "get_allow_all_keys_server_ids", return_value=[]), + patch.object( + MCPServerManager, + "_get_active_submitted_mcp_server_ids_for_user", + new_callable=AsyncMock, + return_value=[], + ), + ): + assert await manager.get_allowed_mcp_servers(auth) == ["granted-id"] + auth.mcp_session_resource_server_id = None + assert set(await manager.get_allowed_mcp_servers(auth)) == {"granted-id", "other-id"} + @pytest.mark.asyncio async def test_get_allowed_mcp_servers_scopes_past_operator_open_union(self): """The intersect applies AFTER the operator-open (allow_all_keys) union, so a scoped @@ -10519,7 +10561,12 @@ class TestSessionResourceScopeIntersect: new_callable=AsyncMock, return_value=["granted-id", "other-id"], ), - patch.object(MCPServerManager, "_get_active_submitted_mcp_server_ids_for_user", new_callable=AsyncMock, return_value=[]), + patch.object( + MCPServerManager, + "_get_active_submitted_mcp_server_ids_for_user", + new_callable=AsyncMock, + return_value=[], + ), ): allowed = await manager.get_allowed_mcp_servers(auth) assert allowed == ["granted-id"] @@ -10531,7 +10578,12 @@ class TestSessionResourceScopeIntersect: new_callable=AsyncMock, side_effect=RuntimeError("resolver down"), ), - patch.object(MCPServerManager, "_get_active_submitted_mcp_server_ids_for_user", new_callable=AsyncMock, return_value=[]), + patch.object( + MCPServerManager, + "_get_active_submitted_mcp_server_ids_for_user", + new_callable=AsyncMock, + return_value=[], + ), ): fallback = await manager.get_allowed_mcp_servers(auth) assert fallback == ["granted-id"] @@ -10645,9 +10697,7 @@ class TestClientForwardedDiscoveryFailureIsNotFatal: @pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) @pytest.mark.asyncio - async def test_client_forwarded_servers_keep_discovering_their_front_door_endpoints( - self, auth_type: MCPAuthType - ): + async def test_client_forwarded_servers_keep_discovering_their_front_door_endpoints(self, auth_type: MCPAuthType): """Exempting these modes from the FAILURE must not exempt them from discovery itself. ``/authorize``, ``/token`` and ``/register`` read the discovered endpoints for these servers diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py index 0b211255218..054146d474d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py @@ -6,7 +6,6 @@ an ordered set of top K tools based on semantic similarity. """ import asyncio -import os import sys from unittest.mock import AsyncMock, Mock, patch @@ -15,7 +14,6 @@ import pytest if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 from exceptiongroup import BaseExceptionGroup -sys.path.insert(0, os.path.abspath("../..")) from mcp.types import Tool as MCPTool diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py index 066d33dc187..82528c58ae0 100644 --- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py +++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py @@ -4,14 +4,11 @@ Unit tests for AgentRequestHandler - Agent permission management for keys and te import hashlib import json -import os -import sys from typing import Final from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry diff --git a/tests/test_litellm/proxy/agent_endpoints/test_model_list_helpers.py b/tests/test_litellm/proxy/agent_endpoints/test_model_list_helpers.py index ccf5942c89d..939ab1cab40 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_model_list_helpers.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_model_list_helpers.py @@ -4,10 +4,7 @@ Test appending A2A agents to model lists. Maps to: litellm/proxy/agent_endpoints/model_list_helpers.py """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) from unittest.mock import AsyncMock, Mock, patch diff --git a/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py b/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py index 36f656a7adc..2309b0a931f 100644 --- a/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py +++ b/tests/test_litellm/proxy/auth/test_admin_viewer_handler_access.py @@ -11,15 +11,12 @@ The principle (see Admin Viewer role doc): anything Proxy Admin can read, Admin Viewer can read. No writes, no cost-incurring actions. """ -import os -import sys import types from unittest.mock import AsyncMock, MagicMock import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../")) import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 762d2cbf3c7..a34df54adfa 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys from types import SimpleNamespace from typing import TYPE_CHECKING, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -9,9 +7,6 @@ from unittest.mock import AsyncMock, MagicMock, patch if TYPE_CHECKING: from litellm.router import Router -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from datetime import datetime, timedelta, timezone diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 721857e5411..b0094b81112 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -26,9 +24,6 @@ from prisma.errors import ( UniqueViolationError, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm._logging import verbose_proxy_logger from litellm.exceptions import BudgetExceededError diff --git a/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py b/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py index 6b2d2babedc..e3b76cac8ce 100644 --- a/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py +++ b/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py @@ -18,15 +18,12 @@ NOTE: This test does NOT require proxy extras (apscheduler, etc.) because it tests at the auth_checks level, not the full proxy_server level. """ -import os -import sys import time from typing import Any, Dict, List, Optional from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache diff --git a/tests/test_litellm/proxy/auth/test_litellm_license.py b/tests/test_litellm/proxy/auth/test_litellm_license.py index 77dd45046a0..8da365cb587 100644 --- a/tests/test_litellm/proxy/auth/test_litellm_license.py +++ b/tests/test_litellm/proxy/auth/test_litellm_license.py @@ -1,12 +1,7 @@ import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy.auth.litellm_license import LicenseCheck diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index c589014f276..1c66acf8678 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -109,7 +109,7 @@ async def test_authenticate_user_admin_login_with_ui_credentials(): @pytest.mark.asyncio -async def test_authenticate_user_admin_login_with_master_key_as_password(): +async def test_authenticate_user_admin_login_with_master_key_as_password(monkeypatch): """Test admin login when UI_PASSWORD is not set, should use master_key""" master_key = "sk-1234" ui_username = "admin" @@ -131,39 +131,35 @@ async def test_authenticate_user_admin_login_with_master_key_as_password(): with patch.dict(os.environ, env_vars, clear=False): # Explicitly remove UI_PASSWORD if it exists - original_ui_password = os.environ.pop("UI_PASSWORD", None) - try: + monkeypatch.delenv("UI_PASSWORD", raising=False) + with patch( + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + ) as mock_generate_key: + mock_generate_key.return_value = { + "token": "test-token-123", + "user_id": LITELLM_PROXY_ADMIN_NAME, + } + with patch( - "litellm.proxy.auth.login_utils.generate_key_helper_fn", + "litellm.proxy.auth.login_utils.user_update", new_callable=AsyncMock, - ) as mock_generate_key: - mock_generate_key.return_value = { - "token": "test-token-123", - "user_id": LITELLM_PROXY_ADMIN_NAME, - } - + return_value=None, + ) as mock_user_update: with patch( - "litellm.proxy.auth.login_utils.user_update", - new_callable=AsyncMock, - return_value=None, - ) as mock_user_update: - with patch( - "litellm.proxy.auth.login_utils.get_secret_bool", - return_value=False, - ): - result = await authenticate_user( - username=ui_username, - password=master_key, - master_key=master_key, - prisma_client=mock_prisma_client, - ) + "litellm.proxy.auth.login_utils.get_secret_bool", + return_value=False, + ): + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + ) - assert isinstance(result, LoginResult) - assert result.user_id == LITELLM_PROXY_ADMIN_NAME - assert result.user_role == LitellmUserRoles.PROXY_ADMIN - finally: - if original_ui_password: - os.environ["UI_PASSWORD"] = original_ui_password + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME + assert result.user_role == LitellmUserRoles.PROXY_ADMIN @pytest.mark.asyncio @@ -319,7 +315,7 @@ async def test_authenticate_user_email_case_insensitive_login(): @pytest.mark.asyncio -async def test_authenticate_user_database_required_for_admin(): +async def test_authenticate_user_database_required_for_admin(monkeypatch): """Test that database is required for admin login""" master_key = "sk-1234" ui_username = "admin" @@ -353,7 +349,7 @@ async def test_authenticate_user_database_required_for_admin(): assert "No Database connected" in exc_info.value.message finally: if original_db_url: - os.environ["DATABASE_URL"] = original_db_url + monkeypatch.setenv("DATABASE_URL", original_db_url) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py b/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py index 2d81d48de1e..315fc1471b3 100644 --- a/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py +++ b/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py @@ -13,14 +13,11 @@ constructs a ``UserAPIKeyAuth`` from them. The fix has two parts: ``"proxy_admin"`` into ``LitellmUserRoles.PROXY_ADMIN``. """ -import os -import sys import pytest from fastapi import Request from starlette.datastructures import Headers -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import LitellmUserRoles from litellm.proxy.auth.oauth2_proxy_hook import ( diff --git a/tests/test_litellm/proxy/auth/test_object_permission_loading.py b/tests/test_litellm/proxy/auth/test_object_permission_loading.py index 0dfd82e0ea0..8db4e210107 100644 --- a/tests/test_litellm/proxy/auth/test_object_permission_loading.py +++ b/tests/test_litellm/proxy/auth/test_object_permission_loading.py @@ -2,13 +2,10 @@ Test that object_permission is automatically loaded when fetching keys and teams. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, diff --git a/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py b/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py index 45e24832274..3c8a793e957 100644 --- a/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py +++ b/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py @@ -10,14 +10,11 @@ organization's budget limit. """ import asyncio -import os -import sys from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../")) import litellm from litellm.proxy._types import ( diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index b3b73723726..2eab03c2947 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1,11 +1,7 @@ import os -import sys from datetime import datetime from unittest.mock import MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import pytest from fastapi import HTTPException, Request diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 043bbb5b76a..c1e235b77f6 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1,15 +1,10 @@ import asyncio import json -import os -import sys from contextlib import contextmanager from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import pytest from fastapi import status diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index 863d9204cff..b548b0b3135 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -29,8 +29,6 @@ added to this layer raises instead of silently passing - the inventory of seams cannot drift without a test failure. """ -import os -import sys from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Dict, Optional @@ -38,7 +36,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm import litellm.proxy.batches_endpoints.endpoints as endpoints diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py index c2858c84c6d..32dfb8d521d 100644 --- a/tests/test_litellm/proxy/client/cli/test_agents.py +++ b/tests/test_litellm/proxy/client/cli/test_agents.py @@ -8,9 +8,6 @@ import pytest import requests from click.testing import CliRunner -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy.client.cli.commands.agents import ( @@ -255,7 +252,7 @@ class TestRunAgent: assert calls["args"] == ("claude", "--resume") def test_missing_binary_raises_with_install_hint(self): - with pytest.raises(AgentRunError, match="claude.*Install it first"): + with pytest.raises(AgentRunError, match=r"claude.*Install it first"): run_agent( "http://localhost:4000", "sk-key", diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index be29269fe25..85a4d90abf9 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -1,12 +1,10 @@ import json import os import stat -import sys import time from pathlib import Path from unittest.mock import Mock, patch -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/proxy/client/cli/test_config_commands.py b/tests/test_litellm/proxy/client/cli/test_config_commands.py index 6f3f4e4b268..611307635e0 100644 --- a/tests/test_litellm/proxy/client/cli/test_config_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_config_commands.py @@ -1,14 +1,11 @@ import json -import os import stat -import sys from pathlib import Path from unittest.mock import patch import pytest from click.testing import CliRunner -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.client.cli import cli diff --git a/tests/test_litellm/proxy/client/cli/test_credentials_commands.py b/tests/test_litellm/proxy/client/cli/test_credentials_commands.py index c751bb675ce..fb9d749dd02 100644 --- a/tests/test_litellm/proxy/client/cli/test_credentials_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_credentials_commands.py @@ -1,15 +1,10 @@ import json -import os -import sys from unittest.mock import MagicMock import pytest import requests from click.testing import CliRunner -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy.client.cli.main import cli diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 9c6fc15b242..0dd388919a5 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,14 +1,12 @@ # stdlib imports import json import os -import sys from pathlib import Path from unittest.mock import Mock, patch import pytest from click.testing import CliRunner -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path import litellm.proxy.client.cli diff --git a/tests/test_litellm/proxy/client/cli/test_keys_commands.py b/tests/test_litellm/proxy/client/cli/test_keys_commands.py index 5d88b031eac..5cc0fb70881 100644 --- a/tests/test_litellm/proxy/client/cli/test_keys_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_keys_commands.py @@ -1,13 +1,9 @@ import json import os -import sys from unittest.mock import patch import requests -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import pytest diff --git a/tests/test_litellm/proxy/client/cli/test_models_commands.py b/tests/test_litellm/proxy/client/cli/test_models_commands.py index 7f47d14656a..80353955e7f 100644 --- a/tests/test_litellm/proxy/client/cli/test_models_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_models_commands.py @@ -1,7 +1,6 @@ # stdlib imports import json import os -import sys import time from unittest.mock import patch @@ -10,9 +9,6 @@ import pytest # third party imports from click.testing import CliRunner -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path # local imports diff --git a/tests/test_litellm/proxy/client/cli/test_users_commands.py b/tests/test_litellm/proxy/client/cli/test_users_commands.py index f18ceb30c22..72539173318 100644 --- a/tests/test_litellm/proxy/client/cli/test_users_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_users_commands.py @@ -1,13 +1,8 @@ -import os -import sys from unittest.mock import patch import pytest from click.testing import CliRunner -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy.client.cli import cli diff --git a/tests/test_litellm/proxy/client/test_client.py b/tests/test_litellm/proxy/client/test_client.py index b0e458da89e..fe3e2c52ce5 100644 --- a/tests/test_litellm/proxy/client/test_client.py +++ b/tests/test_litellm/proxy/client/test_client.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy.client import ChatClient, Client, ModelsManagementClient from litellm.proxy.client.http_client import HTTPClient diff --git a/tests/test_litellm/proxy/client/test_credentials.py b/tests/test_litellm/proxy/client/test_credentials.py index 72c643467b2..41886e3b292 100644 --- a/tests/test_litellm/proxy/client/test_credentials.py +++ b/tests/test_litellm/proxy/client/test_credentials.py @@ -1,12 +1,7 @@ -import os -import sys import pytest import requests -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import responses diff --git a/tests/test_litellm/proxy/client/test_http_client.py b/tests/test_litellm/proxy/client/test_http_client.py index 3d8fe44438a..c0f66b0f98e 100644 --- a/tests/test_litellm/proxy/client/test_http_client.py +++ b/tests/test_litellm/proxy/client/test_http_client.py @@ -1,15 +1,10 @@ """Tests for the HTTP client.""" import json -import os -import sys import pytest import requests -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import responses diff --git a/tests/test_litellm/proxy/client/test_http_commands.py b/tests/test_litellm/proxy/client/test_http_commands.py index 16579cfffbc..04894248ff2 100644 --- a/tests/test_litellm/proxy/client/test_http_commands.py +++ b/tests/test_litellm/proxy/client/test_http_commands.py @@ -1,15 +1,10 @@ """Tests for the HTTP command group.""" import json -import os -import sys import pytest from click.testing import CliRunner -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import responses diff --git a/tests/test_litellm/proxy/client/test_keys.py b/tests/test_litellm/proxy/client/test_keys.py index 620daefb39e..282b97b1c09 100644 --- a/tests/test_litellm/proxy/client/test_keys.py +++ b/tests/test_litellm/proxy/client/test_keys.py @@ -1,13 +1,8 @@ -import os -import sys import traceback import pytest import requests -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import responses diff --git a/tests/test_litellm/proxy/client/test_model_groups.py b/tests/test_litellm/proxy/client/test_model_groups.py index 1c87672e723..9ea8e94ff95 100644 --- a/tests/test_litellm/proxy/client/test_model_groups.py +++ b/tests/test_litellm/proxy/client/test_model_groups.py @@ -1,12 +1,7 @@ -import os -import sys import pytest import requests -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import responses diff --git a/tests/test_litellm/proxy/client/test_models.py b/tests/test_litellm/proxy/client/test_models.py index 33f963b74af..fe053ffd683 100644 --- a/tests/test_litellm/proxy/client/test_models.py +++ b/tests/test_litellm/proxy/client/test_models.py @@ -1,12 +1,7 @@ -import os -import sys import pytest import requests -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import responses diff --git a/tests/test_litellm/proxy/client/test_users.py b/tests/test_litellm/proxy/client/test_users.py index a48cf8f791b..87b8392e402 100644 --- a/tests/test_litellm/proxy/client/test_users.py +++ b/tests/test_litellm/proxy/client/test_users.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy.client.users import ( diff --git a/tests/test_litellm/proxy/common_utils/html_forms/test_native_client_consent.py b/tests/test_litellm/proxy/common_utils/html_forms/test_native_client_consent.py index 600e421c176..6464dd7899a 100644 --- a/tests/test_litellm/proxy/common_utils/html_forms/test_native_client_consent.py +++ b/tests/test_litellm/proxy/common_utils/html_forms/test_native_client_consent.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../")) from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.proxy.common_utils.html_forms.native_client_consent import render_native_client_consent_page diff --git a/tests/test_litellm/proxy/common_utils/html_forms/test_ui_login.py b/tests/test_litellm/proxy/common_utils/html_forms/test_ui_login.py index 436564d24a0..1d4261d278c 100644 --- a/tests/test_litellm/proxy/common_utils/html_forms/test_ui_login.py +++ b/tests/test_litellm/proxy/common_utils/html_forms/test_ui_login.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../")) from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index 77ada4c11a9..66f77db6da9 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -1,13 +1,9 @@ import copy import sys -import os from types import ModuleType, SimpleNamespace import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy.common_utils.callback_utils import ( add_guardrail_scan_id, diff --git a/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py b/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py index 3efeeee9a27..8623d93c0a3 100644 --- a/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py +++ b/tests/test_litellm/proxy/common_utils/test_expired_ui_session_key_cleanup_manager.py @@ -2,15 +2,12 @@ Test expired UI session key cleanup manager functionality. """ -import os -import sys from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException, status -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.constants import ( EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME, diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 869d228d5a4..375c0d2640c 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import orjson @@ -8,9 +6,6 @@ import pytest from fastapi import Request from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py index d6e1d22fdde..dd6c1637cad 100644 --- a/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_e2e.py @@ -11,7 +11,6 @@ Covers the critical gaps: """ import os -import sys from datetime import datetime, timedelta, timezone from typing import cast from unittest.mock import AsyncMock, MagicMock, patch @@ -19,7 +18,6 @@ from uuid import uuid4 import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import ( GenerateKeyResponse, diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py index 3bc62d549b0..6103a40d6c7 100644 --- a/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py @@ -9,13 +9,10 @@ Bug Fixed: Key alias was not passed during auto-rotation, causing secrets to be created at a new location instead of updating in-place. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import ( GenerateKeyResponse, diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_lock.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_lock.py index c0b3611b2b4..27dc6ae6a5e 100644 --- a/tests/test_litellm/proxy/common_utils/test_key_rotation_lock.py +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_lock.py @@ -5,13 +5,10 @@ Verifies that PodLockManager is correctly used to prevent concurrent key rotation across multiple pods in a distributed deployment. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import LiteLLM_VerificationToken from litellm.proxy.common_utils.key_rotation_manager import KeyRotationManager diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py index 18432d106af..40a186a9059 100644 --- a/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_manager.py @@ -2,14 +2,11 @@ Test key rotation manager functionality """ -import os -import sys from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import ( GenerateKeyResponse, diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py index 051ddd2e78c..91055dbac9f 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py @@ -4,13 +4,10 @@ These tests focus on the helper itself — not on the proxy endpoint or Slack integration — so they can run without the full proxy stack. """ -import os -import sys from datetime import date, datetime, timezone from unittest.mock import MagicMock -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.proxy.common_utils.model_deprecation import ( diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 8233b0d3864..25c177a308d 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -1,6 +1,5 @@ import asyncio import json -import os import sys import types from datetime import datetime, timedelta, timezone @@ -12,7 +11,6 @@ import httpx import prisma import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.proxy._types import LiteLLM_VerificationToken from litellm.proxy.common_utils import reset_budget_job as reset_budget_job_module diff --git a/tests/test_litellm/proxy/common_utils/test_static_asset_utils.py b/tests/test_litellm/proxy/common_utils/test_static_asset_utils.py index 93f7ccc92c2..593158515a5 100644 --- a/tests/test_litellm/proxy/common_utils/test_static_asset_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_static_asset_utils.py @@ -7,11 +7,9 @@ arbitrary local image paths working while refusing non-image files like """ import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy.common_utils.static_asset_utils import ( detect_local_image_media_type, diff --git a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py index dc3917cb48e..0ae74ab6f59 100644 --- a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py @@ -1,13 +1,8 @@ -import os -import sys from datetime import datetime, time, timezone from zoneinfo import ZoneInfo import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.common_utils.timezone_utils import ( diff --git a/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py index e2fa1de6962..dcd8e6881bd 100644 --- a/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/credential_endpoints/test_endpoints.py @@ -1,13 +1,10 @@ """Tests for the credential management endpoints.""" -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py index f357d7fbea8..abb79458318 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_base_update_queue.py @@ -1,15 +1,10 @@ import asyncio import json -import os -import sys from unittest.mock import patch import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.constants import MAX_IN_MEMORY_QUEUE_FLUSH_COUNT from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index a55d4f0dcfd..a00815345aa 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -1,14 +1,9 @@ import asyncio import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.constants import MAX_SIZE_IN_MEMORY_QUEUE from litellm.proxy._types import ( diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py index 7a1ab60c547..ecd5c5f50c0 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_pod_lock_manager.py @@ -1,13 +1,10 @@ import json -import os -import sys from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.constants import DEFAULT_CRON_JOB_LOCK_TTL_SECONDS from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 3325893c5f6..fb0c994a476 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -1,13 +1,8 @@ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.proxy_server import ProxyStartupEvent diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py index 0ed5940dd75..43f1a820885 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys import pytest from fastapi.testclient import TestClient @@ -10,9 +8,6 @@ from litellm.constants import MAX_SIZE_IN_MEMORY_QUEUE from litellm.proxy._types import Litellm_EntityType, SpendUpdateQueueItem from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path @pytest.fixture diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_tool_discovery_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_tool_discovery_queue.py index defdb3834d8..e400ad16e84 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_tool_discovery_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_tool_discovery_queue.py @@ -2,12 +2,9 @@ Unit tests for ToolDiscoveryQueue. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, diff --git a/tests/test_litellm/proxy/db/mcp_server/test_db.py b/tests/test_litellm/proxy/db/mcp_server/test_db.py index 481d1a864c0..aa40ec0d76c 100644 --- a/tests/test_litellm/proxy/db/mcp_server/test_db.py +++ b/tests/test_litellm/proxy/db/mcp_server/test_db.py @@ -1,13 +1,8 @@ -import os -import sys from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy._experimental.mcp_server.db import get_mcp_servers_by_team diff --git a/tests/test_litellm/proxy/db/test_check_migration.py b/tests/test_litellm/proxy/db/test_check_migration.py index 5b182f03c4b..9e2f6a1089c 100644 --- a/tests/test_litellm/proxy/db/test_check_migration.py +++ b/tests/test_litellm/proxy/db/test_check_migration.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path def test_check_migration_out_of_sync(mocker): diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 4113d708196..76a80ac2651 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1,13 +1,8 @@ import asyncio import copy import json -import os import re -import sys -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from collections.abc import Callable diff --git a/tests/test_litellm/proxy/db/test_db_url_settings.py b/tests/test_litellm/proxy/db/test_db_url_settings.py index ee4cf7fbb05..0ceec49de12 100644 --- a/tests/test_litellm/proxy/db/test_db_url_settings.py +++ b/tests/test_litellm/proxy/db/test_db_url_settings.py @@ -504,7 +504,7 @@ def test_apply_to_env_rejects_pinned_sqlite_direct_url(monkeypatch): monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db") monkeypatch.setenv("DIRECT_URL", "sqlite:///data/litellm.db") - with pytest.raises(RuntimeError, match="DIRECT_URL.*sqlite"): + with pytest.raises(RuntimeError, match=r"DIRECT_URL.*sqlite"): _apply() @@ -514,7 +514,7 @@ def test_apply_to_env_rejects_pinned_non_postgres_reader(monkeypatch): "DATABASE_URL_READ_REPLICA", "mysql://u:p@reader.example.com:3306/db" ) - with pytest.raises(RuntimeError, match="DATABASE_URL_READ_REPLICA.*mysql"): + with pytest.raises(RuntimeError, match=r"DATABASE_URL_READ_REPLICA.*mysql"): _apply() diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py index 4c6315024dd..d80e3acb4b8 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler.py +++ b/tests/test_litellm/proxy/db/test_exception_handler.py @@ -1,6 +1,5 @@ import asyncio import json -import os import sys from unittest.mock import MagicMock, patch @@ -21,9 +20,6 @@ from prisma.errors import ( UniqueViolationError, ) -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm._logging import verbose_proxy_logger @@ -335,7 +331,6 @@ def test_is_database_service_unavailable_error_asyncpg(monkeypatch): """asyncpg connection/interface errors map to service-unavailable. asyncpg is not a hard dependency, so inject a stand-in module to exercise the branch deterministically regardless of the install environment.""" - import sys import types fake_asyncpg = types.ModuleType("asyncpg") diff --git a/tests/test_litellm/proxy/db/test_exception_handler_reconnect_retry.py b/tests/test_litellm/proxy/db/test_exception_handler_reconnect_retry.py index 0a25ed55e90..4286da23242 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler_reconnect_retry.py +++ b/tests/test_litellm/proxy/db/test_exception_handler_reconnect_retry.py @@ -8,15 +8,12 @@ LiteLLM 1.83.x and started emitting `db_exceptions` alerts on transient `httpx.ReadError` flaps that used to self-heal in 1.82.6. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import httpx import pytest from prisma.errors import ClientNotConnectedError, UniqueViolationError -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py index 395f17e85ef..b1ecbfeff8e 100644 --- a/tests/test_litellm/proxy/db/test_prisma_client.py +++ b/tests/test_litellm/proxy/db/test_prisma_client.py @@ -8,9 +8,6 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy.db.prisma_client import PrismaWrapper, should_update_prisma_schema diff --git a/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py b/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py index 95e794012ec..f3f742b2023 100644 --- a/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py +++ b/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py @@ -40,9 +40,6 @@ import pytest from prisma import Prisma as GeneratedPrisma from prisma.engine.errors import EngineConnectionError -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy.db.prisma_client import PrismaWrapper from litellm.proxy.utils import PrismaClient diff --git a/tests/test_litellm/proxy/db/test_prisma_self_heal.py b/tests/test_litellm/proxy/db/test_prisma_self_heal.py index cc47cf4a7e4..10a48941693 100644 --- a/tests/test_litellm/proxy/db/test_prisma_self_heal.py +++ b/tests/test_litellm/proxy/db/test_prisma_self_heal.py @@ -8,9 +8,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy.utils import PrismaClient, ProxyLogging diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py index 11ed63cf8f0..dcc0036ff04 100644 --- a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -7,7 +7,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) # NOTE: do NOT patch sys.modules["prisma"] file-wide via an autouse fixture. diff --git a/tests/test_litellm/proxy/db/test_tool_registry_writer.py b/tests/test_litellm/proxy/db/test_tool_registry_writer.py index 7bf1ffda4fe..6318e4422cf 100644 --- a/tests/test_litellm/proxy/db/test_tool_registry_writer.py +++ b/tests/test_litellm/proxy/db/test_tool_registry_writer.py @@ -3,14 +3,11 @@ Unit tests for tool_registry_writer.py — uses a mock prisma client that exposes litellm_tooltable.upsert / find_many / find_unique. """ -import os -import sys from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.db.tool_registry_writer import ( ToolPolicyRegistry, diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index 37f5e6046ca..f4da8c941a4 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -1,12 +1,10 @@ import os -import sys from unittest.mock import MagicMock, patch import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.discovery_endpoints.ui_discovery_endpoints import router from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry diff --git a/tests/test_litellm/proxy/experimental/mcp_server/test_tool_registry.py b/tests/test_litellm/proxy/experimental/mcp_server/test_tool_registry.py index d5ba9744c7d..9fc2e8744c1 100644 --- a/tests/test_litellm/proxy/experimental/mcp_server/test_tool_registry.py +++ b/tests/test_litellm/proxy/experimental/mcp_server/test_tool_registry.py @@ -1,12 +1,7 @@ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy._experimental.mcp_server.tool_registry import MCPToolRegistry diff --git a/tests/test_litellm/proxy/fine_tuning_endpoints/test_endpoints.py b/tests/test_litellm/proxy/fine_tuning_endpoints/test_endpoints.py index b54787bf428..7ed1a436cb6 100644 --- a/tests/test_litellm/proxy/fine_tuning_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/fine_tuning_endpoints/test_endpoints.py @@ -11,15 +11,12 @@ seam stayed untouched, so a guard that raises after the provider call would stil """ import base64 -import os -import sys from contextlib import ExitStack from dataclasses import dataclass from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from fastapi import Response diff --git a/tests/test_litellm/proxy/google_endpoints/test_endpoints.py b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py index f3518999f72..92001118e2c 100644 --- a/tests/test_litellm/proxy/google_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/google_endpoints/test_endpoints.py @@ -13,7 +13,6 @@ from starlette.requests import Request load_dotenv() -sys.path.insert(0, os.path.abspath("../../../..")) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/google_endpoints/test_google_api_endpoints.py b/tests/test_litellm/proxy/google_endpoints/test_google_api_endpoints.py index 99f587e87a3..e4cd7d9dfa8 100644 --- a/tests/test_litellm/proxy/google_endpoints/test_google_api_endpoints.py +++ b/tests/test_litellm/proxy/google_endpoints/test_google_api_endpoints.py @@ -3,15 +3,10 @@ Test to verify the Google GenAI proxy API endpoints """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path def _build_test_client(): diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/_cisco_ai_defense_test_utils.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/_cisco_ai_defense_test_utils.py index 4f29d83d4a5..f09135dd56d 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/_cisco_ai_defense_test_utils.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/_cisco_ai_defense_test_utils.py @@ -1,6 +1,5 @@ import json import os -import sys from contextlib import contextmanager from datetime import datetime from types import SimpleNamespace @@ -39,7 +38,6 @@ def _make_model_response_with_content(content: str) -> ModelResponse: ) -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import DualCache from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py index 62d25f1b9c0..be55ac47bde 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py @@ -4,14 +4,10 @@ Tests for the Content Filter Guardrail import json import os -import sys from unittest.mock import MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../") -) # Adds the parent directory to the system path from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_gdpr_policy_e2e.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_gdpr_policy_e2e.py index 238331b32c8..4af9bd99ed1 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_gdpr_policy_e2e.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_gdpr_policy_e2e.py @@ -3,12 +3,9 @@ End-to-end tests for GDPR Art. 32 EU PII Protection policy template Tests the complete policy with various EU PII patterns """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../")) from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py index c942e5fe820..d702b9e0116 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py @@ -4,11 +4,9 @@ Tests for content filter pattern loading from JSON import json import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../")) from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.patterns import ( PATTERN_CATEGORIES, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index 729dcb54309..112bc5e6e49 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -4,9 +4,7 @@ Test OpenAI Moderation Guardrail """ import os -import sys -sys.path.insert(0, os.path.abspath("../../../../../..")) from unittest.mock import MagicMock, patch diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 5d971bf1212..dd339d4e51f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -3,7 +3,6 @@ Unit tests for Bedrock Guardrails """ import json -import os import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -11,7 +10,6 @@ import httpx import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../../../..")) import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index ceb59571389..d842a1ee5f9 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -6,14 +6,11 @@ All Bedrock HTTP calls are mocked; no real AWS calls are made. import json import logging -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.exceptions import ModifyResponseException from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py index c23fbc0234e..d319d619ff7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py @@ -1,8 +1,6 @@ import asyncio import json -import os import ssl -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -18,15 +16,11 @@ from litellm.proxy.guardrails.guardrail_hooks.cato_networks.cato_networks import from litellm.proxy.proxy_server import UserAPIKeyAuth from litellm.types.utils import ModelResponse, ResponsesAPIResponse -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 def test_cato_guard_config(): - litellm.set_verbose = True litellm.guardrail_name_config_map = {} init_guardrails_v2( @@ -47,7 +41,6 @@ def test_cato_guard_config(): def test_cato_guard_config_no_api_key(monkeypatch): monkeypatch.delenv("CATO_API_KEY", raising=False) - litellm.set_verbose = True litellm.guardrail_name_config_map = {} with pytest.raises(CatoNetworksGuardrailMissingSecrets, match="Couldn't get Cato Networks api key"): init_guardrails_v2( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py index 8974a18593b..779075a40d9 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py @@ -44,7 +44,6 @@ from tests.test_litellm.proxy.guardrails.guardrail_hooks._cisco_ai_defense_test_ def test_cisco_ai_defense_config_via_init_v2_chat(monkeypatch): monkeypatch.setenv("CISCO_AI_DEFENSE_API_KEY", "test-key") - litellm.set_verbose = True litellm.guardrail_name_config_map = {} init_guardrails_v2( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_deepkeep.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_deepkeep.py index a2b8894910c..03f418e6d7a 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_deepkeep.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_deepkeep.py @@ -1,10 +1,8 @@ import os -import sys import pytest from unittest.mock import patch, MagicMock, AsyncMock from httpx import Response, Request -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.guardrails.guardrail_hooks.deepkeep.deepkeep import ( @@ -17,14 +15,13 @@ from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.exceptions import GuardrailRaisedException -def test_deepkeep_guard_config(): +def test_deepkeep_guard_config(monkeypatch: pytest.MonkeyPatch): """Test DeepKeep guard configuration with init_guardrails_v2.""" - litellm.set_verbose = True - litellm.guardrail_name_config_map = {} + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) - os.environ["DEEPKEEP_API_KEY"] = "test-key" - os.environ["DEEPKEEP_API_BASE"] = "https://test.deepkeep.ai" - os.environ["DEEPKEEP_FIREWALL_ID"] = "fw-123" + monkeypatch.setenv("DEEPKEEP_API_KEY", "test-key") + monkeypatch.setenv("DEEPKEEP_API_BASE", "https://test.deepkeep.ai") + monkeypatch.setenv("DEEPKEEP_FIREWALL_ID", "fw-123") init_guardrails_v2( all_guardrails=[ @@ -42,9 +39,6 @@ def test_deepkeep_guard_config(): ) # Clean up - del os.environ["DEEPKEEP_API_KEY"] - del os.environ["DEEPKEEP_API_BASE"] - del os.environ["DEEPKEEP_FIREWALL_ID"] class TestDeepKeepGuardrail: @@ -108,11 +102,11 @@ class TestDeepKeepGuardrail: == "https://test.deepkeep.ai/v3/openai/beta/litellm_basic_guardrail_api" ) - def test_initialization_with_env_vars(self): + def test_initialization_with_env_vars(self, monkeypatch: pytest.MonkeyPatch): """should initialize successfully using environment variables.""" - os.environ["DEEPKEEP_API_KEY"] = "env-key" - os.environ["DEEPKEEP_API_BASE"] = "https://env.deepkeep.ai" - os.environ["DEEPKEEP_FIREWALL_ID"] = "fw-env-456" + monkeypatch.setenv("DEEPKEEP_API_KEY", "env-key") + monkeypatch.setenv("DEEPKEEP_API_BASE", "https://env.deepkeep.ai") + monkeypatch.setenv("DEEPKEEP_FIREWALL_ID", "fw-env-456") guardrail = DeepKeepGuardrail( guardrail_name="deepkeep-env-test", diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py index c5b182a00ab..1b2108c837d 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_hiddenlayer.py @@ -1,5 +1,4 @@ import os -import sys import uuid from typing import List, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -8,7 +7,6 @@ import pytest from fastapi import HTTPException from httpx import Request, Response -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import ModelResponse @@ -26,13 +24,12 @@ from litellm.types.utils import ( ) -def test_hiddenlayer_config_saas(): +def test_hiddenlayer_config_saas(monkeypatch: pytest.MonkeyPatch): """Test Hiddenlayer SaaS configuration with init_guardrails_v2.""" - litellm.set_verbose = True - litellm.guardrail_name_config_map = {} + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) # Set environment variables for testing - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") init_guardrails_v2( all_guardrails=[ @@ -50,8 +47,6 @@ def test_hiddenlayer_config_saas(): ) # Clean up - if "HIDDENLAYER_API_BASE" in os.environ: - del os.environ["HIDDENLAYER_API_BASE"] class TestHiddenlayerGuardrail: @@ -71,9 +66,9 @@ class TestHiddenlayerGuardrail: if key in os.environ: del os.environ[key] - def test_initialization(self): + def test_initialization(self, monkeypatch: pytest.MonkeyPatch): """Test successful initialization with default values.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -84,19 +79,18 @@ class TestHiddenlayerGuardrail: assert guardrail.guardrail_name == "hiddenlayer" assert guardrail.event_hook == "pre_call" - def test_initialization_fails_when_api_key_missing(self): + def test_initialization_fails_when_api_key_missing(self, monkeypatch: pytest.MonkeyPatch): """Test that initialization fails when API key is not set.""" # Ensure API key is not set - if "HIDDENLAYER_CLIENT_SECRET" in os.environ: - del os.environ["HIDDENLAYER_CLIENT_SECRET"] + monkeypatch.delenv("HIDDENLAYER_CLIENT_SECRET", raising=False) with pytest.raises(RuntimeError): HiddenlayerGuardrail(guardrail_name="hiddenlayer", event_hook="pre_call") @pytest.mark.asyncio - async def test_apply_guardrail_request_no_violations(self): + async def test_apply_guardrail_request_no_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for request with no violations detected.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") # Setup guardrail guardrail = HiddenlayerGuardrail( @@ -151,9 +145,9 @@ class TestHiddenlayerGuardrail: assert call_args.args[0] == f"{guardrail.api_base}/detection/v1/interactions" @pytest.mark.asyncio - async def test_apply_guardrail_request_with_violations(self): + async def test_apply_guardrail_request_with_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for request with violations detected.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") # Setup guardrail guardrail = HiddenlayerGuardrail( @@ -209,9 +203,9 @@ class TestHiddenlayerGuardrail: assert "Blocked by Hiddenlayer" in str(exc_info.value.detail) @pytest.mark.asyncio - async def test_apply_guardrail_response_no_violations(self): + async def test_apply_guardrail_response_no_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response with no violations detected.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") # Setup guardrail guardrail = HiddenlayerGuardrail( @@ -279,10 +273,10 @@ class TestHiddenlayerGuardrail: mock_post.assert_called_once() @pytest.mark.asyncio - async def test_apply_guardrail_response_with_violations(self): + async def test_apply_guardrail_response_with_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response with violations detected.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") # Setup guardrail guardrail = HiddenlayerGuardrail( @@ -348,10 +342,10 @@ class TestHiddenlayerGuardrail: assert exc_info.value.status_code == 400 @pytest.mark.asyncio - async def test_apply_guardrail_api_error_handling(self): + async def test_apply_guardrail_api_error_handling(self, monkeypatch: pytest.MonkeyPatch): """Test handling of API errors in apply_guardrail.""" # Set required API key - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -391,10 +385,10 @@ class TestHiddenlayerGuardrail: assert result == inputs @pytest.mark.asyncio - async def test_validate_with_call_hiddenlayer_method(self): + async def test_validate_with_call_hiddenlayer_method(self, monkeypatch: pytest.MonkeyPatch): """Test the _validate_with_guard_server internal method.""" # Set required API key - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -433,9 +427,9 @@ class TestHiddenlayerGuardrail: ) @pytest.mark.asyncio - async def test_apply_guardrail_request_with_image(self): + async def test_apply_guardrail_request_with_image(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail sends multimodal content (image) to HiddenLayer v1.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -498,9 +492,9 @@ class TestHiddenlayerGuardrail: assert result is not None @pytest.mark.asyncio - async def test_apply_guardrail_redact_with_image_content(self): + async def test_apply_guardrail_redact_with_image_content(self, monkeypatch: pytest.MonkeyPatch): """Test that REDACT action with multimodal content extracts text properly into inputs['texts'].""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrail( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -570,12 +564,11 @@ class TestHiddenlayerGuardrail: assert config_model.__name__ == "HiddenlayerGuardrailConfigModel" -def test_hiddenlayer_config_v2(): +def test_hiddenlayer_config_v2(monkeypatch: pytest.MonkeyPatch): """Test HiddenLayer V2 configuration with init_guardrails_v2.""" - litellm.set_verbose = True - litellm.guardrail_name_config_map = {} + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") init_guardrails_v2( all_guardrails=[ @@ -593,8 +586,6 @@ def test_hiddenlayer_config_v2(): config_file_path="", ) - if "HIDDENLAYER_API_BASE" in os.environ: - del os.environ["HIDDENLAYER_API_BASE"] class TestHiddenlayerGuardrailV2: @@ -612,9 +603,9 @@ class TestHiddenlayerGuardrailV2: if key in os.environ: del os.environ[key] - def test_initialization(self): + def test_initialization(self, monkeypatch: pytest.MonkeyPatch): """Test successful initialization with default values.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -624,18 +615,17 @@ class TestHiddenlayerGuardrailV2: assert guardrail.guardrail_name == "hiddenlayer" assert guardrail.event_hook == "pre_call" - def test_initialization_fails_when_api_key_missing(self): + def test_initialization_fails_when_api_key_missing(self, monkeypatch: pytest.MonkeyPatch): """Test that initialization fails when API key is not set for SaaS.""" - if "HIDDENLAYER_CLIENT_SECRET" in os.environ: - del os.environ["HIDDENLAYER_CLIENT_SECRET"] + monkeypatch.delenv("HIDDENLAYER_CLIENT_SECRET", raising=False) with pytest.raises(RuntimeError): HiddenlayerGuardrailV2(guardrail_name="hiddenlayer", event_hook="pre_call") @pytest.mark.asyncio - async def test_apply_guardrail_request_no_violations(self): + async def test_apply_guardrail_request_no_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for request with no violations detected.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -691,9 +681,9 @@ class TestHiddenlayerGuardrailV2: assert "detection/v2/request-evaluations" in call_args.args[0] @pytest.mark.asyncio - async def test_apply_guardrail_request_with_violations(self): + async def test_apply_guardrail_request_with_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for request with violations detected (block via header).""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -751,9 +741,9 @@ class TestHiddenlayerGuardrailV2: assert "Blocked by Hiddenlayer" in str(exc_info.value.detail) @pytest.mark.asyncio - async def test_apply_guardrail_response_no_violations(self): + async def test_apply_guardrail_response_no_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response with no violations detected.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="post_call", default_on=True @@ -816,9 +806,9 @@ class TestHiddenlayerGuardrailV2: assert "detection/v2/response-evaluations" in call_args.args[0] @pytest.mark.asyncio - async def test_apply_guardrail_response_with_violations(self): + async def test_apply_guardrail_response_with_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response with violations detected (block via header).""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="post_call", default_on=True @@ -863,9 +853,9 @@ class TestHiddenlayerGuardrailV2: assert "Blocked by Hiddenlayer" in str(exc_info.value.detail) @pytest.mark.asyncio - async def test_apply_guardrail_response_with_tool_calls(self): + async def test_apply_guardrail_response_with_tool_calls(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response containing tool calls.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="post_call", default_on=True @@ -924,9 +914,9 @@ class TestHiddenlayerGuardrailV2: assert "detection/v2/response-evaluations" in call_args.args[0] @pytest.mark.asyncio - async def test_call_hiddenlayer_uses_correct_endpoints(self): + async def test_call_hiddenlayer_uses_correct_endpoints(self, monkeypatch: pytest.MonkeyPatch): """Test that _call_hiddenlayer uses the v2 request/response evaluation endpoints.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -959,9 +949,9 @@ class TestHiddenlayerGuardrailV2: assert "detection/v2/response-evaluations" in mock_post.call_args.args[0] @pytest.mark.asyncio - async def test_apply_guardrail_request_with_image(self): + async def test_apply_guardrail_request_with_image(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail sends multimodal content (image) to HiddenLayer v2.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True @@ -1030,9 +1020,9 @@ class TestHiddenlayerGuardrailV2: assert texts == ["how much is on this receipt?"] @pytest.mark.asyncio - async def test_apply_guardrail_request_with_image_multimodal_response(self): + async def test_apply_guardrail_request_with_image_multimodal_response(self, monkeypatch: pytest.MonkeyPatch): """Test that new_texts extraction handles multimodal content (list) returned by HiddenLayer v2.""" - os.environ["HIDDENLAYER_API_BASE"] = "https://my.hiddenlayer" + monkeypatch.setenv("HIDDENLAYER_API_BASE", "https://my.hiddenlayer") guardrail = HiddenlayerGuardrailV2( guardrail_name="hiddenlayer", event_hook="pre_call", default_on=True diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_lasso.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_lasso.py index 16185cadbdf..870c5e6d4a0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_lasso.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_lasso.py @@ -1,12 +1,10 @@ import os -import sys import pytest import uuid from unittest.mock import patch, MagicMock from httpx import Response, Request from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import DualCache @@ -19,13 +17,13 @@ from litellm.proxy.guardrails.guardrail_hooks.lasso.lasso import ( from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 -def test_lasso_guard_config(): +def test_lasso_guard_config(monkeypatch): """Test Lasso guard configuration with init_guardrails_v2.""" litellm.set_verbose = True litellm.guardrail_name_config_map = {} # Set environment variable for testing - os.environ["LASSO_API_KEY"] = "test-key" + monkeypatch.setenv("LASSO_API_KEY", "test-key") init_guardrails_v2( all_guardrails=[ diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py index 713f089e158..596c11908cb 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_mcp_end_user_permission.py @@ -2,15 +2,10 @@ Tests for MCP End User Permission Guardrail Hook """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.exceptions import GuardrailRaisedException from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 613cbbce8b4..da66c36328e 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -2,13 +2,10 @@ import asyncio import base64 import io import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) import httpx from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_onyx.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_onyx.py index c7a6df1361e..9208e0b3075 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_onyx.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_onyx.py @@ -1,5 +1,3 @@ -import os -import sys import uuid from unittest.mock import AsyncMock, MagicMock, patch @@ -8,8 +6,6 @@ import pytest from fastapi import HTTPException from httpx import Request, Response -sys.path.insert(0, os.path.abspath("../..")) - import litellm from litellm import ModelResponse from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -18,14 +14,13 @@ from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.types.utils import Choices, GenericGuardrailAPIInputs, Message -def test_onyx_guard_config(): +def test_onyx_guard_config(monkeypatch: pytest.MonkeyPatch): """Test Onyx guard configuration with init_guardrails_v2.""" - litellm.set_verbose = True - litellm.guardrail_name_config_map = {} + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) + monkeypatch.setattr(litellm, "callbacks", []) - # Set environment variables for testing - os.environ["ONYX_API_BASE"] = "https://test.onyx.security" - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_BASE", "https://test.onyx.security") + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") init_guardrails_v2( all_guardrails=[ @@ -41,18 +36,17 @@ def test_onyx_guard_config(): config_file_path="", ) - # Clean up - if "ONYX_API_BASE" in os.environ: - del os.environ["ONYX_API_BASE"] - if "ONYX_API_KEY" in os.environ: - del os.environ["ONYX_API_KEY"] + registered = [c for c in litellm.callbacks if isinstance(c, OnyxGuardrail)] + assert len(registered) == 1 + assert registered[0].guardrail_name == "onyx-guard" + assert registered[0].default_on is True + assert registered[0].event_hook == "pre_call" -def test_onyx_guard_with_custom_timeout_from_kwargs(): +def test_onyx_guard_with_custom_timeout_from_kwargs(monkeypatch: pytest.MonkeyPatch): """Test Onyx guard instantiation with custom timeout passed via kwargs.""" - # Set environment variables for testing - os.environ["ONYX_API_BASE"] = "https://test.onyx.security" - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_BASE", "https://test.onyx.security") + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -74,23 +68,16 @@ def test_onyx_guard_with_custom_timeout_from_kwargs(): assert timeout_param.read == 45.0 assert timeout_param.connect == 5.0 - # Clean up - if "ONYX_API_BASE" in os.environ: - del os.environ["ONYX_API_BASE"] - if "ONYX_API_KEY" in os.environ: - del os.environ["ONYX_API_KEY"] - -def test_onyx_guard_with_timeout_none_uses_env_var(): +def test_onyx_guard_with_timeout_none_uses_env_var(monkeypatch: pytest.MonkeyPatch): """Test Onyx guard with timeout=None uses ONYX_TIMEOUT env var. When timeout=None is passed (as it would be from config model with default None), the ONYX_TIMEOUT environment variable should be used. """ - # Set environment variables for testing - os.environ["ONYX_API_BASE"] = "https://test.onyx.security" - os.environ["ONYX_API_KEY"] = "test-api-key" - os.environ["ONYX_TIMEOUT"] = "60" + monkeypatch.setenv("ONYX_API_BASE", "https://test.onyx.security") + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") + monkeypatch.setenv("ONYX_TIMEOUT", "60") with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -112,23 +99,13 @@ def test_onyx_guard_with_timeout_none_uses_env_var(): assert timeout_param.read == 60.0 assert timeout_param.connect == 5.0 - # Clean up - if "ONYX_API_BASE" in os.environ: - del os.environ["ONYX_API_BASE"] - if "ONYX_API_KEY" in os.environ: - del os.environ["ONYX_API_KEY"] - if "ONYX_TIMEOUT" in os.environ: - del os.environ["ONYX_TIMEOUT"] - -def test_onyx_guard_with_timeout_none_defaults_to_10(): +def test_onyx_guard_with_timeout_none_defaults_to_10(monkeypatch: pytest.MonkeyPatch): """Test Onyx guard with timeout=None and no env var defaults to 10 seconds.""" - # Set environment variables for testing - os.environ["ONYX_API_BASE"] = "https://test.onyx.security" - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_BASE", "https://test.onyx.security") + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") # Ensure ONYX_TIMEOUT is not set - if "ONYX_TIMEOUT" in os.environ: - del os.environ["ONYX_TIMEOUT"] + monkeypatch.delenv("ONYX_TIMEOUT", raising=False) with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -150,34 +127,18 @@ def test_onyx_guard_with_timeout_none_defaults_to_10(): assert timeout_param.read == 10.0 assert timeout_param.connect == 5.0 - # Clean up - if "ONYX_API_BASE" in os.environ: - del os.environ["ONYX_API_BASE"] - if "ONYX_API_KEY" in os.environ: - del os.environ["ONYX_API_KEY"] - class TestOnyxGuardrail: """Test suite for Onyx Security Guardrail integration.""" - def setup_method(self): - """Setup test environment.""" - # Clean up any existing environment variables - for key in ["ONYX_API_BASE", "ONYX_API_KEY", "ONYX_TIMEOUT"]: - if key in os.environ: - del os.environ[key] + @pytest.fixture(autouse=True) + def clear_onyx_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + for key in ("ONYX_API_BASE", "ONYX_API_KEY", "ONYX_TIMEOUT"): + monkeypatch.delenv(key, raising=False) - def teardown_method(self): - """Clean up test environment.""" - # Clean up any environment variables set during tests - for key in ["ONYX_API_BASE", "ONYX_API_KEY", "ONYX_TIMEOUT"]: - if key in os.environ: - del os.environ[key] - - def test_initialization_with_defaults(self): + def test_initialization_with_defaults(self, monkeypatch: pytest.MonkeyPatch): """Test successful initialization with default values.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -189,10 +150,10 @@ class TestOnyxGuardrail: assert guardrail.guardrail_name == "test-guard" assert guardrail.event_hook == "pre_call" - def test_initialization_with_env_vars(self): + def test_initialization_with_env_vars(self, monkeypatch: pytest.MonkeyPatch): """Test initialization with environment variables.""" - os.environ["ONYX_API_BASE"] = "https://custom.onyx.security" - os.environ["ONYX_API_KEY"] = "custom-api-key" + monkeypatch.setenv("ONYX_API_BASE", "https://custom.onyx.security") + monkeypatch.setenv("ONYX_API_KEY", "custom-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="post_call", default_on=True @@ -202,20 +163,19 @@ class TestOnyxGuardrail: assert guardrail.api_key == "custom-api-key" assert guardrail.event_hook == "post_call" - def test_initialization_fails_when_api_key_missing(self): + def test_initialization_fails_when_api_key_missing(self, monkeypatch: pytest.MonkeyPatch): """Test that initialization fails when API key is not set.""" # Ensure API key is not set - if "ONYX_API_KEY" in os.environ: - del os.environ["ONYX_API_KEY"] + monkeypatch.delenv("ONYX_API_KEY", raising=False) with pytest.raises( ValueError, match="ONYX_API_KEY environment variable is not set" ): OnyxGuardrail(guardrail_name="test-guard", event_hook="pre_call") - def test_initialization_with_default_timeout(self): + def test_initialization_with_default_timeout(self, monkeypatch: pytest.MonkeyPatch): """Test that default timeout is 10.0 seconds.""" - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -232,9 +192,9 @@ class TestOnyxGuardrail: assert timeout_param.read == 10.0 assert timeout_param.connect == 5.0 - def test_initialization_with_custom_timeout_parameter(self): + def test_initialization_with_custom_timeout_parameter(self, monkeypatch: pytest.MonkeyPatch): """Test initialization with custom timeout parameter.""" - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -254,14 +214,14 @@ class TestOnyxGuardrail: assert timeout_param.read == 30.0 assert timeout_param.connect == 5.0 - def test_initialization_with_timeout_from_env_var(self): + def test_initialization_with_timeout_from_env_var(self, monkeypatch: pytest.MonkeyPatch): """Test initialization with timeout from ONYX_TIMEOUT environment variable. Note: The env var is only used when timeout=None is explicitly passed, since the default parameter value is 10.0 (not None). """ - os.environ["ONYX_API_KEY"] = "test-api-key" - os.environ["ONYX_TIMEOUT"] = "25" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") + monkeypatch.setenv("ONYX_TIMEOUT", "25") with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -282,10 +242,10 @@ class TestOnyxGuardrail: assert timeout_param.read == 25.0 assert timeout_param.connect == 5.0 - def test_initialization_timeout_parameter_overrides_env_var(self): + def test_initialization_timeout_parameter_overrides_env_var(self, monkeypatch: pytest.MonkeyPatch): """Test that timeout parameter overrides ONYX_TIMEOUT environment variable.""" - os.environ["ONYX_API_KEY"] = "test-api-key" - os.environ["ONYX_TIMEOUT"] = "25" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") + monkeypatch.setenv("ONYX_TIMEOUT", "25") with patch( "litellm.proxy.guardrails.guardrail_hooks.onyx.onyx.get_async_httpx_client" @@ -306,10 +266,9 @@ class TestOnyxGuardrail: assert timeout_param.connect == 5.0 @pytest.mark.asyncio - async def test_apply_guardrail_request_no_violations(self): + async def test_apply_guardrail_request_no_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for request with no violations detected.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") # Setup guardrail guardrail = OnyxGuardrail( @@ -372,10 +331,9 @@ class TestOnyxGuardrail: assert call_args.kwargs["json"]["conversation_id"] == "test-call-id" @pytest.mark.asyncio - async def test_apply_guardrail_request_with_violations(self): + async def test_apply_guardrail_request_with_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for request with violations detected.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") # Setup guardrail guardrail = OnyxGuardrail( @@ -423,10 +381,9 @@ class TestOnyxGuardrail: assert "prompt_injection" in str(exc_info.value.detail) @pytest.mark.asyncio - async def test_apply_guardrail_response_no_violations(self): + async def test_apply_guardrail_response_no_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response with no violations detected.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") # Setup guardrail guardrail = OnyxGuardrail( @@ -497,10 +454,9 @@ class TestOnyxGuardrail: assert call_args.kwargs["json"]["conversation_id"] == "test-call-id-2" @pytest.mark.asyncio - async def test_apply_guardrail_response_with_violations(self): + async def test_apply_guardrail_response_with_violations(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail for response with violations detected.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") # Setup guardrail guardrail = OnyxGuardrail( @@ -558,10 +514,9 @@ class TestOnyxGuardrail: assert "illegal_instructions" in str(exc_info.value.detail) @pytest.mark.asyncio - async def test_apply_guardrail_api_error_handling(self): + async def test_apply_guardrail_api_error_handling(self, monkeypatch: pytest.MonkeyPatch): """Test handling of API errors in apply_guardrail.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -591,10 +546,9 @@ class TestOnyxGuardrail: assert result == inputs @pytest.mark.asyncio - async def test_apply_guardrail_timeout_error_handling(self): + async def test_apply_guardrail_timeout_error_handling(self, monkeypatch: pytest.MonkeyPatch): """Test handling of timeout errors in apply_guardrail (graceful degradation).""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", @@ -629,10 +583,9 @@ class TestOnyxGuardrail: assert result == inputs @pytest.mark.asyncio - async def test_apply_guardrail_read_timeout_error_handling(self): + async def test_apply_guardrail_read_timeout_error_handling(self, monkeypatch: pytest.MonkeyPatch): """Test handling of read timeout errors in apply_guardrail.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", @@ -667,10 +620,9 @@ class TestOnyxGuardrail: assert result == inputs @pytest.mark.asyncio - async def test_apply_guardrail_connect_timeout_error_handling(self): + async def test_apply_guardrail_connect_timeout_error_handling(self, monkeypatch: pytest.MonkeyPatch): """Test handling of connect timeout errors in apply_guardrail.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", @@ -705,10 +657,9 @@ class TestOnyxGuardrail: assert result == inputs @pytest.mark.asyncio - async def test_apply_guardrail_no_logging_obj(self): + async def test_apply_guardrail_no_logging_obj(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail without logging object (uses UUID).""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -747,10 +698,9 @@ class TestOnyxGuardrail: assert call_args.kwargs["json"]["conversation_id"] == "test-uuid" @pytest.mark.asyncio - async def test_validate_with_guard_server_method(self): + async def test_validate_with_guard_server_method(self, monkeypatch: pytest.MonkeyPatch): """Test the _validate_with_guard_server internal method.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -788,10 +738,9 @@ class TestOnyxGuardrail: ) @pytest.mark.asyncio - async def test_validate_with_guard_server_blocked(self): + async def test_validate_with_guard_server_blocked(self, monkeypatch: pytest.MonkeyPatch): """Test _validate_with_guard_server when request is blocked.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -825,10 +774,9 @@ class TestOnyxGuardrail: assert config_model.__name__ == "OnyxGuardrailConfigModel" @pytest.mark.asyncio - async def test_apply_guardrail_with_modelresponse(self): + async def test_apply_guardrail_with_modelresponse(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail with ModelResponse object for response type.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="post_call", default_on=True @@ -880,10 +828,9 @@ class TestOnyxGuardrail: assert "payload" in call_args.kwargs["json"] @pytest.mark.asyncio - async def test_apply_guardrail_response_error_handling(self): + async def test_apply_guardrail_response_error_handling(self, monkeypatch: pytest.MonkeyPatch): """Test error handling when processing response data.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="post_call", default_on=True @@ -925,11 +872,11 @@ class TestOnyxIntegration: """Test integration scenarios.""" @pytest.mark.asyncio - async def test_full_guardrail_flow(self): + async def test_full_guardrail_flow(self, monkeypatch: pytest.MonkeyPatch): """Test full guardrail flow with multiple hooks.""" # Set environment variables - os.environ["ONYX_API_BASE"] = "https://test.onyx.security" - os.environ["ONYX_API_KEY"] = "test-key" + monkeypatch.setenv("ONYX_API_BASE", "https://test.onyx.security") + monkeypatch.setenv("ONYX_API_KEY", "test-key") init_guardrails_v2( all_guardrails=[ @@ -966,17 +913,11 @@ class TestOnyxIntegration: ) assert len(custom_loggers) >= 3 - # Clean up - if "ONYX_API_BASE" in os.environ: - del os.environ["ONYX_API_BASE"] - if "ONYX_API_KEY" in os.environ: - del os.environ["ONYX_API_KEY"] @pytest.mark.asyncio - async def test_apply_guardrail_empty_request_data(self): + async def test_apply_guardrail_empty_request_data(self, monkeypatch: pytest.MonkeyPatch): """Test apply_guardrail with empty request data.""" - # Set required API key - os.environ["ONYX_API_KEY"] = "test-api-key" + monkeypatch.setenv("ONYX_API_KEY", "test-api-key") guardrail = OnyxGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index c779150ad3e..60be3be5e8b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -4,14 +4,11 @@ Tests PII detection and masking for different message formats """ import asyncio -import os -import sys from contextlib import asynccontextmanager from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../../../..")) import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_repelloai.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_repelloai.py index 55f01ebddfd..1ef25b6e7ab 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_repelloai.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_repelloai.py @@ -1,11 +1,9 @@ import os -import sys import pytest from fastapi import HTTPException from httpx import ConnectError, Request, Response -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import DualCache @@ -93,24 +91,24 @@ class TestRepelloAIInitialization: with pytest.raises(ValueError, match="asset_id"): RepelloAIGuardrail(api_key="test-api-key", guardrail_name="t") - def test_api_key_from_env(self): - os.environ["REPELLOAI_API_KEY"] = "env-key" + def test_api_key_from_env(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("REPELLOAI_API_KEY", "env-key") guardrail = RepelloAIGuardrail(asset_id="asset-123", guardrail_name="t") assert guardrail.repelloai_api_key == "env-key" - def test_api_key_from_argus_env(self): - os.environ["ARGUS_API_KEY"] = "argus-key" + def test_api_key_from_argus_env(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ARGUS_API_KEY", "argus-key") guardrail = RepelloAIGuardrail(asset_id="asset-123", guardrail_name="t") assert guardrail.repelloai_api_key == "argus-key" - def test_argus_env_preferred_over_legacy(self): - os.environ["ARGUS_API_KEY"] = "argus-key" - os.environ["REPELLOAI_API_KEY"] = "legacy-key" + def test_argus_env_preferred_over_legacy(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ARGUS_API_KEY", "argus-key") + monkeypatch.setenv("REPELLOAI_API_KEY", "legacy-key") guardrail = RepelloAIGuardrail(asset_id="asset-123", guardrail_name="t") assert guardrail.repelloai_api_key == "argus-key" - def test_explicit_api_key_preferred_over_env(self): - os.environ["ARGUS_API_KEY"] = "argus-key" + def test_explicit_api_key_preferred_over_env(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ARGUS_API_KEY", "argus-key") guardrail = RepelloAIGuardrail( api_key="explicit-key", asset_id="asset-123", guardrail_name="t" ) @@ -145,10 +143,10 @@ class TestRepelloAIInitialization: assert guardrail.api_base == DEFAULT_REPELLOAI_API_BASE assert guardrail.unreachable_fallback == "fail_closed" - def test_init_guardrails_v2_wiring(self): + def test_init_guardrails_v2_wiring(self, monkeypatch: pytest.MonkeyPatch): """The guardrail registers and constructs via the config.yaml path.""" - litellm.guardrail_name_config_map = {} - os.environ["REPELLOAI_API_KEY"] = "test-key" + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) + monkeypatch.setenv("REPELLOAI_API_KEY", "test-key") init_guardrails_v2( all_guardrails=[ { diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 0c5addbc143..0dbd4591ac9 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -3,16 +3,13 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics) """ import json -import os import re -import sys from unittest.mock import patch import pytest from litellm.caching.dual_cache import DualCache -sys.path.insert(0, os.path.abspath("../../../../../..")) from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py index 8b9b6820e8c..9113ac5015f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py @@ -2,15 +2,12 @@ Unit tests for ToolPolicyGuardrail. """ -import os -import sys from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../../../..")) from litellm.proxy.guardrails.guardrail_hooks.tool_policy.tool_policy_guardrail import ( ToolPolicyGuardrail, diff --git a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py index 291ce732fc6..e70fc61de30 100644 --- a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py @@ -15,14 +15,11 @@ Streaming: CSW.__anext__ stores args on logging_obj at stream end. """ import asyncio -import os -import sys from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 71ff9111b60..45f5afef1bc 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -1,15 +1,10 @@ import json -import os -import sys from datetime import datetime from typing import Dict, List, Optional from unittest.mock import AsyncMock import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py index 8edb56ce25e..82363302d2e 100644 --- a/tests/test_litellm/proxy/guardrails/test_init_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_init_guardrails.py @@ -1,11 +1,8 @@ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler from litellm.types.guardrails import SupportedGuardrailIntegrations diff --git a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py index 48f6b3ba2b9..c6be433399a 100644 --- a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py @@ -7,13 +7,10 @@ and following LiteLLM testing patterns and best practices. # Standard library imports import importlib -import os -import sys from typing import Any, Dict from unittest.mock import Mock, patch # Add parent directory to path for imports -sys.path.insert(0, os.path.abspath("../../..")) # Third-party imports import json @@ -65,7 +62,6 @@ def setup_and_teardown(): asyncio.set_event_loop(loop) # Set up litellm state - litellm.set_verbose = True litellm.guardrail_name_config_map = {} yield diff --git a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py index c8f22e6c15e..26beaa78a46 100644 --- a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py @@ -1,5 +1,3 @@ -import os -import sys from fastapi.exceptions import HTTPException from unittest.mock import patch, AsyncMock from httpx import Response, Request @@ -12,21 +10,17 @@ from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security im PromptSecurityGuardrail, ) -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 -def test_prompt_security_guard_config(): +def test_prompt_security_guard_config(monkeypatch: pytest.MonkeyPatch): """Test guardrail initialization with proper configuration""" - litellm.set_verbose = True - litellm.guardrail_name_config_map = {} + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) + monkeypatch.setattr(litellm, "callbacks", []) - # Set environment variables for testing - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") init_guardrails_v2( all_guardrails=[ @@ -42,21 +36,19 @@ def test_prompt_security_guard_config(): config_file_path="", ) - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] + registered = [c for c in litellm.callbacks if isinstance(c, PromptSecurityGuardrail)] + assert len(registered) == 1 + assert registered[0].guardrail_name == "prompt_security" + assert registered[0].default_on is True + assert registered[0].event_hook == "during_call" -def test_prompt_security_guard_config_no_api_key(): +def test_prompt_security_guard_config_no_api_key(monkeypatch: pytest.MonkeyPatch): """Test that initialization fails when API key is missing""" - litellm.set_verbose = True - litellm.guardrail_name_config_map = {} + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) - # Ensure API key is not in environment - if "PROMPT_SECURITY_API_KEY" in os.environ: - del os.environ["PROMPT_SECURITY_API_KEY"] - if "PROMPT_SECURITY_API_BASE" in os.environ: - del os.environ["PROMPT_SECURITY_API_BASE"] + monkeypatch.delenv("PROMPT_SECURITY_API_KEY", raising=False) + monkeypatch.delenv("PROMPT_SECURITY_API_BASE", raising=False) with pytest.raises( PromptSecurityGuardrailMissingSecrets, @@ -78,10 +70,10 @@ def test_prompt_security_guard_config_no_api_key(): @pytest.mark.asyncio -async def test_apply_guardrail_block_request(): +async def test_apply_guardrail_block_request(monkeypatch: pytest.MonkeyPatch): """Test that apply_guardrail blocks malicious prompts""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -126,16 +118,12 @@ async def test_apply_guardrail_block_request(): assert "prompt_injection" in str(excinfo.value.detail) assert "jailbreak" in str(excinfo.value.detail) - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_apply_guardrail_modify_request(): +async def test_apply_guardrail_modify_request(monkeypatch: pytest.MonkeyPatch): """Test that apply_guardrail modifies prompts when needed""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -177,16 +165,12 @@ async def test_apply_guardrail_modify_request(): assert result["texts"] == ["User prompt with PII: SSN [REDACTED]"] - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_apply_guardrail_allow_request(): +async def test_apply_guardrail_allow_request(monkeypatch: pytest.MonkeyPatch): """Test that apply_guardrail allows safe prompts""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -220,16 +204,12 @@ async def test_apply_guardrail_allow_request(): assert result == inputs - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_apply_guardrail_block_response(): +async def test_apply_guardrail_block_response(monkeypatch: pytest.MonkeyPatch): """Test that apply_guardrail blocks malicious responses""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="post_call", default_on=True @@ -267,16 +247,12 @@ async def test_apply_guardrail_block_response(): assert "Blocked by Prompt Security" in str(excinfo.value.detail) assert "pii_exposure" in str(excinfo.value.detail) - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_apply_guardrail_modify_response(): +async def test_apply_guardrail_modify_response(monkeypatch: pytest.MonkeyPatch): """Test that apply_guardrail modifies responses when needed""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="post_call", default_on=True @@ -311,16 +287,12 @@ async def test_apply_guardrail_modify_response(): assert result["texts"] == ["Your SSN is [REDACTED]"] - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_file_sanitization(): +async def test_file_sanitization(monkeypatch: pytest.MonkeyPatch): """Test file sanitization for images""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -401,16 +373,12 @@ async def test_file_sanitization(): # Should complete without errors and return the data assert result is not None - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_file_sanitization_block(): +async def test_file_sanitization_block(monkeypatch: pytest.MonkeyPatch): """Test that file sanitization blocks malicious files""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -485,16 +453,12 @@ async def test_file_sanitization_block(): assert "File blocked by Prompt Security" in str(excinfo.value.detail) assert "malware_detected" in str(excinfo.value.detail) - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_user_api_key_alias_forwarding(): +async def test_user_api_key_alias_forwarding(monkeypatch: pytest.MonkeyPatch): """Test that user API key alias is properly sent via headers and payload""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -530,15 +494,12 @@ async def test_user_api_key_alias_forwarding(): payload = call_kwargs["json"] assert payload["user"] == "vk-alias" - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_role_filtering(): +async def test_role_filtering(monkeypatch: pytest.MonkeyPatch): """Test that tool/function messages are filtered out by default""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -594,17 +555,13 @@ async def test_role_filtering(): assert len(sent_messages) == 3 assert all(msg["role"] in ["system", "user", "assistant"] for msg in sent_messages) - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - @pytest.mark.asyncio -async def test_check_tool_results_enabled(): +async def test_check_tool_results_enabled(monkeypatch: pytest.MonkeyPatch): """Test with check_tool_results=True: transforms tool/function to 'other' role""" - os.environ["PROMPT_SECURITY_API_KEY"] = "test-key" - os.environ["PROMPT_SECURITY_API_BASE"] = "https://test.prompt.security" - os.environ["PROMPT_SECURITY_CHECK_TOOL_RESULTS"] = "true" + monkeypatch.setenv("PROMPT_SECURITY_API_KEY", "test-key") + monkeypatch.setenv("PROMPT_SECURITY_API_BASE", "https://test.prompt.security") + monkeypatch.setenv("PROMPT_SECURITY_CHECK_TOOL_RESULTS", "true") guardrail = PromptSecurityGuardrail( guardrail_name="test-guard", event_hook="pre_call", default_on=True @@ -680,7 +637,3 @@ async def test_check_tool_results_enabled(): assert "indirect_prompt_injection" in str(excinfo.value.detail) - # Clean up - del os.environ["PROMPT_SECURITY_API_KEY"] - del os.environ["PROMPT_SECURITY_API_BASE"] - del os.environ["PROMPT_SECURITY_CHECK_TOOL_RESULTS"] diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index ff143bd055f..1665fa03639 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -8,15 +8,12 @@ detail 404'd, overview omitted them (or rendered them as Custom/Guardrail orphans), and logs missed their logical-name alias. """ -import os -import sys from datetime import datetime from typing import Any, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) from fastapi import HTTPException from prisma.errors import TableNotFoundError diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index e576ba87e88..62919200d47 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1,13 +1,8 @@ -import os -import sys import time from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import httpx import pytest diff --git a/tests/test_litellm/proxy/hooks/test_async_post_call_streaming_iterator_hook.py b/tests/test_litellm/proxy/hooks/test_async_post_call_streaming_iterator_hook.py index 9a097230c19..9c785d59830 100644 --- a/tests/test_litellm/proxy/hooks/test_async_post_call_streaming_iterator_hook.py +++ b/tests/test_litellm/proxy/hooks/test_async_post_call_streaming_iterator_hook.py @@ -7,16 +7,11 @@ Verifies that the hook: 3. Actually yields chunks from async generators """ -import os -import sys from typing import AsyncGenerator, Any from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 6c717d6f71c..0ff8b67b1a7 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -6,14 +6,12 @@ Core tests to validate that priority weights are respected (0.9/0.1) instead of import asyncio import os -import sys import time from datetime import datetime, timedelta from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm import DualCache, Router @@ -42,7 +40,7 @@ def time_controller(monkeypatch): @pytest.mark.asyncio -async def test_priority_weight_allocation(): +async def test_priority_weight_allocation(monkeypatch): """ Test that priority weights are correctly applied instead of equal splitting. @@ -53,7 +51,7 @@ async def test_priority_weight_allocation(): This validates the core fix where before it would split 50/50. """ # Set up environment for premium feature - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") # Set up priority reservations litellm.priority_reservation = {"high": 0.9, "low": 0.1} @@ -128,7 +126,7 @@ async def test_priority_weight_allocation(): @pytest.mark.asyncio -async def test_concurrent_priority_requests(): +async def test_concurrent_priority_requests(monkeypatch): """ Test the core issue: 5 concurrent requests with different priorities should get proper allocation based on priority weights, not equal splitting. @@ -136,7 +134,7 @@ async def test_concurrent_priority_requests(): This tests the exact scenario mentioned: priorities 0.9 and 0.1 should be 0.9/0.1, not 0.5/0.5. """ # Set up environment for premium feature - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") # Set up the exact scenario from the issue litellm.priority_reservation = {"high": 0.9, "low": 0.1} @@ -214,7 +212,7 @@ async def test_concurrent_priority_requests(): @pytest.mark.asyncio -async def test_100_concurrent_priority_requests(time_controller): +async def test_100_concurrent_priority_requests(time_controller, monkeypatch): """ Stress test: 100 concurrent requests with mixed priorities over 10 seconds. @@ -224,7 +222,7 @@ async def test_100_concurrent_priority_requests(time_controller): - Spread across 10 seconds to simulate real-world load """ # Set up environment for premium feature - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") # Set up priority reservations litellm.priority_reservation = {"high": 0.9, "low": 0.1} @@ -384,7 +382,7 @@ async def test_100_concurrent_priority_requests(time_controller): @pytest.mark.asyncio -async def test_concurrent_pre_call_hooks_stress(): +async def test_concurrent_pre_call_hooks_stress(monkeypatch): """ Stress test: 50 concurrent pre-call hooks with saturation-aware priority enforcement. @@ -394,7 +392,7 @@ async def test_concurrent_pre_call_hooks_stress(): Standard users (20% allocation) should have ~70% success rate with 30% random limiting. """ # Set up environment for premium feature - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"premium": 0.8, "standard": 0.2} @@ -634,7 +632,7 @@ async def test_concurrent_pre_call_hooks_stress(): @pytest.mark.asyncio -async def test_fake_calls_case_1_no_rate_limiting_at_capacity(): +async def test_fake_calls_case_1_no_rate_limiting_at_capacity(monkeypatch): """ Test Case 1: Saturation-Aware Rate Limiting at 50% Threshold @@ -650,7 +648,7 @@ async def test_fake_calls_case_1_no_rate_limiting_at_capacity(): Once saturation hits 50%, strict mode enforces priority-based limits. """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") # Set up priority reservations litellm.priority_reservation = {"key_a": 0.75, "key_b": 0.25} @@ -759,7 +757,7 @@ async def test_fake_calls_case_1_no_rate_limiting_at_capacity(): @pytest.mark.asyncio -async def test_fake_calls_case_2_priority_queue_during_saturation(): +async def test_fake_calls_case_2_priority_queue_during_saturation(monkeypatch): """ Test Case 2: Priority Queue Behavior During Saturation @@ -773,7 +771,7 @@ async def test_fake_calls_case_2_priority_queue_during_saturation(): When total traffic exceeds capacity, rate limiting enforces priority reservations. """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"key_a": 0.75, "key_b": 0.25} @@ -886,7 +884,7 @@ async def test_fake_calls_case_2_priority_queue_during_saturation(): @pytest.mark.asyncio -async def test_fake_calls_case_3_spillover_capacity_default_keys(): +async def test_fake_calls_case_3_spillover_capacity_default_keys(monkeypatch): """ Test Case 3: Spillover Capacity for Default Keys @@ -906,7 +904,7 @@ async def test_fake_calls_case_3_spillover_capacity_default_keys(): Tests spillover behavior where default keys share remaining capacity. """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"key_a": 0.75} litellm.priority_reservation_settings.default_priority = 0.25 @@ -1025,7 +1023,7 @@ async def test_fake_calls_case_3_spillover_capacity_default_keys(): @pytest.mark.asyncio -async def test_fake_calls_case_4_over_allocated_with_normalization(): +async def test_fake_calls_case_4_over_allocated_with_normalization(monkeypatch): """ Test Case 4: Over-Allocated Priority reservations with Normalization @@ -1042,7 +1040,7 @@ async def test_fake_calls_case_4_over_allocated_with_normalization(): - Due to concurrent burst, total successful may exceed 100 RPM in the test window - This test verifies normalization works and total capacity is reasonably bounded """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"key_a": 0.60, "key_b": 0.80} @@ -1156,7 +1154,7 @@ async def test_fake_calls_case_4_over_allocated_with_normalization(): @pytest.mark.asyncio -async def test_fake_calls_case_5_default_value_priority_reservation(): +async def test_fake_calls_case_5_default_value_priority_reservation(monkeypatch): """ Test Case 5: Default value for priority reservation @@ -1176,7 +1174,7 @@ async def test_fake_calls_case_5_default_value_priority_reservation(): Tests complex scenario with explicit priorities and default priority. """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"key_a": 0.50, "key_b": 0.20, "key_c": 0.05} litellm.priority_reservation_settings.default_priority = 0.05 @@ -1296,7 +1294,7 @@ async def test_fake_calls_case_5_default_value_priority_reservation(): @pytest.mark.asyncio -async def test_default_priority_shared_pool(): +async def test_default_priority_shared_pool(monkeypatch): """ Test that keys without explicit priority share ONE default pool, not get individual allocations. @@ -1304,7 +1302,7 @@ async def test_default_priority_shared_pool(): - Key A, B, C (no priority) should share ONE 25 RPM pool - NOT get 25 RPM each (which would be 75 RPM total) """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"prod": 0.75} litellm.priority_reservation_settings.default_priority = 0.25 @@ -1382,7 +1380,7 @@ async def test_default_priority_shared_pool(): @pytest.mark.asyncio -async def test_async_log_success_event_increments_by_actual_tokens(): +async def test_async_log_success_event_increments_by_actual_tokens(monkeypatch): """ Test that async_log_success_event increments token counters by actual token usage. @@ -1394,7 +1392,7 @@ async def test_async_log_success_event_increments_by_actual_tokens(): from litellm.types.utils import ModelResponse, Usage - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"dev": 0.1, "prod": 0.9} dual_cache = DualCache() @@ -1483,7 +1481,7 @@ async def test_async_log_success_event_increments_by_actual_tokens(): @pytest.mark.asyncio -async def test_saturation_check_cache_ttl_configuration(): +async def test_saturation_check_cache_ttl_configuration(monkeypatch): """ Test that saturation_check_cache_ttl controls how long saturation values are cached locally. @@ -1492,7 +1490,7 @@ async def test_saturation_check_cache_ttl_configuration(): - After expiration, fresh values should be fetched from Redis - This prevents nodes from having stale saturation data in multi-node deployments """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") # Set a short TTL for testing (5 seconds) original_ttl = litellm.priority_reservation_settings.saturation_check_cache_ttl @@ -1587,7 +1585,7 @@ async def test_saturation_check_cache_ttl_configuration(): @pytest.mark.asyncio -async def test_async_log_success_event_uses_team_priority_from_auth_metadata(): +async def test_async_log_success_event_uses_team_priority_from_auth_metadata(monkeypatch): """ Test that async_log_success_event correctly retrieves priority from user_api_key_auth_metadata. @@ -1598,7 +1596,7 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(): from litellm.types.utils import ModelResponse, Usage - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"team_priority": 0.8, "default": 0.2} dual_cache = DualCache() @@ -1680,7 +1678,7 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(): @pytest.mark.asyncio -async def test_priority_429_includes_model_name_and_configured_limits(): +async def test_priority_429_includes_model_name_and_configured_limits(monkeypatch): """ The priority-based 429 should tell operators which model was hit and what the model's configured TPM/RPM are, so they can decide whether to tune the @@ -1694,7 +1692,7 @@ async def test_priority_429_includes_model_name_and_configured_limits(): """ from fastapi import HTTPException - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"prod": 0.5} dual_cache = DualCache() @@ -1774,7 +1772,7 @@ async def test_priority_429_includes_model_name_and_configured_limits(): @pytest.mark.asyncio -async def test_tpm_only_model_enforces_priority_and_model_capacity(): +async def test_tpm_only_model_enforces_priority_and_model_capacity(monkeypatch): """Regression: a model configured with ONLY tpm (no rpm) must still be rate limited. @@ -1789,7 +1787,7 @@ async def test_tpm_only_model_enforces_priority_and_model_capacity(): from litellm.types.utils import ModelResponse, Usage - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"dev": 0.25, "prod": 0.5} dual_cache = DualCache() diff --git a/tests/test_litellm/proxy/hooks/test_image_generation_guardrails.py b/tests/test_litellm/proxy/hooks/test_image_generation_guardrails.py index 04fdc00e114..fd8299b07ec 100644 --- a/tests/test_litellm/proxy/hooks/test_image_generation_guardrails.py +++ b/tests/test_litellm/proxy/hooks/test_image_generation_guardrails.py @@ -9,14 +9,11 @@ These tests verify: 3. A guardrail that raises blocks the response (exception propagates). """ -import os -import sys from typing import Any, Optional from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index fa7320b2bc6..860fb762450 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -5,13 +5,10 @@ Validates that email and secret manager operations are independent and non-block """ import asyncio -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks diff --git a/tests/test_litellm/proxy/hooks/test_post_call_failure_hook_integration.py b/tests/test_litellm/proxy/hooks/test_post_call_failure_hook_integration.py index f9cb586d405..55e058d86a1 100644 --- a/tests/test_litellm/proxy/hooks/test_post_call_failure_hook_integration.py +++ b/tests/test_litellm/proxy/hooks/test_post_call_failure_hook_integration.py @@ -5,13 +5,10 @@ Tests verify that the failure hook can transform error responses sent to clients similar to how async_post_call_success_hook can transform successful responses. """ -import os -import sys import pytest from typing import Optional from unittest.mock import patch -sys.path.insert(0, os.path.abspath("../../../..")) from fastapi import HTTPException from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py b/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py index 660b0b0162a..a896ab62bef 100644 --- a/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py +++ b/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py @@ -5,13 +5,10 @@ Tests verify that CustomLogger callbacks can inject custom HTTP response headers into success (streaming and non-streaming) and failure responses. """ -import os -import sys import pytest from typing import Any, Dict, Optional from unittest.mock import patch -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/hooks/test_post_call_streaming_hook_integration.py b/tests/test_litellm/proxy/hooks/test_post_call_streaming_hook_integration.py index 22349ec9821..e539bd3a0b2 100644 --- a/tests/test_litellm/proxy/hooks/test_post_call_streaming_hook_integration.py +++ b/tests/test_litellm/proxy/hooks/test_post_call_streaming_hook_integration.py @@ -4,13 +4,10 @@ Integration tests for async_post_call_streaming_hook. Tests verify that the streaming hook can transform streaming responses sent to clients. """ -import os -import sys import pytest from typing import Any from unittest.mock import patch, MagicMock -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/test_litellm/proxy/hooks/test_post_call_success_hook_integration.py b/tests/test_litellm/proxy/hooks/test_post_call_success_hook_integration.py index 219f436f985..50208cc278e 100644 --- a/tests/test_litellm/proxy/hooks/test_post_call_success_hook_integration.py +++ b/tests/test_litellm/proxy/hooks/test_post_call_success_hook_integration.py @@ -5,13 +5,10 @@ Tests verify that the success hook can transform responses sent to clients. This mirrors the behavior of CustomGuardrail hooks and streaming iterator hooks. """ -import os -import sys import pytest from typing import Any from unittest.mock import patch, MagicMock -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 50c93ed5275..871f4b4bcd1 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1,11 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py b/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py index 1c1e8eee145..23c717b0e3a 100644 --- a/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py +++ b/tests/test_litellm/proxy/hooks/test_rate_limiter_toctou.py @@ -18,12 +18,10 @@ check-and-increment becomes atomic. import asyncio import os -import sys from typing import List import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm import DualCache, Router @@ -189,7 +187,7 @@ async def test_batch_limiter_uses_atomic_check_and_increment(): @pytest.mark.asyncio -async def test_dynamic_rate_limiter_v3_concurrent_bypasses_model_capacity(): +async def test_dynamic_rate_limiter_v3_concurrent_bypasses_model_capacity(monkeypatch): """ DynamicRateLimitHandler PHASE 1 (read_only check) → PHASE 3 (increment) is non-atomic: dynamic_rate_limiter_v3.py:463-548. @@ -209,7 +207,7 @@ async def test_dynamic_rate_limiter_v3_concurrent_bypasses_model_capacity(): # RPM + 1 successes before the next sees counter > RPM. MAX_SEQUENTIAL_SUCCESSES = MODEL_RPM + 1 - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"high": 0.9, "low": 0.1} dual_cache = DualCache() @@ -273,7 +271,7 @@ async def test_dynamic_rate_limiter_v3_concurrent_bypasses_model_capacity(): @pytest.mark.asyncio -async def test_dynamic_rate_limiter_v3_uses_atomic_check_and_increment(): +async def test_dynamic_rate_limiter_v3_uses_atomic_check_and_increment(monkeypatch): """ Regression test: dynamic limiter's enforced descriptors flow through `atomic_check_and_increment_by_n`, not the legacy @@ -283,7 +281,7 @@ async def test_dynamic_rate_limiter_v3_uses_atomic_check_and_increment(): bundled into the atomic call alongside model_saturation_check. When not enforced, priority counter is incremented for tracking only. """ - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"high": 0.9, "low": 0.1} dual_cache = DualCache() @@ -413,7 +411,7 @@ async def test_batch_zero_token_consumes_rpm_only(): @pytest.mark.asyncio -async def test_dynamic_rate_limiter_v3_fails_closed_on_unknown_descriptor(): +async def test_dynamic_rate_limiter_v3_fails_closed_on_unknown_descriptor(monkeypatch): """ Fail-closed guard: when atomic_check_and_increment_by_n returns overall_code=OVER_LIMIT but with a descriptor_key the dispatcher does @@ -425,7 +423,7 @@ async def test_dynamic_rate_limiter_v3_fails_closed_on_unknown_descriptor(): """ from fastapi import HTTPException - os.environ["LITELLM_LICENSE"] = "test-license-key" + monkeypatch.setenv("LITELLM_LICENSE", "test-license-key") litellm.priority_reservation = {"high": 0.9, "low": 0.1} dual_cache = DualCache() diff --git a/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py b/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py index f5410ef0d70..91fff717d25 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py +++ b/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py @@ -1,13 +1,11 @@ import asyncio import os -import sys from pathlib import Path from unittest import mock import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.proxy.proxy_server import app, initialize diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py index 6970e34f759..9853ce7e1cf 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../") -) # Adds the parent directory to the system path from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, Member from litellm.proxy.management_endpoints.scim.scim_transformations import ( diff --git a/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py b/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py index a64397d9818..7b895cd7fdb 100644 --- a/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py +++ b/tests/test_litellm/proxy/management_endpoints/search_endpoints/test_search_tool_management.py @@ -1,6 +1,4 @@ import contextlib -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch @@ -8,9 +6,6 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index 016e10859b6..e8f768c14ef 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -2,8 +2,6 @@ Tests for access group management endpoints. """ -import os -import sys import types from contextlib import asynccontextmanager from datetime import datetime @@ -21,7 +19,6 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) -sys.path.insert(0, os.path.abspath("../../../")) def _make_access_group_record( diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py index c973c6a8346..db0557cfbf0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_management.py @@ -2,15 +2,10 @@ Test access group management endpoints """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm import Router from litellm.proxy.management_endpoints.model_management_endpoints import ( diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 5c61f8c557c..805168c84ac 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -2,8 +2,6 @@ Unit tests for auto router management endpoints """ -import os -import sys from pathlib import Path from typing import Final @@ -11,7 +9,6 @@ import pytest from fastapi import HTTPException from pydantic import ValidationError -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path from litellm.proxy._types import ( LitellmUserRoles, diff --git a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py index 6a9e894feb5..0bad0d24be5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py @@ -1,7 +1,5 @@ # tests/test_budget_endpoints.py -import os -import sys import types from datetime import datetime, timedelta, timezone import pytest @@ -12,9 +10,6 @@ import litellm.proxy.proxy_server as ps from litellm.proxy.proxy_server import app from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles, CommonProxyErrors -sys.path.insert( - 0, os.path.abspath("../../../") -) # Adds the parent directory to the system path @pytest.fixture diff --git a/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py index 2504b5744fc..9a2dd914866 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cache_settings_endpoints.py @@ -4,13 +4,10 @@ Unit tests for cache settings management endpoints import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path import litellm from litellm.proxy._types import LitellmTableNames, LitellmUserRoles diff --git a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py index dfc9f0361c6..b2a242bf8f2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_callback_management_endpoints.py @@ -1,13 +1,11 @@ import json import os -import sys from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) # from typing import cast diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 1491782419f..1bcb331430e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -1,5 +1,3 @@ -import os -import sys from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final @@ -9,7 +7,6 @@ import pytest from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path from litellm.proxy.management_endpoints.common_daily_activity import ( _adjust_dates_for_timezone, diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index 33e45ccb22c..dcbe515d5de 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -2,12 +2,9 @@ Unit tests for compliance check endpoints (EU AI Act and GDPR). """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path from litellm.proxy.compliance_checks import ComplianceChecker from litellm.types.proxy.compliance_endpoints import ComplianceCheckRequest diff --git a/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py index 2e78a4ca0e3..4481a87c9e7 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py @@ -4,14 +4,11 @@ Unit tests for coordination Redis settings management endpoints import asyncio import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path import litellm from litellm.caching.caching import RedisCache diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index ea86731eba4..e1eb031abc2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -4,14 +4,11 @@ Tests for cost tracking settings management endpoints. Tests the GET and PATCH endpoints for managing cost discount configuration. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.proxy.management_endpoints.cost_tracking_settings import router diff --git a/tests/test_litellm/proxy/management_endpoints/test_delete_callbacks_endpoint.py b/tests/test_litellm/proxy/management_endpoints/test_delete_callbacks_endpoint.py index 4ba656d1286..291f3d8fe2f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_delete_callbacks_endpoint.py +++ b/tests/test_litellm/proxy/management_endpoints/test_delete_callbacks_endpoint.py @@ -1,11 +1,8 @@ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import ( CallbackDelete, diff --git a/tests/test_litellm/proxy/management_endpoints/test_delete_verification_tokens_failed.py b/tests/test_litellm/proxy/management_endpoints/test_delete_verification_tokens_failed.py index 63e584e49bc..e33945df7dc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_delete_verification_tokens_failed.py +++ b/tests/test_litellm/proxy/management_endpoints/test_delete_verification_tokens_failed.py @@ -8,12 +8,9 @@ its result dict in all scenarios, populated with any token hashes that could not be deleted. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from unittest.mock import AsyncMock, MagicMock diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index da51d513b39..f86e17c61b0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,15 +1,10 @@ import json -import os -import sys from datetime import datetime, timezone from types import SimpleNamespace import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import ( LiteLLM_UserTableFiltered, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index fff6368cfc6..0c615cbaa32 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1,16 +1,10 @@ import json -import os -import sys import litellm import pytest import yaml from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path - from unittest.mock import AsyncMock, MagicMock, patch from fastapi import HTTPException @@ -8125,26 +8119,22 @@ async def test_default_key_generate_params_duration(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) # Set default_key_generate_params with duration - original_value = litellm.default_key_generate_params - litellm.default_key_generate_params = {"duration": "180d"} + monkeypatch.setattr(litellm, "default_key_generate_params", {"duration": "180d"}) - try: - request = GenerateKeyRequest() # No duration specified - response = await _common_key_generation_helper( - data=request, - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-1234", - user_id="1234", - ), - litellm_changed_by=None, - team_table=None, - ) + request = GenerateKeyRequest() # No duration specified + response = await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) - # Verify duration was applied from defaults - assert request.duration == "180d" - finally: - litellm.default_key_generate_params = original_value + # Verify duration was applied from defaults + assert request.duration == "180d" async def test_default_key_generate_params_object_permission_applied_when_absent( @@ -8184,28 +8174,28 @@ async def test_default_key_generate_params_object_permission_applied_when_absent monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - original_value = litellm.default_key_generate_params - litellm.default_key_generate_params = { - "object_permission": {"vector_stores": ["default-vs"]} - } + monkeypatch.setattr( + litellm, + "default_key_generate_params", + { + "object_permission": {"vector_stores": ["default-vs"]} + }, + ) - try: - request = GenerateKeyRequest() # No object_permission specified - await _common_key_generation_helper( - data=request, - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-1234", - user_id="1234", - ), - litellm_changed_by=None, - team_table=None, - ) + request = GenerateKeyRequest() # No object_permission specified + await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) - created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] - assert created_data["vector_stores"] == ["default-vs"] - finally: - litellm.default_key_generate_params = original_value + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["vector_stores"] == ["default-vs"] async def test_default_key_generate_params_object_permission_merges_partial( @@ -8247,31 +8237,31 @@ async def test_default_key_generate_params_object_permission_merges_partial( monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - original_value = litellm.default_key_generate_params - litellm.default_key_generate_params = { - "object_permission": {"vector_stores": ["default-vs"]} - } + monkeypatch.setattr( + litellm, + "default_key_generate_params", + { + "object_permission": {"vector_stores": ["default-vs"]} + }, + ) - try: - request = GenerateKeyRequest( - object_permission=LiteLLM_ObjectPermissionBase(agents=["agent-1"]) - ) - await _common_key_generation_helper( - data=request, - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-1234", - user_id="1234", - ), - litellm_changed_by=None, - team_table=None, - ) + request = GenerateKeyRequest( + object_permission=LiteLLM_ObjectPermissionBase(agents=["agent-1"]) + ) + await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) - created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] - assert created_data["agents"] == ["agent-1"] - assert created_data["vector_stores"] == ["default-vs"] - finally: - litellm.default_key_generate_params = original_value + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["agents"] == ["agent-1"] + assert created_data["vector_stores"] == ["default-vs"] async def test_default_key_generate_params_object_permission_does_not_override_explicit( @@ -8312,32 +8302,32 @@ async def test_default_key_generate_params_object_permission_does_not_override_e monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - original_value = litellm.default_key_generate_params - litellm.default_key_generate_params = { - "object_permission": {"vector_stores": ["default-vs"]} - } + monkeypatch.setattr( + litellm, + "default_key_generate_params", + { + "object_permission": {"vector_stores": ["default-vs"]} + }, + ) - try: - request = GenerateKeyRequest( - object_permission=LiteLLM_ObjectPermissionBase( - vector_stores=["explicit-vs"] - ) - ) - await _common_key_generation_helper( - data=request, - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-1234", - user_id="1234", - ), - litellm_changed_by=None, - team_table=None, + request = GenerateKeyRequest( + object_permission=LiteLLM_ObjectPermissionBase( + vector_stores=["explicit-vs"] ) + ) + await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="1234", + ), + litellm_changed_by=None, + team_table=None, + ) - created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] - assert created_data["vector_stores"] == ["explicit-vs"] - finally: - litellm.default_key_generate_params = original_value + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["vector_stores"] == ["explicit-vs"] async def test_default_key_generate_params_object_permission_not_rejected_for_non_admin_personal_key( @@ -8380,29 +8370,29 @@ async def test_default_key_generate_params_object_permission_not_rejected_for_no monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - original_value = litellm.default_key_generate_params - litellm.default_key_generate_params = { - "object_permission": {"vector_stores": ["default-vs"]} - } + monkeypatch.setattr( + litellm, + "default_key_generate_params", + { + "object_permission": {"vector_stores": ["default-vs"]} + }, + ) - try: - request = GenerateKeyRequest(user_id="alice") # No object_permission specified - response = await _common_key_generation_helper( - data=request, - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, - api_key="sk-alice", - user_id="alice", - ), - litellm_changed_by=None, - team_table=None, - ) + request = GenerateKeyRequest(user_id="alice") # No object_permission specified + response = await _common_key_generation_helper( + data=request, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + api_key="sk-alice", + user_id="alice", + ), + litellm_changed_by=None, + team_table=None, + ) - assert response is not None - created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] - assert created_data["vector_stores"] == ["default-vs"] - finally: - litellm.default_key_generate_params = original_value + assert response is not None + created_data = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] + assert created_data["vector_stores"] == ["default-vs"] @pytest.mark.asyncio @@ -9261,10 +9251,8 @@ async def test_key_aliases_admin_sees_all(): class TestValidateKeyAliasFormat: @pytest.fixture(autouse=True) - def reset_key_alias_flag(self): - litellm.enable_key_alias_format_validation = False - yield - litellm.enable_key_alias_format_validation = False + def reset_key_alias_flag(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "enable_key_alias_format_validation", False) def test_validation_skipped_when_flag_disabled(self): """When enable_key_alias_format_validation is False (default), no charset/length validation occurs.""" @@ -9305,12 +9293,12 @@ class TestValidateKeyAliasFormat: assert str(exc.value.code) == "400" assert "Invalid key_alias" in str(exc.value.message) - def test_validate_key_alias_format_valid(self): + def test_validate_key_alias_format_valid(self, monkeypatch): from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, ) - litellm.enable_key_alias_format_validation = True + monkeypatch.setattr(litellm, "enable_key_alias_format_validation", True) # Valid cases _validate_key_alias_format(None) # OK _validate_key_alias_format("valid-alias") @@ -9322,13 +9310,13 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format("user/user@example.com") _validate_key_alias_format("team/user@example.com") - def test_validate_key_alias_format_invalid(self): + def test_validate_key_alias_format_invalid(self, monkeypatch): from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, ) from litellm.proxy._types import ProxyException - litellm.enable_key_alias_format_validation = True + monkeypatch.setattr(litellm, "enable_key_alias_format_validation", True) invalid_aliases = [ "", # empty " ", # whitespace @@ -10956,10 +10944,8 @@ class TestKeyAliasSkipValidationOnUnchanged: """ @pytest.fixture(autouse=True) - def enable_validation(self): - litellm.enable_key_alias_format_validation = True - yield - litellm.enable_key_alias_format_validation = False + def enable_validation(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "enable_key_alias_format_validation", True) @pytest.fixture def mock_prisma(self): @@ -11075,146 +11061,142 @@ class TestKeyAliasSkipValidationOnUnchanged: # --- Tests: _enforce_upperbound_key_params --- -def test_enforce_upperbound_rejects_over_limit_on_generate(): +def test_enforce_upperbound_rejects_over_limit_on_generate(monkeypatch): """Test that key generation is rejected when values exceed upperbound.""" import litellm from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - tpm_limit=1000, rpm_limit=100, max_budget=10.0 - ) - data = GenerateKeyRequest(tpm_limit=5000) - with pytest.raises(HTTPException) as exc_info: - _enforce_upperbound_key_params(data, fill_defaults=True) - assert exc_info.value.status_code == 400 - assert "tpm_limit" in str(exc_info.value.detail) - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ), + ) + data = GenerateKeyRequest(tpm_limit=5000) + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=True) + assert exc_info.value.status_code == 400 + assert "tpm_limit" in str(exc_info.value.detail) -def test_enforce_upperbound_fills_defaults_on_generate(): +def test_enforce_upperbound_fills_defaults_on_generate(monkeypatch): """Test that None values are filled with upperbound defaults during generation.""" import litellm from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - tpm_limit=1000, rpm_limit=100 - ) - data = GenerateKeyRequest() # tpm_limit=None, rpm_limit=None - _enforce_upperbound_key_params(data, fill_defaults=True) - assert data.tpm_limit == 1000 - assert data.rpm_limit == 100 - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100 + ), + ) + data = GenerateKeyRequest() # tpm_limit=None, rpm_limit=None + _enforce_upperbound_key_params(data, fill_defaults=True) + assert data.tpm_limit == 1000 + assert data.rpm_limit == 100 -def test_enforce_upperbound_skips_none_on_update(): +def test_enforce_upperbound_skips_none_on_update(monkeypatch): """Test that None values are NOT filled during update (fill_defaults=False).""" import litellm from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - tpm_limit=1000, rpm_limit=100 - ) - data = UpdateKeyRequest(key="sk-test") # tpm_limit=None, rpm_limit=None - _enforce_upperbound_key_params(data, fill_defaults=False) - assert data.tpm_limit is None # should NOT be filled - assert data.rpm_limit is None # should NOT be filled - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100 + ), + ) + data = UpdateKeyRequest(key="sk-test") # tpm_limit=None, rpm_limit=None + _enforce_upperbound_key_params(data, fill_defaults=False) + assert data.tpm_limit is None # should NOT be filled + assert data.rpm_limit is None # should NOT be filled -def test_enforce_upperbound_rejects_over_limit_on_update(): +def test_enforce_upperbound_rejects_over_limit_on_update(monkeypatch): """Test that key update is rejected when values exceed upperbound.""" import litellm from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - tpm_limit=1000, rpm_limit=100, max_budget=10.0 - ) - data = UpdateKeyRequest(key="sk-test", tpm_limit=5000) - with pytest.raises(HTTPException) as exc_info: - _enforce_upperbound_key_params(data, fill_defaults=False) - assert exc_info.value.status_code == 400 - assert "tpm_limit" in str(exc_info.value.detail) - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ), + ) + data = UpdateKeyRequest(key="sk-test", tpm_limit=5000) + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=False) + assert exc_info.value.status_code == 400 + assert "tpm_limit" in str(exc_info.value.detail) -def test_enforce_upperbound_allows_within_limit_on_update(): +def test_enforce_upperbound_allows_within_limit_on_update(monkeypatch): """Test that key update passes when values are within upperbound.""" import litellm from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - tpm_limit=1000, rpm_limit=100, max_budget=10.0 - ) - data = UpdateKeyRequest( - key="sk-test", tpm_limit=500, rpm_limit=50, max_budget=5.0 - ) - _enforce_upperbound_key_params(data, fill_defaults=False) - # Should not raise - assert data.tpm_limit == 500 - assert data.rpm_limit == 50 - assert data.max_budget == 5.0 - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ), + ) + data = UpdateKeyRequest( + key="sk-test", tpm_limit=500, rpm_limit=50, max_budget=5.0 + ) + _enforce_upperbound_key_params(data, fill_defaults=False) + # Should not raise + assert data.tpm_limit == 500 + assert data.rpm_limit == 50 + assert data.max_budget == 5.0 -def test_enforce_upperbound_duration_over_limit(): +def test_enforce_upperbound_duration_over_limit(monkeypatch): """Test that duration exceeding upperbound is rejected.""" import litellm from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - duration="7d" - ) - data = UpdateKeyRequest(key="sk-test", duration="30d") - with pytest.raises(HTTPException) as exc_info: - _enforce_upperbound_key_params(data, fill_defaults=False) - assert exc_info.value.status_code == 400 - assert "duration" in str(exc_info.value.detail) - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + duration="7d" + ), + ) + data = UpdateKeyRequest(key="sk-test", duration="30d") + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=False) + assert exc_info.value.status_code == 400 + assert "duration" in str(exc_info.value.detail) -def test_enforce_upperbound_no_config_is_noop(): +def test_enforce_upperbound_no_config_is_noop(monkeypatch): """Test that no enforcement happens when upperbound params are not configured.""" import litellm - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = None - data = UpdateKeyRequest(key="sk-test", tpm_limit=999999) - _enforce_upperbound_key_params(data, fill_defaults=False) - # Should not raise — no enforcement configured - assert data.tpm_limit == 999999 - finally: - litellm.upperbound_key_generate_params = original + monkeypatch.setattr(litellm, "upperbound_key_generate_params", None) + data = UpdateKeyRequest(key="sk-test", tpm_limit=999999) + _enforce_upperbound_key_params(data, fill_defaults=False) + # Should not raise — no enforcement configured + assert data.tpm_limit == 999999 # --- Tests: _execute_virtual_key_regeneration enforces upperbound --- @@ -11267,7 +11249,7 @@ def _make_regenerate_existing_key(): @pytest.mark.asyncio -async def test_execute_virtual_key_regeneration_rejects_over_limit_duration(): +async def test_execute_virtual_key_regeneration_rejects_over_limit_duration(monkeypatch): """Regenerate must reject durations exceeding upperbound_key_generate_params.duration.""" from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -11277,91 +11259,34 @@ async def test_execute_virtual_key_regeneration_rejects_over_limit_duration(): LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - duration="1h" - ) - existing_key = _make_regenerate_existing_key() - data = RegenerateKeyRequest(duration="2h") - user_api_key_dict = _make_regenerate_user_api_key_dict() - mock_prisma_client = _make_regenerate_mock_prisma() - - with ( - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", - new_callable=AsyncMock, - return_value="sk-newtoken1234ab12", - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", - new_callable=AsyncMock, - ), - ): - with pytest.raises(HTTPException) as exc_info: - await _execute_virtual_key_regeneration( - prisma_client=mock_prisma_client, - key_in_db=existing_key, - hashed_api_key="abc123", - key="abc123", - data=data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - user_api_key_cache=MagicMock(), - proxy_logging_obj=MagicMock(), - ) - assert exc_info.value.status_code == 400 - assert "duration" in str(exc_info.value.detail) - # Rejected regenerate must not reach the DB update. - assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 - finally: - litellm.upperbound_key_generate_params = original - - -@pytest.mark.asyncio -async def test_execute_virtual_key_regeneration_allows_within_limit_duration(): - """Regenerate must accept durations within upperbound_key_generate_params.duration.""" - from litellm.proxy._types import RegenerateKeyRequest - from litellm.proxy.management_endpoints.key_management_endpoints import ( - _execute_virtual_key_regeneration, - ) - from litellm.types.proxy.management_endpoints.ui_sso import ( - LiteLLM_UpperboundKeyGenerateParams, + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + duration="1h" + ), ) + existing_key = _make_regenerate_existing_key() + data = RegenerateKeyRequest(duration="2h") + user_api_key_dict = _make_regenerate_user_api_key_dict() + mock_prisma_client = _make_regenerate_mock_prisma() - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - duration="1h" - ) - existing_key = _make_regenerate_existing_key() - data = RegenerateKeyRequest(duration="30m") - user_api_key_dict = _make_regenerate_user_api_key_dict() - mock_prisma_client = _make_regenerate_mock_prisma() - - with ( - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", - new_callable=AsyncMock, - return_value="sk-newtoken1234ab12", - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", - new_callable=AsyncMock, - ), - ): + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + ): + with pytest.raises(HTTPException) as exc_info: await _execute_virtual_key_regeneration( prisma_client=mock_prisma_client, key_in_db=existing_key, @@ -11373,13 +11298,70 @@ async def test_execute_virtual_key_regeneration_allows_within_limit_duration(): user_api_key_cache=MagicMock(), proxy_logging_obj=MagicMock(), ) - assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 - finally: - litellm.upperbound_key_generate_params = original + assert exc_info.value.status_code == 400 + assert "duration" in str(exc_info.value.detail) + # Rejected regenerate must not reach the DB update. + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 @pytest.mark.asyncio -async def test_execute_virtual_key_regeneration_rejects_over_limit_max_budget(): +async def test_execute_virtual_key_regeneration_allows_within_limit_duration(monkeypatch): + """Regenerate must accept durations within upperbound_key_generate_params.duration.""" + from litellm.proxy._types import RegenerateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _execute_virtual_key_regeneration, + ) + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + duration="1h" + ), + ) + existing_key = _make_regenerate_existing_key() + data = RegenerateKeyRequest(duration="30m") + user_api_key_dict = _make_regenerate_user_api_key_dict() + mock_prisma_client = _make_regenerate_mock_prisma() + + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", + new_callable=AsyncMock, + ), + ): + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=existing_key, + hashed_api_key="abc123", + key="abc123", + data=data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 + + +@pytest.mark.asyncio +async def test_execute_virtual_key_regeneration_rejects_over_limit_max_budget(monkeypatch): """Regenerate must reject max_budget exceeding upperbound — proves the fix covers non-duration fields.""" from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -11389,52 +11371,52 @@ async def test_execute_virtual_key_regeneration_rejects_over_limit_max_budget(): LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - max_budget=10.0 - ) - existing_key = _make_regenerate_existing_key() - data = RegenerateKeyRequest(max_budget=500.0) - user_api_key_dict = _make_regenerate_user_api_key_dict() - mock_prisma_client = _make_regenerate_mock_prisma() + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + max_budget=10.0 + ), + ) + existing_key = _make_regenerate_existing_key() + data = RegenerateKeyRequest(max_budget=500.0) + user_api_key_dict = _make_regenerate_user_api_key_dict() + mock_prisma_client = _make_regenerate_mock_prisma() - with ( - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", - new_callable=AsyncMock, - return_value="sk-newtoken1234ab12", - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", - new_callable=AsyncMock, - ), - ): - with pytest.raises(HTTPException) as exc_info: - await _execute_virtual_key_regeneration( - prisma_client=mock_prisma_client, - key_in_db=existing_key, - hashed_api_key="abc123", - key="abc123", - data=data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - user_api_key_cache=MagicMock(), - proxy_logging_obj=MagicMock(), - ) - assert exc_info.value.status_code == 400 - assert "max_budget" in str(exc_info.value.detail) - assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 - finally: - litellm.upperbound_key_generate_params = original + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + ): + with pytest.raises(HTTPException) as exc_info: + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=existing_key, + hashed_api_key="abc123", + key="abc123", + data=data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert exc_info.value.status_code == 400 + assert "max_budget" in str(exc_info.value.detail) + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 @pytest.mark.asyncio -async def test_execute_virtual_key_regeneration_skips_none_values(): +async def test_execute_virtual_key_regeneration_skips_none_values(monkeypatch): """Regenerate with data.duration=None must not raise, even when upperbound is set (fill_defaults=False semantic — None means 'inherit from existing key').""" from litellm.proxy._types import RegenerateKeyRequest @@ -11445,100 +11427,96 @@ async def test_execute_virtual_key_regeneration_skips_none_values(): LiteLLM_UpperboundKeyGenerateParams, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( - duration="1h" - ) - existing_key = _make_regenerate_existing_key() - data = RegenerateKeyRequest() # all fields None - user_api_key_dict = _make_regenerate_user_api_key_dict() - mock_prisma_client = _make_regenerate_mock_prisma() + monkeypatch.setattr( + litellm, + "upperbound_key_generate_params", + LiteLLM_UpperboundKeyGenerateParams( + duration="1h" + ), + ) + existing_key = _make_regenerate_existing_key() + data = RegenerateKeyRequest() # all fields None + user_api_key_dict = _make_regenerate_user_api_key_dict() + mock_prisma_client = _make_regenerate_mock_prisma() - with ( - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", - new_callable=AsyncMock, - return_value="sk-newtoken1234ab12", - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", - new_callable=AsyncMock, - ), - ): - await _execute_virtual_key_regeneration( - prisma_client=mock_prisma_client, - key_in_db=existing_key, - hashed_api_key="abc123", - key="abc123", - data=data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - user_api_key_cache=MagicMock(), - proxy_logging_obj=MagicMock(), - ) - assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 - finally: - litellm.upperbound_key_generate_params = original + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", + new_callable=AsyncMock, + ), + ): + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=existing_key, + hashed_api_key="abc123", + key="abc123", + data=data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 @pytest.mark.asyncio -async def test_execute_virtual_key_regeneration_no_upperbound_config_is_noop(): +async def test_execute_virtual_key_regeneration_no_upperbound_config_is_noop(monkeypatch): """Regenerate with no upperbound config set must accept any duration.""" from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( _execute_virtual_key_regeneration, ) - original = litellm.upperbound_key_generate_params - try: - litellm.upperbound_key_generate_params = None - existing_key = _make_regenerate_existing_key() - data = RegenerateKeyRequest(duration="30d") - user_api_key_dict = _make_regenerate_user_api_key_dict() - mock_prisma_client = _make_regenerate_mock_prisma() + monkeypatch.setattr(litellm, "upperbound_key_generate_params", None) + existing_key = _make_regenerate_existing_key() + data = RegenerateKeyRequest(duration="30d") + user_api_key_dict = _make_regenerate_user_api_key_dict() + mock_prisma_client = _make_regenerate_mock_prisma() - with ( - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", - new_callable=AsyncMock, - return_value="sk-newtoken1234ab12", - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", - new_callable=AsyncMock, - ), - patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", - new_callable=AsyncMock, - ), - ): - await _execute_virtual_key_regeneration( - prisma_client=mock_prisma_client, - key_in_db=existing_key, - hashed_api_key="abc123", - key="abc123", - data=data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - user_api_key_cache=MagicMock(), - proxy_logging_obj=MagicMock(), - ) - assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 - finally: - litellm.upperbound_key_generate_params = original + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_rotated_hook", + new_callable=AsyncMock, + ), + ): + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=existing_key, + hashed_api_key="abc123", + key="abc123", + data=data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 1 class TestAllowedRoutesCallerPermission: diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index f8db8433be5..0d639e1cb6a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -17,7 +17,6 @@ from litellm.proxy.management_endpoints import ( mcp_management_endpoints as mgmt_endpoints, ) -sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path from litellm.proxy._types import ( LiteLLM_MCPServerTable, @@ -6482,7 +6481,6 @@ def test_bundled_openapi_registry_parses_and_entries_are_well_formed(): authorization_url would recreate the exact 400 ("authorization url is not set") the catalog exists to prevent for spec-only servers, which never run OAuth endpoint discovery.""" import json - import os registry_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 42e96ad8659..097230108d4 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys from typing import Dict, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -10,9 +8,6 @@ from fastapi.testclient import TestClient from litellm._uuid import uuid -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import ( LiteLLM_ModelTable, LiteLLM_ProxyModelTable, diff --git a/tests/test_litellm/proxy/management_endpoints/test_org_admin_team_access.py b/tests/test_litellm/proxy/management_endpoints/test_org_admin_team_access.py index 9828d104a8b..d5c958f9f84 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_org_admin_team_access.py +++ b/tests/test_litellm/proxy/management_endpoints/test_org_admin_team_access.py @@ -7,14 +7,11 @@ Covers: - _user_is_org_admin route-level check (no privilege escalation) """ -import os -import sys from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../")) from litellm.proxy._types import ( LiteLLM_OrganizationMembershipTable, diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index 3061da336f6..a62c98e56a7 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys from litellm._uuid import uuid from typing import Optional, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -10,7 +8,6 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../")) # Adds the parent directory to the system path @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py index b62f077a62e..308f4d88f02 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py @@ -4,14 +4,11 @@ Tests for router settings management endpoints. Tests the GET endpoints for router settings and router fields. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py index 018979aa19b..71c67837515 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py @@ -1,7 +1,5 @@ import inspect import json -import os -import sys from collections.abc import Sequence from typing import Optional @@ -10,9 +8,6 @@ from fastapi import HTTPException from fastapi.testclient import TestClient from prisma.actions import LiteLLM_VerificationTokenActions -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from contextlib import contextmanager from unittest.mock import AsyncMock, Mock, patch diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py index a485d95db06..265437f97e9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py @@ -3,14 +3,11 @@ Tests for applying default team params during team creation and loading default_team_params from DB on startup. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../")) # Adds the parent directory to the system path import litellm from litellm.proxy._types import ( diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a4c2b7c06bf..34b12aecfed 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys from contextlib import asynccontextmanager from datetime import datetime, timezone from types import SimpleNamespace @@ -14,9 +12,6 @@ from fastapi.testclient import TestClient from litellm._uuid import uuid -sys.path.insert( - 0, os.path.abspath("../../../") -) # Adds the parent directory to the system path from litellm.proxy._types import UserAPIKeyAuth # Import UserAPIKeyAuth from litellm.proxy._types import ( LiteLLM_BudgetTableFull, diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_model_alias_merge.py b/tests/test_litellm/proxy/management_endpoints/test_team_model_alias_merge.py index 45405ba78d6..7cdf60f043e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_model_alias_merge.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_model_alias_merge.py @@ -6,13 +6,10 @@ Concurrent BYOK model creates must not overwrite each other's entries in team.models. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import ( LitellmUserRoles, diff --git a/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py index 18ea5c3f27d..09d14cfe5df 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_tool_management_endpoints.py @@ -8,8 +8,6 @@ imports these inside function bodies to avoid circular imports. """ import inspect -import os -import sys from collections.abc import Sequence from datetime import datetime, timedelta, timezone from typing import Optional @@ -20,7 +18,6 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from prisma.actions import LiteLLM_TeamTableActions -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.management_endpoints.tool_management_endpoints import router from litellm.types.tool_management import LiteLLM_ToolTableRow diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 66cb07ef2c0..3facbf07889 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -1,7 +1,6 @@ import asyncio import json import os -import sys from contextlib import asynccontextmanager from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -11,9 +10,6 @@ from fastapi import HTTPException, Request from litellm._uuid import uuid -sys.path.insert( - 0, os.path.abspath("../../../") -) # Adds the parent directory to the system path import litellm from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse diff --git a/tests/test_litellm/proxy/management_endpoints/test_workflow_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_workflow_management_endpoints.py index 27adb3e0892..0c3d5107cb6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_workflow_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_workflow_management_endpoints.py @@ -4,8 +4,6 @@ Uses FastAPI TestClient with a mocked prisma_client. """ import asyncio -import os -import sys from datetime import datetime, timezone from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -15,7 +13,6 @@ from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient from prisma.errors import UniqueViolationError -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.management_endpoints.workflow_management_endpoints import ( _read_scope_caller, diff --git a/tests/test_litellm/proxy/management_helpers/test_access_group_team_sync.py b/tests/test_litellm/proxy/management_helpers/test_access_group_team_sync.py index eb11292cf42..f99b576019a 100644 --- a/tests/test_litellm/proxy/management_helpers/test_access_group_team_sync.py +++ b/tests/test_litellm/proxy/management_helpers/test_access_group_team_sync.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy.management_helpers.access_group_team_sync import ( invalidate_access_group_caches, diff --git a/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py b/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py index 2d54d249713..b1d111bf1f9 100644 --- a/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py +++ b/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py @@ -25,12 +25,9 @@ from litellm.types.utils import StandardAuditLogPayload @pytest.fixture(autouse=True) -def reset_audit_log_callbacks(): - """Reset audit_log_callbacks before and after each test.""" - original = litellm.audit_log_callbacks - litellm.audit_log_callbacks = [] - yield - litellm.audit_log_callbacks = original +def reset_audit_log_callbacks(monkeypatch: pytest.MonkeyPatch) -> None: + """Every test starts with no audit log callbacks registered.""" + monkeypatch.setattr(litellm, "audit_log_callbacks", []) def _make_audit_log( @@ -115,10 +112,10 @@ class TestBuildAuditLogPayload: class TestDispatchAuditLogToCallbacks: @pytest.mark.asyncio - async def test_dispatches_to_custom_logger_instance(self): + async def test_dispatches_to_custom_logger_instance(self, monkeypatch: pytest.MonkeyPatch): mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) audit_log = _make_audit_log() await _dispatch_audit_log_to_callbacks(audit_log) @@ -132,18 +129,18 @@ class TestDispatchAuditLogToCallbacks: assert payload["action"] == "created" @pytest.mark.asyncio - async def test_no_dispatch_when_callbacks_empty(self): - litellm.audit_log_callbacks = [] + async def test_no_dispatch_when_callbacks_empty(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "audit_log_callbacks", []) audit_log = _make_audit_log() # Should return immediately without error await _dispatch_audit_log_to_callbacks(audit_log) @pytest.mark.asyncio - async def test_resolves_string_callback(self): + async def test_resolves_string_callback(self, monkeypatch: pytest.MonkeyPatch): mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = ["s3_v2"] + monkeypatch.setattr(litellm, "audit_log_callbacks", ["s3_v2"]) with patch( "litellm.proxy.management_helpers.audit_logs._resolve_audit_log_callback", @@ -156,13 +153,13 @@ class TestDispatchAuditLogToCallbacks: mock_logger.async_log_audit_log_event.assert_called_once() @pytest.mark.asyncio - async def test_nonblocking_on_callback_failure(self): + async def test_nonblocking_on_callback_failure(self, monkeypatch: pytest.MonkeyPatch): """Callback errors should not propagate.""" mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock( side_effect=RuntimeError("boom") ) - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) audit_log = _make_audit_log() # Should not raise @@ -170,8 +167,8 @@ class TestDispatchAuditLogToCallbacks: await asyncio.sleep(0.1) @pytest.mark.asyncio - async def test_skips_unresolvable_string_callback(self): - litellm.audit_log_callbacks = ["nonexistent_callback"] + async def test_skips_unresolvable_string_callback(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "audit_log_callbacks", ["nonexistent_callback"]) with patch( "litellm.proxy.management_helpers.audit_logs._resolve_audit_log_callback", @@ -184,10 +181,10 @@ class TestDispatchAuditLogToCallbacks: class TestCreateAuditLogForUpdateWithCallbacks: @pytest.mark.asyncio - async def test_dispatches_to_callbacks_after_db_write(self): + async def test_dispatches_to_callbacks_after_db_write(self, monkeypatch: pytest.MonkeyPatch): mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) with ( patch("litellm.proxy.proxy_server.premium_user", True), @@ -206,10 +203,10 @@ class TestCreateAuditLogForUpdateWithCallbacks: mock_logger.async_log_audit_log_event.assert_called_once() @pytest.mark.asyncio - async def test_no_dispatch_when_not_premium(self): + async def test_no_dispatch_when_not_premium(self, monkeypatch: pytest.MonkeyPatch): mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) with ( patch("litellm.proxy.proxy_server.premium_user", False), @@ -224,10 +221,10 @@ class TestCreateAuditLogForUpdateWithCallbacks: mock_prisma.db.litellm_auditlog.create.assert_not_called() @pytest.mark.asyncio - async def test_no_dispatch_when_store_audit_logs_false(self): + async def test_no_dispatch_when_store_audit_logs_false(self, monkeypatch: pytest.MonkeyPatch): mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) with patch("litellm.store_audit_logs", False): audit_log = _make_audit_log() @@ -237,11 +234,11 @@ class TestCreateAuditLogForUpdateWithCallbacks: mock_logger.async_log_audit_log_event.assert_not_called() @pytest.mark.asyncio - async def test_dispatches_even_when_prisma_client_is_none(self): + async def test_dispatches_even_when_prisma_client_is_none(self, monkeypatch: pytest.MonkeyPatch): """Callbacks should fire even if DB is unavailable.""" mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) with ( patch("litellm.proxy.proxy_server.premium_user", True), @@ -256,11 +253,11 @@ class TestCreateAuditLogForUpdateWithCallbacks: mock_logger.async_log_audit_log_event.assert_called_once() @pytest.mark.asyncio - async def test_dispatches_even_when_db_write_fails(self): + async def test_dispatches_even_when_db_write_fails(self, monkeypatch: pytest.MonkeyPatch): """Callbacks should fire even if the DB write raises.""" mock_logger = MagicMock(spec=CustomLogger) mock_logger.async_log_audit_log_event = AsyncMock() - litellm.audit_log_callbacks = [mock_logger] + monkeypatch.setattr(litellm, "audit_log_callbacks", [mock_logger]) with ( patch("litellm.proxy.proxy_server.premium_user", True), @@ -384,21 +381,21 @@ class TestS3AuditCallbackParamsDecoupling: S3Logger instance, distinct from the singleton serving normal logs.""" @pytest.fixture(autouse=True) - def _isolate_caches_and_globals(self): + def _isolate_caches_and_globals(self, monkeypatch: pytest.MonkeyPatch): from litellm.litellm_core_utils import litellm_logging as ll_logging from litellm.proxy.management_helpers import audit_logs as ll_audit_logs - original_s3 = litellm.s3_callback_params - original_audit = getattr(litellm, "s3_audit_callback_params", None) + monkeypatch.setattr(litellm, "s3_callback_params", litellm.s3_callback_params) + monkeypatch.setattr( + litellm, "s3_audit_callback_params", getattr(litellm, "s3_audit_callback_params", None) + ) ll_audit_logs._audit_log_callback_cache.clear() ll_logging._in_memory_loggers.clear() yield - litellm.s3_callback_params = original_s3 - litellm.s3_audit_callback_params = original_audit ll_audit_logs._audit_log_callback_cache.clear() ll_logging._in_memory_loggers.clear() - def test_opt_in_constructs_separate_instance_with_audit_config(self): + def test_opt_in_constructs_separate_instance_with_audit_config(self, monkeypatch: pytest.MonkeyPatch): """Audit config set → audit resolver returns a fresh S3Logger pointing at the audit bucket, distinct from the normal-log singleton.""" from litellm.integrations.s3_v2 import S3Logger @@ -409,8 +406,8 @@ class TestS3AuditCallbackParamsDecoupling: _resolve_audit_log_callback, ) - litellm.s3_callback_params = {"s3_bucket_name": "normal-bucket"} - litellm.s3_audit_callback_params = {"s3_bucket_name": "audit-bucket"} + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-bucket"}) + monkeypatch.setattr(litellm, "s3_audit_callback_params", {"s3_bucket_name": "audit-bucket"}) with patch("asyncio.create_task"): audit_instance = _resolve_audit_log_callback("s3_v2") @@ -426,7 +423,7 @@ class TestS3AuditCallbackParamsDecoupling: assert audit_instance.s3_bucket_name == "audit-bucket" assert normal_instance.s3_bucket_name == "normal-bucket" - def test_opt_out_preserves_singleton_behavior(self): + def test_opt_out_preserves_singleton_behavior(self, monkeypatch: pytest.MonkeyPatch): """No `s3_audit_callback_params` → audit and normal share the singleton (existing behavior, regression guard).""" from litellm.integrations.s3_v2 import S3Logger @@ -437,8 +434,8 @@ class TestS3AuditCallbackParamsDecoupling: _resolve_audit_log_callback, ) - litellm.s3_callback_params = {"s3_bucket_name": "shared-bucket"} - litellm.s3_audit_callback_params = None + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "shared-bucket"}) + monkeypatch.setattr(litellm, "s3_audit_callback_params", None) with patch("asyncio.create_task"): normal_instance = _init_custom_logger_compatible_class( @@ -452,7 +449,7 @@ class TestS3AuditCallbackParamsDecoupling: assert id(audit_instance) == id(normal_instance) assert audit_instance.s3_bucket_name == "shared-bucket" - def test_empty_dict_opts_in(self): + def test_empty_dict_opts_in(self, monkeypatch: pytest.MonkeyPatch): """`s3_audit_callback_params = {}` is opt-in (truthy-by-presence) and produces a separate instance with no bucket configured (env/IAM-only).""" from litellm.integrations.s3_v2 import S3Logger @@ -463,8 +460,8 @@ class TestS3AuditCallbackParamsDecoupling: _resolve_audit_log_callback, ) - litellm.s3_callback_params = {"s3_bucket_name": "normal-bucket"} - litellm.s3_audit_callback_params = {} + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "normal-bucket"}) + monkeypatch.setattr(litellm, "s3_audit_callback_params", {}) with patch("asyncio.create_task"): audit_instance = _resolve_audit_log_callback("s3_v2") @@ -478,7 +475,7 @@ class TestS3AuditCallbackParamsDecoupling: assert audit_instance.s3_bucket_name is None assert normal_instance.s3_bucket_name == "normal-bucket" - def test_reset_audit_log_callback_cache_clears_audit_instance(self): + def test_reset_audit_log_callback_cache_clears_audit_instance(self, monkeypatch: pytest.MonkeyPatch): """`reset_audit_log_callback_cache()` must drop the cached audit instance so a config reload picks up the new params.""" from litellm.proxy.management_helpers.audit_logs import ( @@ -487,7 +484,7 @@ class TestS3AuditCallbackParamsDecoupling: reset_audit_log_callback_cache, ) - litellm.s3_audit_callback_params = {"s3_bucket_name": "first"} + monkeypatch.setattr(litellm, "s3_audit_callback_params", {"s3_bucket_name": "first"}) with patch("asyncio.create_task"): first = _resolve_audit_log_callback("s3_v2") assert first is not None and "s3_v2" in _audit_log_callback_cache @@ -495,7 +492,7 @@ class TestS3AuditCallbackParamsDecoupling: reset_audit_log_callback_cache() assert "s3_v2" not in _audit_log_callback_cache - litellm.s3_audit_callback_params = {"s3_bucket_name": "second"} + monkeypatch.setattr(litellm, "s3_audit_callback_params", {"s3_bucket_name": "second"}) second = _resolve_audit_log_callback("s3_v2") assert second is not None assert id(second) != id(first) diff --git a/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py b/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py index 504414ea635..bdc2f9065b9 100644 --- a/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py @@ -1,15 +1,10 @@ import json -import os -import sys from datetime import datetime, timezone from litellm._uuid import uuid from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import ( LiteLLM_TeamMembership, diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index d797a27aa67..b129ad0f659 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -1,11 +1,8 @@ import json -import os -import sys import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../..")) from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/test_litellm/proxy/management_helpers/test_team_member_permission_checks.py b/tests/test_litellm/proxy/management_helpers/test_team_member_permission_checks.py index 71999e29f96..36c61eddbb2 100644 --- a/tests/test_litellm/proxy/management_helpers/test_team_member_permission_checks.py +++ b/tests/test_litellm/proxy/management_helpers/test_team_member_permission_checks.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import MagicMock import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import KeyManagementRoutes, Member, ProxyException from litellm.proxy.management_helpers.team_member_permission_checks import ( diff --git a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py index 1acb8e7e016..dfb834dc31f 100644 --- a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py +++ b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py @@ -1,12 +1,9 @@ import asyncio -import os -import sys from unittest.mock import patch import pytest from fastapi import HTTPException -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.management_helpers.team_metadata_validation import ( diff --git a/tests/test_litellm/proxy/memory/test_memory_endpoints.py b/tests/test_litellm/proxy/memory/test_memory_endpoints.py index ec81ef2ff7a..3d99a600a73 100644 --- a/tests/test_litellm/proxy/memory/test_memory_endpoints.py +++ b/tests/test_litellm/proxy/memory/test_memory_endpoints.py @@ -7,8 +7,6 @@ We patch the endpoint module's `_require_prisma` helper so we never need the real proxy_server import chain (which pulls heavy optional deps). """ -import os -import sys from datetime import datetime, timezone from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, patch @@ -16,7 +14,6 @@ from unittest.mock import MagicMock, patch from fastapi import FastAPI from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.memory.memory_endpoints import _visibility_filter, router diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py index 6ffb7daaa2d..2161e345b40 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_common_utils.py @@ -1,11 +1,8 @@ -import os -import sys from types import MappingProxyType from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy.openai_files_endpoints.common_utils import ( apply_unified_file_ids, diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index bf9323cdc6a..237a3092035 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -1,6 +1,4 @@ import json -import os -import sys from typing import List from unittest.mock import ANY, AsyncMock @@ -10,9 +8,6 @@ import httpx from fastapi.testclient import TestClient from pytest_mock import MockerFixture -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm import Router diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 7985faa9e4b..8163d009fef 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -1,16 +1,11 @@ import asyncio import json -import os -import sys from datetime import datetime from typing import Any, Dict, List from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py index 6d7011fe10c..814c1a14f3d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py @@ -1,13 +1,10 @@ import json -import os -import sys from datetime import datetime from unittest.mock import MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.cohere_passthrough_logging_handler import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_comprehend_medical_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_comprehend_medical_passthrough_logging_handler.py index 1804877e688..20ec78cc8de 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_comprehend_medical_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_comprehend_medical_passthrough_logging_handler.py @@ -1,12 +1,9 @@ -import os -import sys from datetime import datetime from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.comprehend_medical_passthrough_logging_handler import ( ComprehendMedicalPassthroughLoggingHandler, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cursor_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cursor_passthrough_logging_handler.py index 2d025a871b7..af2bb1c816e 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cursor_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cursor_passthrough_logging_handler.py @@ -1,12 +1,9 @@ -import os -import sys from datetime import datetime from unittest.mock import MagicMock import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.cursor_passthrough_logging_handler import ( CursorPassthroughLoggingHandler, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py index fae6b6122f5..61d1caacb91 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py @@ -1,15 +1,10 @@ import json -import os -import sys from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index 69819318800..f0b2feeb377 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -1,6 +1,4 @@ import json -import os -import sys from datetime import datetime from typing import Any, Dict, List from unittest.mock import AsyncMock, MagicMock, patch @@ -8,7 +6,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 6568f6aeacf..ac140abe31f 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1,7 +1,6 @@ import contextlib import json import os -import sys import traceback from collections.abc import Mapping from types import MappingProxyType, SimpleNamespace @@ -14,9 +13,6 @@ import pytest from fastapi import HTTPException, Request, Response from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 4c6ba23c88c..25d176e48bb 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -2,7 +2,6 @@ import asyncio import json import logging import os -import sys from contextlib import ExitStack, contextmanager from io import BytesIO from types import SimpleNamespace @@ -15,7 +14,6 @@ from fastapi import Request, UploadFile from starlette.datastructures import FormData, Headers, QueryParams from starlette.datastructures import UploadFile as StarletteUploadFile -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_auth_default.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_auth_default.py index 4cac1cb4d3b..44a75c362e5 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_auth_default.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_auth_default.py @@ -19,14 +19,11 @@ defaults to ``True`` so a config dict (raw, not Pydantic) without an ``auth`` key still requires authentication. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import PassThroughGenericEndpoint from litellm.proxy.auth.user_api_key_auth import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py index 797b22784ae..37d2141e460 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py @@ -1,6 +1,4 @@ import json -import os -import sys import traceback from unittest import mock from unittest.mock import MagicMock, patch @@ -12,9 +10,6 @@ from fastapi.testclient import TestClient from litellm.passthrough.utils import CommonUtils -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from unittest.mock import Mock diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py index 84856fcb0b1..23f258f0362 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_guardrails_field_targeting.py @@ -6,13 +6,10 @@ and send only specified fields to the guardrail. """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.proxy._types import PassThroughGuardrailSettings from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_upstream_usage_headers.py b/tests/test_litellm/proxy/pass_through_endpoints/test_upstream_usage_headers.py index 34c345b620f..701c583a4ab 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_upstream_usage_headers.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_upstream_usage_headers.py @@ -1,10 +1,7 @@ -import os -import sys import httpx import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.upstream_usage_headers import ( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_watsonx_proxy_route.py b/tests/test_litellm/proxy/pass_through_endpoints/test_watsonx_proxy_route.py index 19a2f7a0506..5500bb0aad9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_watsonx_proxy_route.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_watsonx_proxy_route.py @@ -6,16 +6,11 @@ and version parameter injection. """ import json -import os -import sys from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest from fastapi import HTTPException, Request, Response -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py index 840d93eb12c..22d212dd8ae 100644 --- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py @@ -165,7 +165,7 @@ class ContentCheckGuardrail(CustomGuardrail): @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_escalation_step1_fails_step2_blocks(): +async def test_escalation_step1_fails_step2_blocks(monkeypatch): """ Pipeline: simple-filter (on_fail: next) -> advanced-filter (on_fail: block) Input: request that fails simple-filter @@ -182,36 +182,32 @@ async def test_escalation_step1_fails_step2_blocks(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [simple_guard, advanced_guard] + monkeypatch.setattr(litellm, "callbacks", [simple_guard, advanced_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "bad content"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "bad content"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert simple_guard.calls == 1 - assert advanced_guard.calls == 1 - assert result.terminal_action == "block" - assert len(result.step_results) == 2 - assert result.step_results[0].guardrail_name == "simple-filter" - assert result.step_results[0].outcome == "fail" - assert result.step_results[0].action_taken == "next" - assert result.step_results[1].guardrail_name == "advanced-filter" - assert result.step_results[1].outcome == "fail" - assert result.step_results[1].action_taken == "block" - finally: - litellm.callbacks = original_callbacks + assert simple_guard.calls == 1 + assert advanced_guard.calls == 1 + assert result.terminal_action == "block" + assert len(result.step_results) == 2 + assert result.step_results[0].guardrail_name == "simple-filter" + assert result.step_results[0].outcome == "fail" + assert result.step_results[0].action_taken == "next" + assert result.step_results[1].guardrail_name == "advanced-filter" + assert result.step_results[1].outcome == "fail" + assert result.step_results[1].action_taken == "block" @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_block_carries_original_guardrail_exception(): +async def test_block_carries_original_guardrail_exception(monkeypatch): """A blocking step must expose the guardrail's own raised exception on the result so the caller can re-raise it verbatim, giving the policy path the same response/trace as a direct guardrail attachment.""" @@ -219,67 +215,52 @@ async def test_block_carries_original_guardrail_exception(): pipeline = GuardrailPipeline( mode="pre_call", - steps=[ - PipelineStep( - guardrail="moderation-filter", on_fail="block", on_pass="allow" - ) - ], + steps=[PipelineStep(guardrail="moderation-filter", on_fail="block", on_pass="allow")], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [guard] + monkeypatch.setattr(litellm, "callbacks", [guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "bad content"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "bad content"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert result.terminal_action == "block" - assert isinstance(result.original_exception, HTTPException) - assert result.original_exception.status_code == 400 - assert result.original_exception.detail == "Content policy violation" - finally: - litellm.callbacks = original_callbacks + assert result.terminal_action == "block" + assert isinstance(result.original_exception, HTTPException) + assert result.original_exception.status_code == 400 + assert result.original_exception.detail == "Content policy violation" @pytest.mark.asyncio -async def test_unsupported_mode_yields_error_outcome_without_exception(): +async def test_unsupported_mode_yields_error_outcome_without_exception(monkeypatch): """An unexpected hook mode must surface as an error outcome (carrying no original exception), not crash or run the guardrail.""" guard = AlwaysPassGuardrail(guardrail_name="filter") - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [guard] + monkeypatch.setattr(litellm, "callbacks", [guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=[PipelineStep(guardrail="filter", on_error="block", on_fail="block")], - mode="during_call", - data={"messages": [{"role": "user", "content": "hi"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=[PipelineStep(guardrail="filter", on_error="block", on_fail="block")], + mode="during_call", + data={"messages": [{"role": "user", "content": "hi"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert guard.calls == 0 - assert result.terminal_action == "block" - assert result.step_results[0].outcome == "error" - assert ( - "Unsupported pipeline mode: during_call" - in result.step_results[0].error_detail - ) - assert result.original_exception is None - finally: - litellm.callbacks = original_callbacks + assert guard.calls == 0 + assert result.terminal_action == "block" + assert result.step_results[0].outcome == "error" + assert "Unsupported pipeline mode: during_call" in result.step_results[0].error_detail + assert result.original_exception is None @pytest.mark.asyncio -async def test_passthrough_guardrail_failure_can_pipeline_block(): +async def test_passthrough_guardrail_failure_can_pipeline_block(monkeypatch): """ Pipeline: passthrough guardrail (on_fail: block) Expected: passthrough ModifyResponseException is treated as policy fail, @@ -298,35 +279,31 @@ async def test_passthrough_guardrail_failure_can_pipeline_block(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [passthrough_guard] + monkeypatch.setattr(litellm, "callbacks", [passthrough_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={ - "model": "fake-model", - "messages": [{"role": "user", "content": "bad content"}], - }, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={ + "model": "fake-model", + "messages": [{"role": "user", "content": "bad content"}], + }, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert passthrough_guard.calls == 1 - assert result.terminal_action == "block" - assert len(result.step_results) == 1 - assert result.step_results[0].guardrail_name == "passthrough-filter" - assert result.step_results[0].outcome == "fail" - assert result.step_results[0].action_taken == "block" - assert result.error_message == "Content policy violation" - finally: - litellm.callbacks = original_callbacks + assert passthrough_guard.calls == 1 + assert result.terminal_action == "block" + assert len(result.step_results) == 1 + assert result.step_results[0].guardrail_name == "passthrough-filter" + assert result.step_results[0].outcome == "fail" + assert result.step_results[0].action_taken == "block" + assert result.error_message == "Content policy violation" @pytest.mark.asyncio -async def test_custom_code_guardrail_failure_can_pipeline_block(): +async def test_custom_code_guardrail_failure_can_pipeline_block(monkeypatch): """ Pipeline: custom code guardrail (on_fail: block) Expected: custom code keeps its standalone passthrough block behavior, and @@ -334,10 +311,7 @@ async def test_custom_code_guardrail_failure_can_pipeline_block(): """ custom_guard = CustomCodeGuardrail( guardrail_name="custom-code-filter", - custom_code=( - "def apply_guardrail(inputs, request_data, input_type):\n" - ' return block("SSN detected")\n' - ), + custom_code=('def apply_guardrail(inputs, request_data, input_type):\n return block("SSN detected")\n'), ) pipeline = GuardrailPipeline( @@ -351,35 +325,31 @@ async def test_custom_code_guardrail_failure_can_pipeline_block(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [custom_guard] + monkeypatch.setattr(litellm, "callbacks", [custom_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={ - "model": "fake-model", - "messages": [{"role": "user", "content": "123-45-6789"}], - }, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={ + "model": "fake-model", + "messages": [{"role": "user", "content": "123-45-6789"}], + }, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert result.terminal_action == "block" - assert len(result.step_results) == 1 - assert result.step_results[0].guardrail_name == "custom-code-filter" - assert result.step_results[0].outcome == "fail" - assert result.step_results[0].action_taken == "block" - assert result.error_message == "SSN detected" - finally: - litellm.callbacks = original_callbacks + assert result.terminal_action == "block" + assert len(result.step_results) == 1 + assert result.step_results[0].guardrail_name == "custom-code-filter" + assert result.step_results[0].outcome == "fail" + assert result.step_results[0].action_taken == "block" + assert result.error_message == "SSN detected" @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_early_allow_step1_passes_step2_skipped(): +async def test_early_allow_step1_passes_step2_skipped(monkeypatch): """ Pipeline: simple-filter (on_pass: allow) -> advanced-filter Input: clean request that passes simple-filter @@ -396,32 +366,28 @@ async def test_early_allow_step1_passes_step2_skipped(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [simple_guard, advanced_guard] + monkeypatch.setattr(litellm, "callbacks", [simple_guard, advanced_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "clean content"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "clean content"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert simple_guard.calls == 1 - assert advanced_guard.calls == 0 - assert result.terminal_action == "allow" - assert len(result.step_results) == 1 - assert result.step_results[0].outcome == "pass" - assert result.step_results[0].action_taken == "allow" - finally: - litellm.callbacks = original_callbacks + assert simple_guard.calls == 1 + assert advanced_guard.calls == 0 + assert result.terminal_action == "allow" + assert len(result.step_results) == 1 + assert result.step_results[0].outcome == "pass" + assert result.step_results[0].action_taken == "allow" @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_escalation_step1_fails_step2_passes(): +async def test_escalation_step1_fails_step2_passes(monkeypatch): """ Pipeline: simple-filter (on_fail: next) -> advanced-filter (on_pass: allow) Input: request that fails simple but passes advanced @@ -438,34 +404,30 @@ async def test_escalation_step1_fails_step2_passes(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [simple_guard, advanced_guard] + monkeypatch.setattr(litellm, "callbacks", [simple_guard, advanced_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "borderline content"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="content-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "borderline content"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="content-safety", + ) - assert simple_guard.calls == 1 - assert advanced_guard.calls == 1 - assert result.terminal_action == "allow" - assert len(result.step_results) == 2 - assert result.step_results[0].outcome == "fail" - assert result.step_results[0].action_taken == "next" - assert result.step_results[1].outcome == "pass" - assert result.step_results[1].action_taken == "allow" - finally: - litellm.callbacks = original_callbacks + assert simple_guard.calls == 1 + assert advanced_guard.calls == 1 + assert result.terminal_action == "allow" + assert len(result.step_results) == 2 + assert result.step_results[0].outcome == "fail" + assert result.step_results[0].action_taken == "next" + assert result.step_results[1].outcome == "pass" + assert result.step_results[1].action_taken == "allow" @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_data_forwarding_pii_masking(): +async def test_data_forwarding_pii_masking(monkeypatch): """ Pipeline: pii-masker (pass_data: true, on_pass: next) -> content-check (on_pass: allow) Input: "Hello John Smith" @@ -487,31 +449,27 @@ async def test_data_forwarding_pii_masking(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [pii_guard, content_guard] + monkeypatch.setattr(litellm, "callbacks", [pii_guard, content_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "Hello John Smith"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="pii-then-safety", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "Hello John Smith"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="pii-then-safety", + ) - assert pii_guard.calls == 1 - assert content_guard.calls == 1 - assert content_guard.received_messages[0]["content"] == "Hello [REDACTED]" - assert result.terminal_action == "allow" - assert result.modified_data is not None - assert result.modified_data["messages"][0]["content"] == "Hello [REDACTED]" - finally: - litellm.callbacks = original_callbacks + assert pii_guard.calls == 1 + assert content_guard.calls == 1 + assert content_guard.received_messages[0]["content"] == "Hello [REDACTED]" + assert result.terminal_action == "allow" + assert result.modified_data is not None + assert result.modified_data["messages"][0]["content"] == "Hello [REDACTED]" @pytest.mark.asyncio -async def test_guardrail_not_found_uses_on_fail(): +async def test_guardrail_not_found_uses_on_fail(monkeypatch): """ If a guardrail is not found, treat as error and use on_fail action. """ @@ -526,29 +484,25 @@ async def test_guardrail_not_found_uses_on_fail(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [] + monkeypatch.setattr(litellm, "callbacks", []) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "test"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test-policy", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "test"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test-policy", + ) - assert result.terminal_action == "block" - assert result.step_results[0].outcome == "error" - assert "not found" in result.step_results[0].error_detail - finally: - litellm.callbacks = original_callbacks + assert result.terminal_action == "block" + assert result.step_results[0].outcome == "error" + assert "not found" in result.step_results[0].error_detail @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_on_error_next_fallback_on_api_outage_on_fail_blocks_content(): +async def test_on_error_next_fallback_on_api_outage_on_fail_blocks_content(monkeypatch): """ Policy intervention (400) uses on_fail; technical error (503) uses on_error. @@ -574,32 +528,28 @@ async def test_on_error_next_fallback_on_api_outage_on_fail_blocks_content(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [primary, fallback] + monkeypatch.setattr(litellm, "callbacks", [primary, fallback]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "any"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="mod-fallback", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "any"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="mod-fallback", + ) - assert primary.calls == 1 - assert fallback.calls == 1 - assert result.terminal_action == "allow" - assert result.step_results[0].outcome == "error" - assert result.step_results[0].action_taken == "next" - assert result.step_results[1].outcome == "pass" - finally: - litellm.callbacks = original_callbacks + assert primary.calls == 1 + assert fallback.calls == 1 + assert result.terminal_action == "allow" + assert result.step_results[0].outcome == "error" + assert result.step_results[0].action_taken == "next" + assert result.step_results[1].outcome == "pass" @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_on_fail_next_on_content_on_error_block_stops_api_fallback(): +async def test_on_fail_next_on_content_on_error_block_stops_api_fallback(monkeypatch): """ Content policy fail (400) uses on_fail: next; API error uses on_error: block (no second step). """ @@ -625,48 +575,40 @@ async def test_on_fail_next_on_content_on_error_block_stops_api_fallback(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [primary_content, fallback] + monkeypatch.setattr(litellm, "callbacks", [primary_content, fallback]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline_content.steps, - mode=pipeline_content.mode, - data={"messages": [{"role": "user", "content": "bad"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test", - ) - assert result.terminal_action == "allow" - assert primary_content.calls == 1 - assert fallback.calls == 1 - finally: - litellm.callbacks = original_callbacks + result = await PipelineExecutor.execute_steps( + steps=pipeline_content.steps, + mode=pipeline_content.mode, + data={"messages": [{"role": "user", "content": "bad"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test", + ) + assert result.terminal_action == "allow" + assert primary_content.calls == 1 + assert fallback.calls == 1 # API outage: on_error block -> do not run fallback fallback.calls = 0 - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [primary_api, fallback] - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline_content.steps, - mode=pipeline_content.mode, - data={"messages": [{"role": "user", "content": "ok"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test", - ) - assert result.terminal_action == "block" - assert primary_api.calls == 1 - assert fallback.calls == 0 - assert result.step_results[0].outcome == "error" - assert result.step_results[0].action_taken == "block" - finally: - litellm.callbacks = original_callbacks + monkeypatch.setattr(litellm, "callbacks", [primary_api, fallback]) + result = await PipelineExecutor.execute_steps( + steps=pipeline_content.steps, + mode=pipeline_content.mode, + data={"messages": [{"role": "user", "content": "ok"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test", + ) + assert result.terminal_action == "block" + assert primary_api.calls == 1 + assert fallback.calls == 0 + assert result.step_results[0].outcome == "error" + assert result.step_results[0].action_taken == "block" @pytest.mark.asyncio -async def test_guardrail_not_found_with_next_continues(): +async def test_guardrail_not_found_with_next_continues(monkeypatch): """ If a guardrail is not found and on_fail is 'next', continue to next step. """ @@ -688,32 +630,28 @@ async def test_guardrail_not_found_with_next_continues(): ], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [pass_guard] + monkeypatch.setattr(litellm, "callbacks", [pass_guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "test"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test-policy", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "test"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test-policy", + ) - assert result.terminal_action == "allow" - assert len(result.step_results) == 2 - assert result.step_results[0].outcome == "error" - assert result.step_results[0].action_taken == "next" - assert result.step_results[1].outcome == "pass" - assert pass_guard.calls == 1 - finally: - litellm.callbacks = original_callbacks + assert result.terminal_action == "allow" + assert len(result.step_results) == 2 + assert result.step_results[0].outcome == "error" + assert result.step_results[0].action_taken == "next" + assert result.step_results[1].outcome == "pass" + assert pass_guard.calls == 1 @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed") @pytest.mark.asyncio -async def test_single_step_pipeline_block(): +async def test_single_step_pipeline_block(monkeypatch): """Single step pipeline that blocks.""" guard = AlwaysFailGuardrail(guardrail_name="blocker") @@ -722,27 +660,23 @@ async def test_single_step_pipeline_block(): steps=[PipelineStep(guardrail="blocker", on_fail="block")], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [guard] + monkeypatch.setattr(litellm, "callbacks", [guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "test"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "test"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test", + ) - assert result.terminal_action == "block" - assert guard.calls == 1 - finally: - litellm.callbacks = original_callbacks + assert result.terminal_action == "block" + assert guard.calls == 1 @pytest.mark.asyncio -async def test_single_step_pipeline_allow(): +async def test_single_step_pipeline_allow(monkeypatch): """Single step pipeline that allows.""" guard = AlwaysPassGuardrail(guardrail_name="passer") @@ -751,27 +685,23 @@ async def test_single_step_pipeline_allow(): steps=[PipelineStep(guardrail="passer", on_pass="allow")], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [guard] + monkeypatch.setattr(litellm, "callbacks", [guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "test"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "test"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test", + ) - assert result.terminal_action == "allow" - assert guard.calls == 1 - finally: - litellm.callbacks = original_callbacks + assert result.terminal_action == "allow" + assert guard.calls == 1 @pytest.mark.asyncio -async def test_step_results_include_duration(): +async def test_step_results_include_duration(monkeypatch): """Step results should include timing information.""" guard = AlwaysPassGuardrail(guardrail_name="timed") @@ -780,23 +710,19 @@ async def test_step_results_include_duration(): steps=[PipelineStep(guardrail="timed")], ) - original_callbacks = litellm.callbacks.copy() - litellm.callbacks = [guard] + monkeypatch.setattr(litellm, "callbacks", [guard]) - try: - result = await PipelineExecutor.execute_steps( - steps=pipeline.steps, - mode=pipeline.mode, - data={"messages": [{"role": "user", "content": "test"}]}, - user_api_key_dict=MagicMock(), - call_type="completion", - policy_name="test", - ) + result = await PipelineExecutor.execute_steps( + steps=pipeline.steps, + mode=pipeline.mode, + data={"messages": [{"role": "user", "content": "test"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="test", + ) - assert result.step_results[0].duration_seconds is not None - assert result.step_results[0].duration_seconds >= 0 - finally: - litellm.callbacks = original_callbacks + assert result.step_results[0].duration_seconds is not None + assert result.step_results[0].duration_seconds >= 0 class _PolicyOptOutGuardrail(CustomGuardrail): diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 47f01fe096d..b0b2c68e30d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -10,6 +10,7 @@ from __future__ import annotations import json import os +import re from types import SimpleNamespace from typing import Any, Dict from unittest.mock import AsyncMock, MagicMock @@ -303,7 +304,7 @@ def test_resolve_routing_plugins_rejects_non_routing_plugin(tmp_path): plugin_file = tmp_path / "bad_rs_plugin.py" plugin_file.write_text("not_a_plugin = object()\n") - with pytest.raises(ValueError, match="router_settings.plugins"): + with pytest.raises(ValueError, match=re.escape("router_settings.plugins")): resolve_routing_plugins( plugin_paths=["bad_rs_plugin.not_a_plugin"], config_file_path=str(tmp_path / "config.yaml"), diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 72e7b1c18d6..31430da71e8 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1,11 +1,8 @@ -import os -import sys from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) from fastapi import FastAPI from fastapi.testclient import TestClient diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 15a117bd6fc..b08de04e801 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -6,16 +6,11 @@ Covers: """ import io -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 9840de8bcb1..66eeb3cef34 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -5,8 +5,6 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: """ import json -import os -import sys import time from unittest.mock import AsyncMock, MagicMock, patch @@ -14,7 +12,6 @@ import httpx import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth diff --git a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py index dbab627d76f..45aa065380d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py @@ -1,11 +1,8 @@ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../../..")) import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 9006288bdae..bb8345a9142 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -1,7 +1,4 @@ -import os -import sys -sys.path.insert(0, os.path.abspath("../../../..")) import pytest @@ -1011,3 +1008,122 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): llm_router=lambda: router, ) assert with_deployment_rate.autorouter > at_public_rate.autorouter + + +def _routed_decision() -> dict: + return {"savings_baseline_model": "anthropic/claude-opus-5", "conversation_continuing": True} + + +def test_recorded_savings_win_over_recomputation(): + """The figure the logging path stamped is the one the rollup keeps, so the + per-request record and the daily rollup cannot disagree.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + recorded_autorouter_savings=0.5, + ) + assert result.autorouter == 0.5 + + +def test_recorded_savings_survive_an_unusable_usage_object(): + """A recorded figure was computed when the usage still parsed; a later row whose + usage_object no longer does must keep the number, not zero it.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=_routed_decision(), + usage_object={"prompt_tokens": ["not", "a", "number"]}, + recorded_autorouter_savings=0.25, + ) + assert result.autorouter == 0.25 + + +def test_a_boolean_is_not_a_recorded_savings_figure(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=None, + usage_object=_cached_usage_object(), + recorded_autorouter_savings=True, + ) + assert result.autorouter == 0.0 + + +def test_rows_written_before_the_field_shipped_recompute(): + """No recorded figure means the row predates the logging-path stamp; the writer + recomputes exactly what the one shared helper would have recorded.""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request + + recomputed = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + ) + direct = autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + ) + assert direct is not None and direct != 0.0 + assert recomputed.autorouter == direct + + +def test_driver_off_is_none_not_zero_for_the_request_helper(): + """None and 0.0 are different facts on the logging payload: absence means the + request was never auto-routed, zero is a real figure for a routed request.""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request + + assert ( + autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision=None, + usage_object=_cached_usage_object(), + ) + is None + ) + assert ( + autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + is None + ) + + +def test_logging_payload_never_stamps_internal_calls(): + """Shadow eval and classifier sub-calls carry a real routing decision but are not + requests the caller made; a stamped figure would report savings for traffic no + user sent, which the spend writer deliberately zeroes.""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload + + routed_metadata = {"routing_decision": _routed_decision()} + stamped = autorouter_savings_for_logging_payload( + request_metadata=routed_metadata, + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + model_id=None, + usage_object=_cached_usage_object(), + cost_breakdown=None, + ) + assert stamped is not None and stamped != 0.0 + + internal = autorouter_savings_for_logging_payload( + request_metadata={**routed_metadata, "internal_call_origin": "shadow_eval_shadow"}, + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + model_id=None, + usage_object=_cached_usage_object(), + cost_breakdown=None, + ) + assert internal is None diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index b2ec500d045..2b062d9020d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2,18 +2,13 @@ import asyncio import collections import datetime import json -import os import re -import sys from datetime import timezone import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch @@ -468,6 +463,7 @@ ignored_keys = [ "metadata.additional_usage_values.iterations", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", + "metadata.autorouter_savings", "metadata.user_api_key", "metadata.user_api_key_alias", "metadata.user_api_key_team_id", diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py index 19083486974..ef68d9ce178 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py @@ -6,14 +6,11 @@ GitHub Issue: #17487 """ import datetime -import os -import sys from datetime import timezone from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_spend_by_team, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index f2dd66ee677..6c8e641642b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1,16 +1,11 @@ import asyncio import datetime import json -import os -import sys from datetime import timezone from typing import Any, Final, cast import pytest -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path from unittest.mock import AsyncMock, MagicMock, patch @@ -3569,3 +3564,38 @@ def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed(): assert result is not None assert result != already_hashed assert result == hash_token(already_hashed) + + +def test_autorouter_savings_flow_from_logging_payload_into_spend_log_metadata(): + """The figure the logging path computed is what the spend writer reads back, so it + is threaded from the StandardLoggingPayload like cost_breakdown, never re-derived.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + "standard_logging_object": {"autorouter_savings": 0.42, "metadata": {}, "model_map_information": None}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-ar-savings", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["autorouter_savings"] == 0.42 + + +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_caller_forged_autorouter_savings_is_discarded(bucket): + """The raw request bucket is client-writable and _get_spend_logs_metadata projects + every SpendLogsMetadata key from it, so the logging payload's value must overwrite + unconditionally or a caller could report savings the router never produced.""" + payload = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {bucket: {"user_api_key": "test-key", "autorouter_savings": 999.0}}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-forged-savings", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["autorouter_savings"] is None diff --git a/tests/test_litellm/proxy/test_batch_expiry.py b/tests/test_litellm/proxy/test_batch_expiry.py index 38c4a71608d..f7b8fbde0fb 100644 --- a/tests/test_litellm/proxy/test_batch_expiry.py +++ b/tests/test_litellm/proxy/test_batch_expiry.py @@ -2,15 +2,10 @@ Tests for batch output_expires_after passthrough and team-level expiry enforcement. """ -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/proxy/test_batch_metadata_none_fix.py b/tests/test_litellm/proxy/test_batch_metadata_none_fix.py index dbc2a402032..aba9b190b66 100644 --- a/tests/test_litellm/proxy/test_batch_metadata_none_fix.py +++ b/tests/test_litellm/proxy/test_batch_metadata_none_fix.py @@ -5,8 +5,6 @@ This test verifies that the fix for handling None metadata in batch requests wor """ import asyncio -import os -import sys from unittest.mock import patch, MagicMock, AsyncMock import pytest @@ -16,9 +14,6 @@ import litellm from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy._types import UserAPIKeyAuth -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path def test_add_key_level_controls_with_none_metadata(): diff --git a/tests/test_litellm/proxy/test_batch_retrieve_bedrock.py b/tests/test_litellm/proxy/test_batch_retrieve_bedrock.py index 13945750092..50d531d22d8 100644 --- a/tests/test_litellm/proxy/test_batch_retrieve_bedrock.py +++ b/tests/test_litellm/proxy/test_batch_retrieve_bedrock.py @@ -14,14 +14,11 @@ must round-trip through `client.files.content(...)` back to bedrock with AWS credentials and the raw S3 URI intact. """ -import os -import sys import httpx import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/proxy/test_caching_routes.py b/tests/test_litellm/proxy/test_caching_routes.py index 840ba054cc9..707d4a3f2c9 100644 --- a/tests/test_litellm/proxy/test_caching_routes.py +++ b/tests/test_litellm/proxy/test_caching_routes.py @@ -1,13 +1,8 @@ import json -import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/test_litellm/proxy/test_custom_proxy.py b/tests/test_litellm/proxy/test_custom_proxy.py index 3663183d211..b646a4e80e7 100644 --- a/tests/test_litellm/proxy/test_custom_proxy.py +++ b/tests/test_litellm/proxy/test_custom_proxy.py @@ -1,5 +1,4 @@ import os -import sys import uvicorn from dotenv import load_dotenv @@ -8,9 +7,6 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse load_dotenv() -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path # Set the SERVER_ROOT_PATH environment variable to match the custom mount path os.environ["SERVER_ROOT_PATH"] = "/my-custom-path" diff --git a/tests/test_litellm/proxy/test_empty_model_list.py b/tests/test_litellm/proxy/test_empty_model_list.py index dde2f06126a..dd4643fcf90 100644 --- a/tests/test_litellm/proxy/test_empty_model_list.py +++ b/tests/test_litellm/proxy/test_empty_model_list.py @@ -5,16 +5,11 @@ These tests verify that /v2/model/info and /model_group/info endpoints return empty data arrays instead of 500 errors when no models are configured. """ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system-path from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.proxy_server import app diff --git a/tests/test_litellm/proxy/test_fastapi_offline_routes.py b/tests/test_litellm/proxy/test_fastapi_offline_routes.py index f3fc3d3ea28..e06e87ed344 100644 --- a/tests/test_litellm/proxy/test_fastapi_offline_routes.py +++ b/tests/test_litellm/proxy/test_fastapi_offline_routes.py @@ -5,12 +5,7 @@ This test verifies that the /routes endpoint works correctly when the proxy server is initialized using FastAPIOffline instead of regular FastAPI. """ -import os -import sys -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import pytest from fastapi.testclient import TestClient diff --git a/tests/test_litellm/proxy/test_filter_models_by_team_access_group.py b/tests/test_litellm/proxy/test_filter_models_by_team_access_group.py index 2d8a9f30c1b..1a514ed2c57 100644 --- a/tests/test_litellm/proxy/test_filter_models_by_team_access_group.py +++ b/tests/test_litellm/proxy/test_filter_models_by_team_access_group.py @@ -7,13 +7,10 @@ looking up deployments — matching the behavior of the auth path in auth_checks.py:model_in_access_group(). """ -import os -import sys from unittest.mock import AsyncMock, MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.proxy_server import _filter_models_by_team_id diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index f223241baf4..f2d95131e5e 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -1,13 +1,10 @@ import asyncio -import os -import sys import time from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.proxy.health_endpoints._health_endpoints import ( _aggregate_health_check_results, diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index d14c89342e9..111f11f85ba 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -2,10 +2,10 @@ import asyncio import copy import json import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest +from botocore.credentials import Credentials from fastapi import Request from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers @@ -26,17 +26,19 @@ from litellm.proxy.litellm_pre_call_utils import ( _update_model_if_key_alias_exists, add_guardrails_from_policy_engine, add_litellm_data_to_request, + add_provider_specific_headers_to_request, check_if_token_is_service_account, clean_headers, ) +from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, +) from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, ) +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.utils import CredentialItem -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path def test_check_if_token_is_service_account(): @@ -3167,6 +3169,129 @@ def test_get_chain_id_from_headers_generic_vendor_session_id(): ) +CODEX_USER_AGENT = "codex_cli_rs/0.62.0 (Mac OS 25.5.0; arm64) Apple_Terminal" +CODEX_SESSION_UUID = "0199f0c2-8b41-7c3e-9a52-6d1f4b8e2a77" + + +@pytest.mark.parametrize( + "user_agent", + [ + "codex-tui", + "codex-tui/0.149.0 (Mac OS 26.5.1; arm64) ghostty/1.3.1 (codex-tui; 0.149.0)", + "codex_cli_rs/0.62.0 (Mac OS 25.5.0; arm64) Apple_Terminal", + "codex_exec/0.62.0 (Linux 6.1; x86_64) unknown", + "codex_vscode/0.62.0 (Mac OS 26.5.1; arm64) vscode/1.99.0", + "Codex CLI/1.0", + ], +) +def test_is_codex_user_agent_accepts_every_first_party_originator(user_agent: str): + """Codex ships several originators sharing only the `codex` stem, and the TUI + sends a bare `codex-tui` with no version, so matching one spelling misses real clients.""" + from litellm.proxy.litellm_pre_call_utils import is_codex_user_agent + + assert is_codex_user_agent(user_agent) is True + + +@pytest.mark.parametrize( + "user_agent", + ["codexify/1.0", "mycodex-tui/1.0", "curl/8.7.1", "claude-cli/2.1.0 (external, cli)", ""], +) +def test_is_codex_user_agent_rejects_non_codex_clients(user_agent: str): + from litellm.proxy.litellm_pre_call_utils import is_codex_user_agent + + assert is_codex_user_agent(user_agent) is False + + +def test_get_chain_id_from_headers_codex_tui_user_agent(): + """The real Codex TUI user agent must group turns, not just the codex_cli_rs spelling.""" + from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers + + ua = "codex-tui/0.149.0 (Mac OS 26.5.1; arm64) ghostty/1.3.1 (codex-tui; 0.149.0)" + assert get_chain_id_from_headers({"user-agent": ua, "session-id": CODEX_SESSION_UUID}) == CODEX_SESSION_UUID + assert ( + get_chain_id_from_headers({"user-agent": "codex-tui", "session-id": CODEX_SESSION_UUID}) == CODEX_SESSION_UUID + ) + + +@pytest.mark.parametrize( + "header", + ["session-id", "session_id", "thread-id", "conversation_id", "Session-Id"], +) +def test_get_chain_id_from_headers_codex_unprefixed_session_id(header: str): + """Codex sends its conversation uuid unprefixed, so the x--session-id regex misses it.""" + from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers + + assert get_chain_id_from_headers({"user-agent": CODEX_USER_AGENT, header: CODEX_SESSION_UUID}) == CODEX_SESSION_UUID + + +@pytest.mark.parametrize( + "user_agent", + ["curl/8.7.1", "claude-cli/2.1.0 (external, cli)", "OpenAI/Python 1.0.0"], +) +def test_get_chain_id_from_headers_unprefixed_session_id_requires_codex(user_agent: str): + """An unprefixed session-id from a non-Codex caller must not group traces. + + The name is generic enough that two unrelated callers could collide on a value + and have their sessions merged, so the bare-header path is Codex-only. + """ + from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers + + assert get_chain_id_from_headers({"user-agent": user_agent, "session-id": CODEX_SESSION_UUID}) is None + assert get_chain_id_from_headers({"session-id": CODEX_SESSION_UUID}) is None + + +def test_get_chain_id_from_headers_codex_prefers_session_over_thread(): + from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers + + assert ( + get_chain_id_from_headers( + { + "user-agent": CODEX_USER_AGENT, + "thread-id": "e96634a3-fa28-4083-b354-55542e2dca01", + "session-id": CODEX_SESSION_UUID, + } + ) + == CODEX_SESSION_UUID + ) + + +def test_get_chain_id_from_headers_codex_ignores_implausible_value(): + from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers + + assert get_chain_id_from_headers({"user-agent": CODEX_USER_AGENT, "session-id": "short"}) is None + assert get_chain_id_from_headers({"user-agent": CODEX_USER_AGENT, "session-id": "has spaces!!"}) is None + + +def test_get_chain_id_from_headers_explicit_beats_codex_header(): + from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers + + assert ( + get_chain_id_from_headers( + { + "user-agent": CODEX_USER_AGENT, + "x-litellm-trace-id": "explicit-id-value", + "session-id": CODEX_SESSION_UUID, + } + ) + == "explicit-id-value" + ) + + +def test_add_litellm_metadata_groups_codex_turns_into_one_session(): + """Every turn of a Codex session must log under one session id, not a fresh per-call trace id.""" + headers = {"user-agent": CODEX_USER_AGENT, "session-id": CODEX_SESSION_UUID} + turns = [{"litellm_metadata": {}}, {"litellm_metadata": {}}] + for turn in turns: + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=turn, _metadata_variable_name="litellm_metadata" + ) + + for turn in turns: + assert turn["litellm_session_id"] == CODEX_SESSION_UUID + assert turn["litellm_trace_id"] == CODEX_SESSION_UUID + assert turn["litellm_metadata"]["session_id"] == CODEX_SESSION_UUID + + def test_trace_id_from_traceparent_valid(): from litellm.proxy.litellm_pre_call_utils import _trace_id_from_traceparent @@ -6211,7 +6336,17 @@ async def test_add_litellm_data_to_request_redacts_oauth_header_from_logging_cop assert updated["proxy_server_request"]["headers"] is updated[metadata_variable_name]["headers"] - assert updated["provider_specific_header"]["extra_headers"]["Authorization"] == _OAUTH_TOKEN + from litellm.litellm_core_utils.get_provider_specific_headers import ( + ProviderSpecificHeaderUtils, + ) + + assert ( + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=updated["provider_specific_header"], + custom_llm_provider="anthropic", + )["Authorization"] + == _OAUTH_TOKEN + ) @pytest.mark.asyncio @@ -6885,3 +7020,193 @@ async def test_add_litellm_data_to_request_caller_tags_empty_when_caller_sends_n assert updated["metadata"]["tags"] == ["key-supplied"] assert updated["metadata"]["caller_tags"] == () + + +OAUTH_TOKEN = "Bearer sk-ant-oat01-fake-subscription-token-for-testing-0123456789" +GOOGLE_ACCESS_TOKEN = "Bearer ya29.fake-google-access-token-for-testing" +BEDROCK_API_KEY = "ABSKQmVkcm9ja0FQSUtleUZvclRlc3Rpbmc=" +CROSS_ACCOUNT_AUTHORIZATION = "Bearer deliberately-configured-pass-through-token" + +SIGV4_PREFIX = "AWS4-HMAC-SHA256" +AUTHORIZATION_HEADER_CASINGS = ["authorization", "Authorization", "AUTHORIZATION"] +LEAK_TARGET_PROVIDERS = ["bedrock", "bedrock_converse", "vertex_ai"] + +BEDROCK_ENDPOINT = ( + "https://bedrock-runtime.us-west-2.amazonaws.com" + "/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke" +) +BEDROCK_REGION = "us-west-2" +BEDROCK_REQUEST_DATA = {"messages": [{"role": "user", "content": "Say OK"}], "max_tokens": 32} +SIGV4_OPTIONAL_PARAMS = { + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "aws_region_name": BEDROCK_REGION, +} + + +def _client_headers(authorization_header_name: str | None = "authorization") -> dict: + headers = { + "content-type": "application/json", + "anthropic-version": "2023-06-01", + "user-agent": "claude-cli/2.1.239", + } + if authorization_header_name is not None: + headers[authorization_header_name] = OAUTH_TOKEN + return headers + + +def _headers_forwarded_to(client_headers: dict, custom_llm_provider: str) -> dict: + data: dict = {} + add_provider_specific_headers_to_request(data=data, headers=client_headers) + return ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=data.get("provider_specific_header"), + custom_llm_provider=custom_llm_provider, + ) + + +def _authorization_values(headers) -> list: + return [value for name, value in headers.items() if name.lower() == "authorization"] + + +def _signed_headers_for_bedrock(request_headers: dict, api_key: str | None = None) -> dict: + with patch.dict(os.environ, {"AWS_BEARER_TOKEN_BEDROCK": ""}): + signed_headers, _ = BaseAWSLLM()._sign_request( + service_name="bedrock", + headers=request_headers, + optional_params=SIGV4_OPTIONAL_PARAMS, + request_data=BEDROCK_REQUEST_DATA, + api_base=BEDROCK_ENDPOINT, + api_key=api_key, + ) + return signed_headers + + +def _signed_headers_component(signature: str, component: str) -> str: + for part in signature.removeprefix(SIGV4_PREFIX).split(","): + name, _, value = part.strip().partition("=") + if name == component: + return value + raise AssertionError(f"{component} missing from {signature}") + + +@pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS) +@pytest.mark.parametrize("custom_llm_provider", LEAK_TARGET_PROVIDERS) +def test_oauth_credential_is_never_forwarded_to_bedrock_or_vertex( + authorization_header_name, custom_llm_provider +): + """ + A client's Anthropic OAuth credential is meaningless to AWS and Google, and sending it + there both breaks the request and hands a third-party cloud a credential it should + never hold. It must not survive the pre-call path for any non-Anthropic provider. + """ + forwarded = _headers_forwarded_to(_client_headers(authorization_header_name), custom_llm_provider) + + assert _authorization_values(forwarded) == [] + assert OAUTH_TOKEN not in forwarded.values() + + +@pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS) +def test_oauth_credential_still_reaches_anthropic_unchanged(authorization_header_name): + forwarded = _headers_forwarded_to(_client_headers(authorization_header_name), "anthropic") + + assert forwarded[authorization_header_name] == OAUTH_TOKEN + assert _authorization_values(forwarded) == [OAUTH_TOKEN] + + +def test_oauth_credential_entry_is_scoped_to_anthropic_alone(): + data: dict = {} + add_provider_specific_headers_to_request(data=data, headers=_client_headers()) + + scoped_headers = data["provider_specific_header"] + if not isinstance(scoped_headers, list): + scoped_headers = [scoped_headers] + + credential_entries = [ + entry for entry in scoped_headers if OAUTH_TOKEN in entry["extra_headers"].values() + ] + assert [entry["custom_llm_provider"] for entry in credential_entries] == ["anthropic"] + + +def test_no_provider_specific_header_when_client_sends_nothing_anthropic(): + data: dict = {} + add_provider_specific_headers_to_request( + data=data, headers={"content-type": "application/json", "authorization": "Bearer sk-a-normal-key"} + ) + + assert "provider_specific_header" not in data + + +def test_bedrock_sigv4_signature_survives_a_client_oauth_header(): + forwarded = _headers_forwarded_to(_client_headers(), "bedrock") + + signed = _signed_headers_for_bedrock({"Content-Type": "application/json", **forwarded}) + + authorizations = _authorization_values(signed) + assert len(authorizations) == 1 + assert authorizations[0].startswith(SIGV4_PREFIX) + assert signed["X-Amz-Date"] + + +def test_bedrock_sigv4_signing_is_unchanged_by_the_client_oauth_header(): + without_oauth = _signed_headers_for_bedrock( + {"Content-Type": "application/json", **_headers_forwarded_to(_client_headers(None), "bedrock")} + ) + with_oauth = _signed_headers_for_bedrock( + {"Content-Type": "application/json", **_headers_forwarded_to(_client_headers(), "bedrock")} + ) + + assert without_oauth["Authorization"].startswith(SIGV4_PREFIX) + assert _signed_headers_component(with_oauth["Authorization"], "SignedHeaders") == ( + _signed_headers_component(without_oauth["Authorization"], "SignedHeaders") + ) + + +def test_bedrock_get_request_headers_keeps_the_sigv4_signature(): + forwarded = _headers_forwarded_to(_client_headers(), "bedrock") + + with patch.dict(os.environ, {"AWS_BEARER_TOKEN_BEDROCK": ""}): + prepped = BaseAWSLLM().get_request_headers( + credentials=Credentials( + SIGV4_OPTIONAL_PARAMS["aws_access_key_id"], + SIGV4_OPTIONAL_PARAMS["aws_secret_access_key"], + ), + aws_region_name=BEDROCK_REGION, + extra_headers=forwarded, + endpoint_url=BEDROCK_ENDPOINT, + data=json.dumps(BEDROCK_REQUEST_DATA), + headers={"Content-Type": "application/json", **forwarded}, + ) + + authorizations = _authorization_values(prepped.headers) + assert len(authorizations) == 1 + assert authorizations[0].startswith(SIGV4_PREFIX) + + +def test_bedrock_api_key_deployment_keeps_its_own_bearer_token(): + forwarded = _headers_forwarded_to(_client_headers(), "bedrock") + + signed = _signed_headers_for_bedrock( + {"Content-Type": "application/json", **forwarded}, api_key=BEDROCK_API_KEY + ) + + assert _authorization_values(signed) == [f"Bearer {BEDROCK_API_KEY}"] + + +def test_deliberately_configured_authorization_still_overrides_sigv4(): + signed = _signed_headers_for_bedrock( + {"Content-Type": "application/json", "Authorization": CROSS_ACCOUNT_AUTHORIZATION} + ) + + assert _authorization_values(signed) == [CROSS_ACCOUNT_AUTHORIZATION] + + +def test_vertex_sends_exactly_one_authorization_header(): + forwarded = _headers_forwarded_to(_client_headers(), "vertex_ai") + + vertex_request_headers = { + "content-type": "application/json", + "Authorization": GOOGLE_ACCESS_TOKEN, + } + vertex_request_headers.update(forwarded) + + assert _authorization_values(vertex_request_headers) == [GOOGLE_ACCESS_TOKEN] diff --git a/tests/test_litellm/proxy/test_model_deprecations_endpoint.py b/tests/test_litellm/proxy/test_model_deprecations_endpoint.py index c942408bd14..6495cf408e1 100644 --- a/tests/test_litellm/proxy/test_model_deprecations_endpoint.py +++ b/tests/test_litellm/proxy/test_model_deprecations_endpoint.py @@ -1,11 +1,8 @@ -import os -import sys from unittest.mock import MagicMock import pytest from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.proxy import proxy_server diff --git a/tests/test_litellm/proxy/test_pricing_field_strip.py b/tests/test_litellm/proxy/test_pricing_field_strip.py index bbdddd1cd8c..1707f5bbc05 100644 --- a/tests/test_litellm/proxy/test_pricing_field_strip.py +++ b/tests/test_litellm/proxy/test_pricing_field_strip.py @@ -10,8 +10,6 @@ strips them at the boundary; an opt-in key/team flag preserves the override for operators who actually want it. """ -import os -import sys from unittest.mock import MagicMock import pytest @@ -27,7 +25,6 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.types.utils import CustomPricingLiteLLMParams -sys.path.insert(0, os.path.abspath("../../..")) def _make_request_mock() -> Request: diff --git a/tests/test_litellm/proxy/test_provider_url_destination_guard.py b/tests/test_litellm/proxy/test_provider_url_destination_guard.py index cd993a076e8..24c4e991adf 100644 --- a/tests/test_litellm/proxy/test_provider_url_destination_guard.py +++ b/tests/test_litellm/proxy/test_provider_url_destination_guard.py @@ -6,8 +6,6 @@ an SSRF primitive — guarded centrally in ``litellm_pre_call_utils`` so SDK users keep working but proxy users default-deny. """ -import os -import sys from unittest.mock import MagicMock import pytest @@ -20,7 +18,6 @@ from litellm.proxy.litellm_pre_call_utils import ( add_litellm_data_to_request, ) -sys.path.insert(0, os.path.abspath("../../..")) class TestRejectUrlValuedDestinations: diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 48c56a41ad5..6ea6f208bb5 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1,6 +1,5 @@ import inspect import os -import sys from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -9,9 +8,6 @@ import click import fastapi import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system-path import builtins import types diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 76aeea894e1..31d2a6cef98 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2,9 +2,9 @@ import asyncio import importlib import json import os +import re import socket import subprocess -import sys import types from datetime import datetime, timedelta, timezone from pathlib import Path @@ -19,7 +19,6 @@ from fastapi import FastAPI from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system-path import litellm import litellm.proxy.proxy_server as proxy_server_module @@ -2683,7 +2682,7 @@ async def test_get_config_from_file(tmp_path, monkeypatch): with open(empty_file, "w") as f: f.write("") # Write empty content which will result in None when loaded - with pytest.raises(Exception, match="Config cannot be None or Empty."): + with pytest.raises(Exception, match=re.escape("Config cannot be None or Empty.")): await proxy_config._get_config_from_file(str(empty_file)) # Test Case 5: Using global user_config_file_path when no config_file_path provided diff --git a/tests/test_litellm/proxy/test_proxy_types.py b/tests/test_litellm/proxy/test_proxy_types.py index 4a93e9ac7ba..634b90e445a 100644 --- a/tests/test_litellm/proxy/test_proxy_types.py +++ b/tests/test_litellm/proxy/test_proxy_types.py @@ -1,10 +1,8 @@ import asyncio import importlib import json -import os import socket import subprocess -import sys from unittest import mock from unittest.mock import AsyncMock, MagicMock, mock_open, patch @@ -15,9 +13,6 @@ import yaml from fastapi import FastAPI from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system-path def test_audit_log_masking(): @@ -177,3 +172,107 @@ def test_project_io_token_limits_are_stored_in_metadata(request_type): assert request.metadata == limits assert request.model_dump(exclude_none=True)["metadata"] == limits + + +def test_a_jwt_issuer_must_pick_audience_validation_or_opt_out(): + from pydantic import ValidationError + + from litellm.proxy._types import JWTIssuerConfig + + with pytest.raises(ValidationError, match="must configure audience or set disable_audience_validation"): + JWTIssuerConfig(issuer="https://issuer.example.com") + + with pytest.raises(ValidationError, match="cannot set audience and disable_audience_validation"): + JWTIssuerConfig( + issuer="https://issuer.example.com", + audience="litellm-proxy", + disable_audience_validation=True, + ) + + assert JWTIssuerConfig(issuer="https://issuer.example.com", audience="litellm-proxy").audience == "litellm-proxy" + assert ( + JWTIssuerConfig(issuer="https://issuer.example.com", disable_audience_validation=True).audience + is None + ) + + +def test_a_jwt_issuer_rejects_a_field_it_does_not_define(): + from pydantic import ValidationError + + from litellm.proxy._types import JWTIssuerConfig + + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + JWTIssuerConfig(issuer="https://issuer.example.com", audience="a", jwks_uri="https://issuer/jwks") + + +def test_a_temp_budget_needs_both_halves_or_neither(): + from pydantic import ValidationError + + from litellm.proxy._types import UpdateKeyRequest + + with pytest.raises(ValidationError, match="temp_budget_increase and temp_budget_expiry must be set together"): + UpdateKeyRequest(key="sk-1234", temp_budget_increase=10) + + with pytest.raises(ValidationError, match="temp_budget_increase and temp_budget_expiry must be set together"): + UpdateKeyRequest(key="sk-1234", temp_budget_expiry="2026-01-01") + + both = UpdateKeyRequest(key="sk-1234", temp_budget_increase=10, temp_budget_expiry="2026-01-01") + assert both.temp_budget_increase == 10 + + +def test_an_empty_max_budget_is_read_as_no_limit(): + from litellm.proxy._types import GenerateKeyRequest + + assert GenerateKeyRequest(max_budget="").max_budget is None + assert GenerateKeyRequest(max_budget=25).max_budget == 25 + + +def test_an_organization_member_can_only_take_a_role_the_organization_has(): + from pydantic import ValidationError + + from litellm.proxy._types import LitellmUserRoles, OrganizationMemberUpdateRequest + + with pytest.raises(ValidationError, match="Invalid role"): + OrganizationMemberUpdateRequest( + organization_id="org-1", user_id="user-1", role=LitellmUserRoles.PROXY_ADMIN + ) + + allowed = OrganizationMemberUpdateRequest( + organization_id="org-1", user_id="user-1", role=LitellmUserRoles.ORG_ADMIN + ) + assert allowed.role == LitellmUserRoles.ORG_ADMIN + + +def test_an_llm_backed_injection_check_needs_the_call_it_would_make(): + from pydantic import ValidationError + + from litellm.proxy._types import LiteLLMPromptInjectionParams + + for missing in ("llm_api_name", "llm_api_system_prompt", "llm_api_fail_call_string"): + complete = { + "llm_api_name": "gpt-4o", + "llm_api_system_prompt": "is this an injection", + "llm_api_fail_call_string": "yes", + } + del complete[missing] + with pytest.raises(ValidationError, match=f"{missing} must be provided"): + LiteLLMPromptInjectionParams(llm_api_check=True, **complete) + + assert LiteLLMPromptInjectionParams(llm_api_check=False).llm_api_name is None + + +@pytest.mark.parametrize( + "field, forged, default", + [ + ("mcp_admitted_user_subject", "someone-else", False), + ("mcp_source_team_rpm_limits", {"team-1": 10_000}, None), + ("mcp_session_resource_server_id", "server-1", None), + ("via_virtual_key", "sk-someone-elses-key", False), + ], +) +def test_a_server_only_marker_is_not_taken_from_the_caller(field, forged, default): + from litellm.proxy._types import UserAPIKeyAuth + + auth = UserAPIKeyAuth(api_key="sk-1234", **{field: forged}) + + assert getattr(auth, field) == default diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index fe79ef25da6..fb01216982f 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1,7 +1,5 @@ import datetime as real_datetime -import os import smtplib -import sys import pytest from fastapi import HTTPException @@ -12,9 +10,6 @@ from litellm.proxy._types import ProxyErrorTypes from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from unittest.mock import MagicMock, patch @@ -1644,3 +1639,193 @@ async def test_post_mcp_call_hook_skips_opted_out_guardrail(restore_callbacks): assert guardrail.call_count == 0 assert [item.text for item in returned.content] == ["jane@example.com"] + + +FAILURE_USAGE_MODEL = "gpt-4o" +ONE_USER_MESSAGE = [{"role": "user", "content": "hi"}] + + +class _LoggingObj: + def __init__(self, model_call_details): + self.model_call_details = model_call_details + + +@pytest.mark.parametrize( + "system_input, expected", + [ + ("be brief", "be brief"), + ([{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], "ab"), + (["a", {"text": "b"}], "ab"), + ([{"type": "image"}], ""), + (None, ""), + (17, ""), + ], +) +def test_a_system_prompt_reads_the_same_whatever_shape_it_arrived_in(system_input, expected): + from litellm.proxy.utils import _system_prompt_text + + assert _system_prompt_text(system_input) == expected + + +def test_a_system_prompt_is_counted_on_top_of_the_request(): + from litellm.proxy.utils import _count_request_input_tokens + + without = _count_request_input_tokens(FAILURE_USAGE_MODEL, "hello world", None) + with_system = _count_request_input_tokens(FAILURE_USAGE_MODEL, "hello world", "be brief") + + assert without > 0 + assert with_system > without + + +def test_a_request_with_nothing_in_it_counts_zero(): + from litellm.proxy.utils import _count_request_input_tokens + + assert _count_request_input_tokens(FAILURE_USAGE_MODEL, [], None) == 0 + assert _count_request_input_tokens(FAILURE_USAGE_MODEL, None, None) == 0 + + +def test_a_failed_dispatch_is_estimated_as_input_only(): + from litellm.proxy.utils import _count_request_input_tokens, _estimate_dispatched_failure_usage + + usage = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) + + assert usage is not None + assert usage.prompt_tokens == _count_request_input_tokens( + FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None + ) + assert usage.completion_tokens == 0 + assert usage.total_tokens == usage.prompt_tokens + + +@pytest.mark.parametrize("request_input", [[], object()]) +def test_nothing_is_estimated_when_there_is_nothing_to_count(request_input): + from litellm.proxy.utils import _estimate_dispatched_failure_usage + + assert _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, request_input, None) is None + + +def test_usage_the_stream_already_recovered_beats_an_estimate(): + from litellm.proxy.utils import _failure_usage_to_lift + from litellm.types.utils import Usage + + recovered = Usage(prompt_tokens=5, completion_tokens=7, total_tokens=12) + + lifted = _failure_usage_to_lift( + model_call_details={"combined_usage_object": recovered, "response_cost": 0.25}, + request_body={}, + dispatched=True, + ) + + assert lifted == (recovered, 0.25) + + +def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost(): + from litellm.proxy.utils import _failure_usage_to_lift + + lifted = _failure_usage_to_lift( + model_call_details={ + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + }, + request_body={}, + dispatched=True, + ) + + assert lifted is not None + usage, response_cost = lifted + assert usage.prompt_tokens > 0 + assert usage.completion_tokens == 0 + assert response_cost == 0.0 + + +@pytest.mark.parametrize( + "model_call_details, dispatched", + [ + ({"call_type": "acompletion", "model": FAILURE_USAGE_MODEL, "messages": ONE_USER_MESSAGE}, False), + ( + { + "litellm_no_upstream_llm_call": True, + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + }, + True, + ), + ({"call_type": "afile_content", "model": FAILURE_USAGE_MODEL, "messages": ONE_USER_MESSAGE}, True), + ], + ids=["never dispatched", "no upstream call", "call type has no input to price"], +) +def test_a_failure_that_cost_the_provider_nothing_lifts_nothing(model_call_details, dispatched): + from litellm.proxy.utils import _failure_usage_to_lift + + assert _failure_usage_to_lift( + model_call_details=model_call_details, request_body={}, dispatched=dispatched + ) is None + + +def test_the_no_upstream_call_key_the_module_uses_is_the_one_asserted_above(): + from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL + + assert LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL == "litellm_no_upstream_llm_call" + + +def test_the_dispatched_system_prompt_wins_over_the_one_in_the_request_body(): + from litellm.proxy.utils import _failure_usage_to_lift + + def lift(model_call_details, request_body): + lifted = _failure_usage_to_lift( + model_call_details=model_call_details, request_body=request_body, dispatched=True + ) + assert lifted is not None + return lifted[0].prompt_tokens + + base = { + "call_type": "aanthropic_messages", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + } + long_system = "answer as briefly as you possibly can, in one short sentence" + + from_body = lift(base, {"system": long_system}) + from_params = lift({**base, "optional_params": {"system": "x"}}, {"system": long_system}) + body_only_short = lift(base, {"system": "x"}) + + assert from_body > body_only_short + assert from_params == body_only_short + + +def test_a_failure_with_no_logging_object_lifts_nothing(): + from litellm.proxy.utils import _failure_fields_to_lift + + assert dict(_failure_fields_to_lift({})) == {} + assert dict(_failure_fields_to_lift({"litellm_logging_obj": _LoggingObj({})})) == {} + + +def test_a_dispatched_failure_lifts_the_four_fields_the_spend_log_needs(): + from litellm.proxy.utils import _failure_fields_to_lift + + lifted = _failure_fields_to_lift( + { + "litellm_logging_obj": _LoggingObj( + { + "first_api_call_start_time": 1700000000.0, + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + "standard_logging_object": {"id": "log-1"}, + } + ) + } + ) + + assert set(lifted) == { + "first_api_call_start_time", + "combined_usage_object", + "response_cost", + "standard_logging_object", + } + assert lifted["first_api_call_start_time"] == 1700000000.0 + assert lifted["response_cost"] == 0.0 + assert lifted["combined_usage_object"].prompt_tokens > 0 + assert lifted["standard_logging_object"] == {"id": "log-1"} diff --git a/tests/test_litellm/proxy/test_response_model_sanitization.py b/tests/test_litellm/proxy/test_response_model_sanitization.py index 621291b8331..c20b1208e8f 100644 --- a/tests/test_litellm/proxy/test_response_model_sanitization.py +++ b/tests/test_litellm/proxy/test_response_model_sanitization.py @@ -1,7 +1,5 @@ import asyncio import json -import os -import sys from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock @@ -9,7 +7,6 @@ import pytest import yaml from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../../..")) import litellm diff --git a/tests/test_litellm/proxy/test_route_a2a_models.py b/tests/test_litellm/proxy/test_route_a2a_models.py index 0523e796543..35308474949 100644 --- a/tests/test_litellm/proxy/test_route_a2a_models.py +++ b/tests/test_litellm/proxy/test_route_a2a_models.py @@ -4,10 +4,7 @@ Test A2A model routing in proxy. Maps to: litellm/proxy/agent_endpoints/a2a_routing.py """ -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) from unittest.mock import AsyncMock, Mock, patch diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 23e0bbfb3ee..41ba57c4615 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1,9 +1,6 @@ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path from unittest.mock import MagicMock diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index a1a02ec427b..b85c70cae12 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1,13 +1,9 @@ import json import os -import sys import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.proxy._types import DefaultInternalUserParams, LitellmUserRoles from litellm.proxy.proxy_server import app diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index 20b2f68bb0c..905928428b7 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -1,14 +1,9 @@ -import os -import sys from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import Request -sys.path.insert( - 0, os.path.abspath("../../../../..") -) # Adds the parent directory to the system path from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/vector_store_files_endpoints/test_endpoints.py b/tests/test_litellm/proxy/vector_store_files_endpoints/test_endpoints.py index da5dd1934e4..4cb3a3d4c7f 100644 --- a/tests/test_litellm/proxy/vector_store_files_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_files_endpoints/test_endpoints.py @@ -10,15 +10,12 @@ is attached to a vector store or read back under shared provider credentials. """ import base64 -import os -import sys from dataclasses import dataclass from typing import Literal from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index 40a26fad3c3..a959326817c 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -26,8 +26,6 @@ patched with autospec so the real __init__ still stores self.data (captured via mock's call args), and a brand-new kwarg added to this layer surfaces as a failure. """ -import os -import sys from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Dict, Optional @@ -36,7 +34,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import orjson import pytest -sys.path.insert(0, os.path.abspath("../../../..")) import litellm.proxy.proxy_server as proxy_server import litellm.proxy.video_endpoints.endpoints as endpoints diff --git a/tests/test_litellm/proxy/video_endpoints/test_utils.py b/tests/test_litellm/proxy/video_endpoints/test_utils.py index ae22ae233b5..efbaaff5f4b 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_utils.py +++ b/tests/test_litellm/proxy/video_endpoints/test_utils.py @@ -12,12 +12,9 @@ is encode_character_id_with_provider, which runs for real; encoding assertions are checked by the genuine decode round-trip. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 9f48d4d427b..a0c0d849e3e 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -1,10 +1,7 @@ import asyncio -import os -import sys import time from unittest.mock import MagicMock -sys.path.insert(0, os.path.abspath("../../..")) import pytest diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 46d1461da50..85777afe81c 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -1,9 +1,6 @@ import logging -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.abspath("../../..")) import litellm diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_handler.py b/tests/test_litellm/responses/litellm_completion_transformation/test_handler.py index 04db7192364..2cfec6a1844 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_handler.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_handler.py @@ -12,13 +12,10 @@ capture the forwarded kwargs; if the flag-setting line is removed the captured kwargs lack the flag and these tests fail. """ -import os -import sys from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../../../..")) from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index aae053c2e8e..5efabed4b8d 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1,12 +1,7 @@ import json -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.responses.litellm_completion_transformation.transformation import ( TOOL_CALLS_CACHE, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py index 9c354101e22..19f240fa3d4 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py @@ -1,15 +1,10 @@ import json -import os -import sys from unittest.mock import AsyncMock, patch import pytest from fastapi import HTTPException from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.responses.litellm_completion_transformation import session_handler from litellm.responses.litellm_completion_transformation.session_handler import ( diff --git a/tests/test_litellm/responses/test_metadata_codex_callback.py b/tests/test_litellm/responses/test_metadata_codex_callback.py index f7d97b164da..f151f36be63 100644 --- a/tests/test_litellm/responses/test_metadata_codex_callback.py +++ b/tests/test_litellm/responses/test_metadata_codex_callback.py @@ -10,12 +10,9 @@ verifies metadata is preserved for custom callbacks via kwargs['litellm_params'] """ import asyncio -import os -import sys from typing import Optional from unittest.mock import AsyncMock, patch -sys.path.insert(0, os.path.abspath("../../..")) import pytest diff --git a/tests/test_litellm/responses/test_no_duplicate_spend_logs.py b/tests/test_litellm/responses/test_no_duplicate_spend_logs.py index 3ef5935933a..c98b519ae67 100644 --- a/tests/test_litellm/responses/test_no_duplicate_spend_logs.py +++ b/tests/test_litellm/responses/test_no_duplicate_spend_logs.py @@ -7,14 +7,9 @@ causing duplicate spend log entries for non-OpenAI providers. """ import asyncio -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/test_litellm/responses/test_responses_api_bridge_flag.py b/tests/test_litellm/responses/test_responses_api_bridge_flag.py index f94c31831bf..d76fa59a888 100644 --- a/tests/test_litellm/responses/test_responses_api_bridge_flag.py +++ b/tests/test_litellm/responses/test_responses_api_bridge_flag.py @@ -6,13 +6,8 @@ Includes file_search emulation: the flag must be forwarded on inner aresponses calls so routed requests do not hit a custom api_base /v1/responses endpoint. """ -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse diff --git a/tests/test_litellm/responses/test_responses_router_cooldown.py b/tests/test_litellm/responses/test_responses_router_cooldown.py index 48e2d2455e7..e173c174521 100644 --- a/tests/test_litellm/responses/test_responses_router_cooldown.py +++ b/tests/test_litellm/responses/test_responses_router_cooldown.py @@ -6,14 +6,11 @@ the "No model_info found" branch and the failing deployment was never added to the cooldown set. """ -import os -import sys from unittest.mock import AsyncMock, patch import httpx import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 2f4a699d307..dddb851acf9 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -1,11 +1,8 @@ import base64 -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index 321abe4cc6d..9c344fc6894 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -15,13 +15,10 @@ Pydantic ValidationError (previously typed as Optional[str]). """ import json -import os -import sys from unittest.mock import Mock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.exceptions import MidStreamFallbackError diff --git a/tests/test_litellm/responses/test_text_format_conversion.py b/tests/test_litellm/responses/test_text_format_conversion.py index a48540b129b..cca7748fd3a 100644 --- a/tests/test_litellm/responses/test_text_format_conversion.py +++ b/tests/test_litellm/responses/test_text_format_conversion.py @@ -1,13 +1,8 @@ import json -import os -import sys import pytest from pydantic import BaseModel -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.types.llms.openai import ( @@ -158,7 +153,6 @@ class TestTextFormatConversion: new=mock_handler, ): litellm._turn_on_debug() - litellm.set_verbose = True # Call aresponses with text_format parameter response = await litellm.aresponses( diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index c71a6b0e27f..36199b45847 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -1,15 +1,10 @@ import asyncio import json -import os -import sys from typing import Any, Dict, Final, List, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path from litellm.router_strategy.auto_router.auto_router import AutoRouter diff --git a/tests/test_litellm/router_strategy/test_base_routing_strategy.py b/tests/test_litellm/router_strategy/test_base_routing_strategy.py index 70259605b2f..154042692d0 100644 --- a/tests/test_litellm/router_strategy/test_base_routing_strategy.py +++ b/tests/test_litellm/router_strategy/test_base_routing_strategy.py @@ -1,13 +1,8 @@ import json -import os -import sys from typing import Any, Dict, List, Optional, Set, Union import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import asyncio from unittest.mock import MagicMock, patch diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 64b60c75f87..e65d79a83f4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -6,15 +6,12 @@ Tests the rule-based complexity scoring and tier assignment logic. import asyncio import logging -import os -import sys from typing import Dict, List from unittest.mock import AsyncMock, MagicMock, patch import pytest from pydantic import ValidationError -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path import litellm from litellm import Router diff --git a/tests/test_litellm/router_strategy/test_litellm_encoder.py b/tests/test_litellm/router_strategy/test_litellm_encoder.py index 6c934e57832..ebd6efe309c 100644 --- a/tests/test_litellm/router_strategy/test_litellm_encoder.py +++ b/tests/test_litellm/router_strategy/test_litellm_encoder.py @@ -1,12 +1,9 @@ """Tests for litellm/router_strategy/auto_router/litellm_encoder.py""" -import os -import sys from typing import Any, Final import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 4edc1e21d6b..6701f4a7aa2 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -5,15 +5,10 @@ # latency list and break the Redis cache sync). Issue #33169. import json -import os -import sys from datetime import datetime, timedelta import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index a54e95ff7a1..4e87652f8b3 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -9,14 +9,11 @@ Covers: - Decision metadata stash + Router.set_response_headers lift. """ -import os -import sys from typing import Any, Dict, List from unittest.mock import MagicMock import pytest -sys.path.insert(0, os.path.abspath("../../..")) from litellm.router_strategy.quality_router.config import ( DEFAULT_COMPLEXITY_TO_QUALITY, diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py index 7d1ed796996..5599c5aad63 100644 --- a/tests/test_litellm/router_strategy/test_router_routing_groups.py +++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py @@ -5,13 +5,10 @@ the implicit `"default"` group driven by the router's top-level `routing_strategy` / `routing_strategy_args`. """ -import os -import sys from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import Router diff --git a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py index 6591478a4e7..212c2627d88 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_regex_routing.py @@ -6,12 +6,9 @@ patterns, verifying that regex-based header matching works correctly alongside existing tag-based routing. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from unittest.mock import MagicMock diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 752136f2b66..72bb6756d24 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -1,14 +1,10 @@ #### What this tests #### # This tests litellm router -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path import logging -import os import litellm diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py index 428eb0ceafd..b3a2bdda53c 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -1,11 +1,8 @@ import asyncio -import os -import sys from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import json diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 510dcf77afd..dac991a41c4 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -15,15 +15,12 @@ The mechanism works without any cache and supports two encoding strategies: encrypted_content back to their original forms before sending to the upstream provider. """ -import os -import sys import time from typing import List, Optional from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.responses.utils import ResponsesAPIRequestUtils diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index d0fff0201e3..f54a1cfa284 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -1,12 +1,9 @@ import asyncio import copy -import os -import sys from typing import List, cast import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.caching.dual_cache import DualCache diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py index 3c6a05e7786..ee7fab7d19f 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py @@ -1,12 +1,9 @@ import asyncio -import os -import sys from typing import Optional from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import json import litellm diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py index 4053e6d118b..a3772a276fa 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py @@ -1,10 +1,7 @@ -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import json diff --git a/tests/test_litellm/router_utils/test_cooldown_cache.py b/tests/test_litellm/router_utils/test_cooldown_cache.py index a48402684b4..68e9aeaa4fc 100644 --- a/tests/test_litellm/router_utils/test_cooldown_cache.py +++ b/tests/test_litellm/router_utils/test_cooldown_cache.py @@ -2,15 +2,12 @@ Unit tests for CooldownCache exception masking functionality """ -import os -import sys import time from unittest.mock import MagicMock import pytest # Add the parent directory to the system path -sys.path.insert(0, os.path.abspath("../../..")) from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache diff --git a/tests/test_litellm/sandbox/test_opensandbox_sandbox.py b/tests/test_litellm/sandbox/test_opensandbox_sandbox.py index 0d7bcbe1e53..2928dea100e 100644 --- a/tests/test_litellm/sandbox/test_opensandbox_sandbox.py +++ b/tests/test_litellm/sandbox/test_opensandbox_sandbox.py @@ -490,7 +490,7 @@ async def test_create_waits_for_endpoint_resolution(monkeypatch): async def test_create_raises_when_endpoint_is_missing(): client = FakeHTTPClient(endpoint_json={"headers": {"X": "y"}}) - with pytest.raises(TimeoutError, match="execd endpoint.*not ready"): + with pytest.raises(TimeoutError, match=r"execd endpoint.*not ready"): await OpenSandboxSandboxConfig().acreate_sandbox( api_key="", api_base=TEST_API_BASE, ready_timeout=0, client=client ) diff --git a/tests/test_litellm/secret_managers/test_base_secret_manager.py b/tests/test_litellm/secret_managers/test_base_secret_manager.py index e1ccb91c381..a9c6695eb82 100644 --- a/tests/test_litellm/secret_managers/test_base_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_base_secret_manager.py @@ -3,12 +3,9 @@ Test raise_if_unsafe_secret_name, the shared guard applied before secret_name reaches a secret manager backend. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_name diff --git a/tests/test_litellm/secret_managers/test_custom_secret_manager.py b/tests/test_litellm/secret_managers/test_custom_secret_manager.py index 0426c5973cc..e22af4f9a18 100644 --- a/tests/test_litellm/secret_managers/test_custom_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_custom_secret_manager.py @@ -2,16 +2,11 @@ Test custom secret manager implementation """ -import os -import sys from typing import Optional, Union import httpx import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager diff --git a/tests/test_litellm/secret_managers/test_get_azure_ad_token_provider.py b/tests/test_litellm/secret_managers/test_get_azure_ad_token_provider.py index cee0da79802..c9ec22ab0df 100644 --- a/tests/test_litellm/secret_managers/test_get_azure_ad_token_provider.py +++ b/tests/test_litellm/secret_managers/test_get_azure_ad_token_provider.py @@ -1,11 +1,9 @@ import json import os -import sys from typing import Optional from unittest.mock import MagicMock, patch # Adds the grandparent directory to sys.path to allow importing project modules -sys.path.insert(0, os.path.abspath("../..")) import pytest diff --git a/tests/test_litellm/test_a2a_registry_lookup.py b/tests/test_litellm/test_a2a_registry_lookup.py index dd745bfe15b..54393e3ae5e 100644 --- a/tests/test_litellm/test_a2a_registry_lookup.py +++ b/tests/test_litellm/test_a2a_registry_lookup.py @@ -4,10 +4,7 @@ Test A2A provider registry lookup functionality. Maps to: litellm/llms/a2a/chat/transformation.py """ -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import pytest diff --git a/tests/test_litellm/test_acompletion_session_reuse_e2e.py b/tests/test_litellm/test_acompletion_session_reuse_e2e.py index 79b947bb146..2c0bc32f84b 100644 --- a/tests/test_litellm/test_acompletion_session_reuse_e2e.py +++ b/tests/test_litellm/test_acompletion_session_reuse_e2e.py @@ -12,13 +12,10 @@ wasting ~100-500ms per request. With reuse, connections are pooled and subsequent requests are 40-60% faster. """ -import os -import sys import inspect import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm diff --git a/tests/test_litellm/test_add_deployment_no_master_key.py b/tests/test_litellm/test_add_deployment_no_master_key.py index 6db20d7d422..2973f1a8f69 100644 --- a/tests/test_litellm/test_add_deployment_no_master_key.py +++ b/tests/test_litellm/test_add_deployment_no_master_key.py @@ -6,12 +6,10 @@ failed when master_key was None. [https://github.com/BerriAI/litellm/issues/1642 """ import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy.proxy_server import ProxyConfig @@ -62,7 +60,7 @@ async def test_add_deployment_without_master_key(): @pytest.mark.asyncio -async def test_add_deployment_without_salt_key_or_master_key(): +async def test_add_deployment_without_salt_key_or_master_key(monkeypatch): """ Test that add_deployment() works when both master_key and LITELLM_SALT_KEY are None. @@ -70,55 +68,50 @@ async def test_add_deployment_without_salt_key_or_master_key(): such as in a local/dev environment or when just saving spend logs. """ # Remove LITELLM_SALT_KEY from environment - old_salt_key = os.environ.pop("LITELLM_SALT_KEY", None) + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) - try: - # Set master_key to None - with patch("litellm.proxy.proxy_server.master_key", None): - # Mock the required dependencies - mock_prisma_client = MagicMock(spec=PrismaClient) - mock_prisma_client.db = MagicMock() - mock_prisma_client.db.litellm_config = MagicMock() - mock_prisma_client.db.litellm_config.find_first = AsyncMock( - return_value=None + # Set master_key to None + with patch("litellm.proxy.proxy_server.master_key", None): + # Mock the required dependencies + mock_prisma_client = MagicMock(spec=PrismaClient) + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_config = MagicMock() + mock_prisma_client.db.litellm_config.find_first = AsyncMock( + return_value=None + ) + + mock_proxy_logging = MagicMock(spec=ProxyLogging) + + # Create ProxyConfig instance + proxy_config = ProxyConfig() + + # Mock the internal methods + proxy_config._should_load_db_object = MagicMock(return_value=False) + proxy_config._init_non_llm_objects_in_db = AsyncMock() + + # This should NOT raise an exception + try: + await proxy_config.add_deployment( + prisma_client=mock_prisma_client, + proxy_logging_obj=mock_proxy_logging, ) - - mock_proxy_logging = MagicMock(spec=ProxyLogging) - - # Create ProxyConfig instance - proxy_config = ProxyConfig() - - # Mock the internal methods - proxy_config._should_load_db_object = MagicMock(return_value=False) - proxy_config._init_non_llm_objects_in_db = AsyncMock() - - # This should NOT raise an exception - try: - await proxy_config.add_deployment( - prisma_client=mock_prisma_client, - proxy_logging_obj=mock_proxy_logging, + assert True + except ValueError as e: + if "Master key is not initialized" in str( + e + ) or "Encryption key is not initialized" in str(e): + pytest.fail( + f"add_deployment raised ValueError about encryption key: {e}" ) - assert True - except ValueError as e: - if "Master key is not initialized" in str( - e - ) or "Encryption key is not initialized" in str(e): - pytest.fail( - f"add_deployment raised ValueError about encryption key: {e}" - ) - raise - except Exception as e: - if "Master key is not initialized" in str( - e - ) or "Encryption key is not initialized" in str(e): - pytest.fail( - f"add_deployment raised exception about encryption key: {e}" - ) - raise - finally: - # Restore LITELLM_SALT_KEY if it was set - if old_salt_key: - os.environ["LITELLM_SALT_KEY"] = old_salt_key + raise + except Exception as e: + if "Master key is not initialized" in str( + e + ) or "Encryption key is not initialized" in str(e): + pytest.fail( + f"add_deployment raised exception about encryption key: {e}" + ) + raise def test_add_deployment_sync_without_master_key(): diff --git a/tests/test_litellm/test_aembedding_session_reuse_e2e.py b/tests/test_litellm/test_aembedding_session_reuse_e2e.py index b24aab72fdb..15662d4d35c 100644 --- a/tests/test_litellm/test_aembedding_session_reuse_e2e.py +++ b/tests/test_litellm/test_aembedding_session_reuse_e2e.py @@ -5,11 +5,8 @@ Ensures shared_session is in all_litellm_params to prevent "Object of type ClientSession is not JSON serializable" errors. """ -import os -import sys import inspect -sys.path.insert(0, os.path.abspath("../../..")) from litellm.types.utils import all_litellm_params diff --git a/tests/test_litellm/test_azure_audio_price_aliases.py b/tests/test_litellm/test_azure_audio_price_aliases.py new file mode 100644 index 00000000000..b87744aeae1 --- /dev/null +++ b/tests/test_litellm/test_azure_audio_price_aliases.py @@ -0,0 +1,75 @@ +"""Undated azure aliases for the audio models must exist and match their dated +variants. Azure deployments are commonly created under an admin-chosen name, so +the served model name means nothing to the cost lookup and `base_model: +azure/gpt-audio-mini` is what prices the call. That key resolved to nothing, the +lookup raised "This model isn't mapped yet", and the proxy logged the request at +$0. Issue #33170.""" + +import json +from pathlib import Path + +import pytest + +import litellm + +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + + +COST_FIELDS = ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", +) + +ALIAS_PAIRS = ( + ("azure/gpt-audio-mini", "azure/gpt-audio-mini-2025-10-06"), + ("azure/gpt-realtime-mini", "azure/gpt-realtime-mini-2025-10-06"), +) + + +def _load_root_cost_map() -> dict: + root_map_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(root_map_path) as f: + return json.load(f) + + +@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS) +def test_undated_azure_audio_alias_matches_dated_entry(undated, dated): + undated_info = litellm.get_model_info(undated) + dated_info = litellm.get_model_info(dated) + + for field in COST_FIELDS: + assert undated_info.get(field) == dated_info.get(field), field + assert (undated_info.get(field) or 0) > 0, f"{undated}.{field} must be non-zero" + + assert undated_info.get("litellm_provider") == "azure" + assert undated_info.get("mode") == dated_info.get("mode") + + +@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS) +def test_undated_azure_audio_alias_is_exact_mirror(undated, dated): + """The undated alias must be a byte-for-byte mirror of its dated entry, covering + every field (incl. realtime-specific cache/audio cost keys) so any future drift + between the pair is caught, not just the core COST_FIELDS.""" + model_map = litellm.model_cost + assert undated in model_map, f"{undated} missing from model cost map" + assert model_map[undated] == model_map[dated], ( + f"{undated} must exactly mirror {dated}; " + f"diff keys: {[k for k in set(model_map[undated]) | set(model_map[dated]) if model_map[undated].get(k) != model_map[dated].get(k)]}" + ) + + +@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS) +def test_undated_azure_audio_alias_is_in_the_root_cost_map(undated, dated): + """`local_model_cost_map` pins `litellm.model_cost` to the packaged backup, but a + proxy left on its defaults fetches the root map instead, and that is the copy + that ships to the CDN. An alias added to only one of the two files still bills + $0 for every proxy reading the other, which is the very bug this file guards, so + assert the root map directly and assert the two files agree.""" + root_map = _load_root_cost_map() + assert undated in root_map, f"{undated} missing from the root cost map" + assert root_map[undated] == root_map[dated], f"{undated} must exactly mirror {dated} in the root cost map" + assert root_map[undated] == litellm.model_cost[undated], ( + f"{undated} differs between the root cost map and the packaged backup" + ) diff --git a/tests/test_litellm/test_command_r7b_pricing.py b/tests/test_litellm/test_command_r7b_pricing.py index b952c365910..498fc0ef55a 100644 --- a/tests/test_litellm/test_command_r7b_pricing.py +++ b/tests/test_litellm/test_command_r7b_pricing.py @@ -11,11 +11,7 @@ swap cannot silently regress. import json import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/test_litellm/test_constants.py b/tests/test_litellm/test_constants.py index b3c13c6e26e..12e473f68a4 100644 --- a/tests/test_litellm/test_constants.py +++ b/tests/test_litellm/test_constants.py @@ -1,8 +1,6 @@ import ast import inspect import json -import os -import sys from unittest import mock import httpx @@ -10,7 +8,6 @@ import pytest import respx from fastapi.testclient import TestClient -sys.path.insert(0, os.path.abspath("../..")) # import importlib diff --git a/tests/test_litellm/test_cost_calculation_log_level.py b/tests/test_litellm/test_cost_calculation_log_level.py index f5c03771cd7..f8d3557c572 100644 --- a/tests/test_litellm/test_cost_calculation_log_level.py +++ b/tests/test_litellm/test_cost_calculation_log_level.py @@ -1,10 +1,7 @@ """Test that cost calculation uses appropriate log levels""" import logging -import os -import sys -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm import completion_cost diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 98938dee62e..8dad4bef07b 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,12 +1,6 @@ -import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - from pydantic import BaseModel @@ -24,6 +18,12 @@ from litellm.types.utils import ModelInfo, ModelResponse, PromptTokensDetailsWra from litellm.utils import TranscriptionResponse +@pytest.fixture +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + def test_cost_per_token_duplicate_openai_prefix_matches_model_cost(monkeypatch): """ Router/proxy configs may use deployment ids like openai/openai/. Cost lookup must @@ -93,14 +93,12 @@ def test_cost_per_token_non_string_model_does_not_hang(): assert result.get("status") in ("returned", "raised") -def test_completion_cost_uses_response_model_for_dynamic_routing(): +def test_completion_cost_uses_response_model_for_dynamic_routing(_local_model_cost_map): """ Test that completion_cost uses the model from the response object when the input model (e.g., azure-model-router) is not in model_cost. This supports Azure Model Router and similar dynamic routing scenarios. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Simulate Azure Model Router: input is generic router, response has actual model response = ModelResponse( @@ -139,9 +137,7 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 -def test_baseten_model_api_pricing_entries(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_baseten_model_api_pricing_entries(_local_model_cost_map): expected_pricing = { "baseten/nvidia/Nemotron-120B-A12B": (3e-07, 7.5e-07), @@ -165,9 +161,7 @@ def test_baseten_model_api_pricing_entries(): assert model_info["output_cost_per_token"] == output_cost -def test_wandb_model_api_pricing_entries(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_wandb_model_api_pricing_entries(_local_model_cost_map): expected_pricing = { "wandb/moonshotai/Kimi-K2.5": (6e-07, 3e-06), @@ -182,9 +176,7 @@ def test_wandb_model_api_pricing_entries(): assert model_info["output_cost_per_token"] == output_cost -def test_openrouter_qwen36_plus_model_info(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_openrouter_qwen36_plus_model_info(_local_model_cost_map): model_info = litellm.model_cost.get("openrouter/qwen/qwen3.6-plus") @@ -208,9 +200,7 @@ def test_openrouter_qwen36_plus_model_info(): "github_copilot/mai-code-1-flash-internal", ], ) -def test_github_copilot_mai_code_1_flash_pricing(model): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_github_copilot_mai_code_1_flash_pricing(_local_model_cost_map, model): model_info = litellm.model_cost.get(model) @@ -238,9 +228,7 @@ def test_github_copilot_mai_code_1_flash_pricing(model): assert completion_usd == pytest.approx(500 * 4.5e-06) -def test_cost_calculator_with_usage(monkeypatch): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): usage = Usage( prompt_tokens=120, @@ -320,11 +308,9 @@ def test_cost_calculator_with_usage(monkeypatch): assert result == expected_cost, f"Got {result}, Expected {expected_cost}" -def test_transcription_cost_uses_token_pricing(): +def test_transcription_cost_uses_token_pricing(_local_model_cost_map): from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") usage = Usage( prompt_tokens=14, @@ -348,11 +334,9 @@ def test_transcription_cost_uses_token_pricing(): assert pytest.approx(cost, rel=1e-6) == expected_cost -def test_transcription_cost_falls_back_to_duration(): +def test_transcription_cost_falls_back_to_duration(_local_model_cost_map): from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") response = TranscriptionResponse(text="demo text") response.duration = 10.0 @@ -368,14 +352,12 @@ def test_transcription_cost_falls_back_to_duration(): assert pytest.approx(cost, rel=1e-6) == expected_cost -def test_vertex_chirp_3_transcription_cost_from_duration(): +def test_vertex_chirp_3_transcription_cost_from_duration(_local_model_cost_map): """Regression: the chirp_3 cost map entry shipped with output_cost_per_second 0.0, and cost_per_second prefers output_cost_per_second whenever it is not None, so every transcription priced to $0.00 instead of using input_cost_per_second.""" from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") response = TranscriptionResponse(text="demo text") response.duration = 18.0 @@ -1127,9 +1109,7 @@ def test_tiered_pricing_only_deployment_completion_cost_is_nonzero(): assert cost > 0 -def test_azure_realtime_cost_calculator(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_azure_realtime_cost_calculator(_local_model_cost_map): cost = handle_realtime_stream_cost_calculation( results=[ @@ -1152,7 +1132,7 @@ def test_azure_realtime_cost_calculator(): assert cost > 0 -def test_azure_audio_output_cost_calculation(): +def test_azure_audio_output_cost_calculation(_local_model_cost_map): """ Test that Azure audio models correctly calculate costs for audio output tokens. @@ -1162,8 +1142,6 @@ def test_azure_audio_output_cost_calculation(): """ from litellm.types.utils import Choices, CompletionTokensDetailsWrapper, Message - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Scenario from issue #19764: # Input: 17 text tokens, 0 audio tokens @@ -1672,7 +1650,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): assert expected_actual_cost == total_cost -def test_azure_ai_cache_cost_calculation(): +def test_azure_ai_cache_cost_calculation(_local_model_cost_map): """ Test that azure_ai provider correctly calculates cache costs using generic_cost_per_token. @@ -1683,8 +1661,6 @@ def test_azure_ai_cache_cost_calculation(): from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.types.utils import PromptTokensDetailsWrapper, Usage - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Register a custom azure_ai model with cache pricing test_model_id = "test-azure-ai-claude-model" @@ -1817,15 +1793,13 @@ def test_vertex_uplift_composes_with_above_128k_pricing(monkeypatch): assert regional_completion == pytest.approx(global_completion * 1.10, rel=1e-9) -def test_cost_discount_vertex_ai(): +def test_cost_discount_vertex_ai(monkeypatch): """ Test that cost discount is applied correctly for Vertex AI provider """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_discount_config = litellm.cost_discount_config.copy() # Create mock response (use a model that exists in model_prices_and_context_window.json) response = ModelResponse( @@ -1838,7 +1812,7 @@ def test_cost_discount_vertex_ai(): ) # Calculate cost without discount - litellm.cost_discount_config = {} + monkeypatch.setattr(litellm, "cost_discount_config", {}) cost_without_discount = completion_cost( completion_response=response, model="vertex_ai/gemini-3-pro-preview", @@ -1846,7 +1820,7 @@ def test_cost_discount_vertex_ai(): ) # Set 5% discount for vertex_ai - litellm.cost_discount_config = {"vertex_ai": 0.05} + monkeypatch.setattr(litellm, "cost_discount_config", {"vertex_ai": 0.05}) # Calculate cost with discount cost_with_discount = completion_cost( @@ -1855,8 +1829,6 @@ def test_cost_discount_vertex_ai(): custom_llm_provider="vertex_ai", ) - # Restore original config - litellm.cost_discount_config = original_discount_config # Verify discount is applied (5% off means 95% of original cost) expected_cost = cost_without_discount * 0.95 @@ -1868,15 +1840,13 @@ def test_cost_discount_vertex_ai(): print(f" - Savings: ${cost_without_discount - cost_with_discount:.6f}") -def test_cost_discount_not_applied_to_other_providers(): +def test_cost_discount_not_applied_to_other_providers(monkeypatch): """ Test that cost discount only applies to configured providers """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_discount_config = litellm.cost_discount_config.copy() # Create mock response for OpenAI response = ModelResponse( @@ -1889,7 +1859,7 @@ def test_cost_discount_not_applied_to_other_providers(): ) # Set discount only for vertex_ai (not openai) - litellm.cost_discount_config = {"vertex_ai": 0.05} + monkeypatch.setattr(litellm, "cost_discount_config", {"vertex_ai": 0.05}) # Calculate cost for OpenAI - should NOT have discount applied cost_with_selective_discount = completion_cost( @@ -1899,15 +1869,13 @@ def test_cost_discount_not_applied_to_other_providers(): ) # Clear discount config - litellm.cost_discount_config = {} + monkeypatch.setattr(litellm, "cost_discount_config", {}) cost_without_discount = completion_cost( completion_response=response, model="gpt-4", custom_llm_provider="openai", ) - # Restore original config - litellm.cost_discount_config = original_discount_config # Costs should be the same (no discount applied to OpenAI) assert cost_with_selective_discount == cost_without_discount @@ -1917,15 +1885,13 @@ def test_cost_discount_not_applied_to_other_providers(): print(f" - Cost remains unchanged: ${cost_with_selective_discount:.6f}") -def test_cost_margin_percentage(): +def test_cost_margin_percentage(monkeypatch): """ Test that percentage-based cost margin is applied correctly """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_margin_config = litellm.cost_margin_config.copy() # Create mock response response = ModelResponse( @@ -1938,7 +1904,7 @@ def test_cost_margin_percentage(): ) # Calculate cost without margin - litellm.cost_margin_config = {} + monkeypatch.setattr(litellm, "cost_margin_config", {}) cost_without_margin = completion_cost( completion_response=response, model="gpt-4", @@ -1946,7 +1912,7 @@ def test_cost_margin_percentage(): ) # Set 10% margin for openai - litellm.cost_margin_config = {"openai": 0.10} + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": 0.10}) # Calculate cost with margin cost_with_margin = completion_cost( @@ -1955,8 +1921,6 @@ def test_cost_margin_percentage(): custom_llm_provider="openai", ) - # Restore original config - litellm.cost_margin_config = original_margin_config # Verify margin is applied (10% margin means 110% of original cost) expected_cost = cost_without_margin * 1.10 @@ -1968,15 +1932,13 @@ def test_cost_margin_percentage(): print(f" - Margin added: ${cost_with_margin - cost_without_margin:.6f}") -def test_cost_margin_fixed_amount(): +def test_cost_margin_fixed_amount(monkeypatch): """ Test that fixed amount cost margin is applied correctly """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_margin_config = litellm.cost_margin_config.copy() # Create mock response response = ModelResponse( @@ -1989,7 +1951,7 @@ def test_cost_margin_fixed_amount(): ) # Calculate cost without margin - litellm.cost_margin_config = {} + monkeypatch.setattr(litellm, "cost_margin_config", {}) cost_without_margin = completion_cost( completion_response=response, model="gpt-4", @@ -1997,7 +1959,7 @@ def test_cost_margin_fixed_amount(): ) # Set $0.001 fixed margin for openai - litellm.cost_margin_config = {"openai": {"fixed_amount": 0.001}} + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": 0.001}}) # Calculate cost with margin cost_with_margin = completion_cost( @@ -2006,8 +1968,6 @@ def test_cost_margin_fixed_amount(): custom_llm_provider="openai", ) - # Restore original config - litellm.cost_margin_config = original_margin_config # Verify fixed margin is applied expected_cost = cost_without_margin + 0.001 @@ -2019,15 +1979,13 @@ def test_cost_margin_fixed_amount(): print(f" - Margin added: ${cost_with_margin - cost_without_margin:.6f}") -def test_cost_margin_combined(): +def test_cost_margin_combined(monkeypatch): """ Test that combined percentage and fixed amount margin is applied correctly """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_margin_config = litellm.cost_margin_config.copy() # Create mock response response = ModelResponse( @@ -2040,7 +1998,7 @@ def test_cost_margin_combined(): ) # Calculate cost without margin - litellm.cost_margin_config = {} + monkeypatch.setattr(litellm, "cost_margin_config", {}) cost_without_margin = completion_cost( completion_response=response, model="gpt-4", @@ -2048,9 +2006,9 @@ def test_cost_margin_combined(): ) # Set 8% margin + $0.0005 fixed for openai - litellm.cost_margin_config = { + monkeypatch.setattr(litellm, "cost_margin_config", { "openai": {"percentage": 0.08, "fixed_amount": 0.0005} - } + }) # Calculate cost with margin cost_with_margin = completion_cost( @@ -2059,8 +2017,6 @@ def test_cost_margin_combined(): custom_llm_provider="openai", ) - # Restore original config - litellm.cost_margin_config = original_margin_config # Verify combined margin is applied expected_cost = cost_without_margin * 1.08 + 0.0005 @@ -2072,15 +2028,13 @@ def test_cost_margin_combined(): print(f" - Margin added: ${cost_with_margin - cost_without_margin:.6f}") -def test_cost_margin_global(): +def test_cost_margin_global(monkeypatch): """ Test that global margin is applied when no provider-specific margin is configured """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_margin_config = litellm.cost_margin_config.copy() # Create mock response response = ModelResponse( @@ -2093,7 +2047,7 @@ def test_cost_margin_global(): ) # Calculate cost without margin - litellm.cost_margin_config = {} + monkeypatch.setattr(litellm, "cost_margin_config", {}) cost_without_margin = completion_cost( completion_response=response, model="gpt-4", @@ -2101,7 +2055,7 @@ def test_cost_margin_global(): ) # Set 5% global margin (no provider-specific margin) - litellm.cost_margin_config = {"global": 0.05} + monkeypatch.setattr(litellm, "cost_margin_config", {"global": 0.05}) # Calculate cost with global margin cost_with_global_margin = completion_cost( @@ -2110,8 +2064,6 @@ def test_cost_margin_global(): custom_llm_provider="openai", ) - # Restore original config - litellm.cost_margin_config = original_margin_config # Verify global margin is applied expected_cost = cost_without_margin * 1.05 @@ -2123,15 +2075,13 @@ def test_cost_margin_global(): print(f" - Margin added: ${cost_with_global_margin - cost_without_margin:.6f}") -def test_cost_margin_provider_overrides_global(): +def test_cost_margin_provider_overrides_global(monkeypatch): """ Test that provider-specific margin overrides global margin """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original config - original_margin_config = litellm.cost_margin_config.copy() # Create mock response response = ModelResponse( @@ -2144,7 +2094,7 @@ def test_cost_margin_provider_overrides_global(): ) # Calculate cost without margin - litellm.cost_margin_config = {} + monkeypatch.setattr(litellm, "cost_margin_config", {}) cost_without_margin = completion_cost( completion_response=response, model="gpt-4", @@ -2152,7 +2102,7 @@ def test_cost_margin_provider_overrides_global(): ) # Set 5% global margin and 10% provider-specific margin - litellm.cost_margin_config = {"global": 0.05, "openai": 0.10} + monkeypatch.setattr(litellm, "cost_margin_config", {"global": 0.05, "openai": 0.10}) # Calculate cost - should use provider-specific margin (10%), not global (5%) cost_with_provider_margin = completion_cost( @@ -2161,8 +2111,6 @@ def test_cost_margin_provider_overrides_global(): custom_llm_provider="openai", ) - # Restore original config - litellm.cost_margin_config = original_margin_config # Verify provider-specific margin is used (not global) expected_cost = cost_without_margin * 1.10 # 10% from provider, not 5% from global @@ -2176,16 +2124,13 @@ def test_cost_margin_provider_overrides_global(): print(f" - Margin added: ${cost_with_provider_margin - cost_without_margin:.6f}") -def test_cost_margin_with_discount(): +def test_cost_margin_with_discount(monkeypatch): """ Test that margin is applied after discount (independent calculation) """ from litellm import completion_cost from litellm.types.utils import Usage - # Save original configs - original_margin_config = litellm.cost_margin_config.copy() - original_discount_config = litellm.cost_discount_config.copy() # Create mock response response = ModelResponse( @@ -2198,8 +2143,8 @@ def test_cost_margin_with_discount(): ) # Calculate base cost - litellm.cost_margin_config = {} - litellm.cost_discount_config = {} + monkeypatch.setattr(litellm, "cost_margin_config", {}) + monkeypatch.setattr(litellm, "cost_discount_config", {}) base_cost = completion_cost( completion_response=response, model="gpt-4", @@ -2207,8 +2152,8 @@ def test_cost_margin_with_discount(): ) # Set 5% discount and 10% margin - litellm.cost_discount_config = {"openai": 0.05} - litellm.cost_margin_config = {"openai": 0.10} + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.05}) + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": 0.10}) # Calculate cost with both discount and margin cost_with_both = completion_cost( @@ -2217,9 +2162,6 @@ def test_cost_margin_with_discount(): custom_llm_provider="openai", ) - # Restore original configs - litellm.cost_margin_config = original_margin_config - litellm.cost_discount_config = original_discount_config # Verify: discount applied first, then margin # Base cost -> discount: base * 0.95 -> margin: (base * 0.95) * 1.10 @@ -2286,12 +2228,10 @@ def test_azure_image_generation_cost_calculator(): assert cost > 0.079 -def test_completion_cost_extracts_service_tier_from_response(): +def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_map): """Test that completion_cost extracts service_tier from completion_response object.""" from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -2338,12 +2278,10 @@ def test_completion_cost_extracts_service_tier_from_response(): ), f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" -def test_completion_cost_extracts_service_tier_from_usage(): +def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map): """Test that completion_cost extracts service_tier from usage object.""" from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -2397,12 +2335,10 @@ def test_completion_cost_extracts_service_tier_from_usage(): ), f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" -def test_completion_cost_service_tier_priority(): +def test_completion_cost_service_tier_priority(_local_model_cost_map): """Test that service_tier extraction follows priority: optional_params > completion_response > usage.""" from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -2457,12 +2393,10 @@ def test_completion_cost_service_tier_priority(): ), "Costs from params and usage should be similar (both flex)" -def test_completion_cost_service_tier_for_bedrock(): +def test_completion_cost_service_tier_for_bedrock(_local_model_cost_map): """Test that Bedrock cost calculation applies service_tier-specific pricing.""" from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "bedrock/us-east-1/test-bedrock-service-tier-cost-model" litellm.register_model( @@ -2507,7 +2441,7 @@ def test_completion_cost_service_tier_for_bedrock(): assert priority_cost > default_cost > flex_cost > 0 -def test_completion_cost_service_tier_for_anthropic(): +def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map): """ Anthropic priority-tier requests must be priced at the priority rate. @@ -2519,8 +2453,6 @@ def test_completion_cost_service_tier_for_anthropic(): from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-service-tier-cost-model" litellm.register_model( @@ -2561,7 +2493,7 @@ def test_completion_cost_service_tier_for_anthropic(): assert priority_cost == pytest.approx(2 * standard_cost) -def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(): +def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(_local_model_cost_map): """ Proxy billing path regression for LIT-3771. @@ -2574,8 +2506,6 @@ def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(): from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-auto-tier-cost-model" litellm.register_model( @@ -2613,7 +2543,7 @@ def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(): assert cost == pytest.approx(expected_priority) -def test_completion_cost_non_string_service_tier_defers_to_served_tier(): +def test_completion_cost_non_string_service_tier_defers_to_served_tier(_local_model_cost_map): """ Regression: a non-string request-level ``service_tier`` (reachable via ``allowed_openai_params``/``drop_params``) must not crash cost tracking. @@ -2627,8 +2557,6 @@ def test_completion_cost_non_string_service_tier_defers_to_served_tier(): from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-non-string-tier-cost-model" litellm.register_model( @@ -2665,7 +2593,7 @@ def test_completion_cost_non_string_service_tier_defers_to_served_tier(): assert cost == pytest.approx(expected_priority) -def test_completion_cost_non_string_response_service_tier_defers_to_served_tier(): +def test_completion_cost_non_string_response_service_tier_defers_to_served_tier(_local_model_cost_map): """ Regression: a non-string ``service_tier`` on the response object must not crash cost tracking. @@ -2679,8 +2607,6 @@ def test_completion_cost_non_string_response_service_tier_defers_to_served_tier( from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-response-non-string-tier-cost-model" litellm.register_model( @@ -2718,7 +2644,7 @@ def test_completion_cost_non_string_response_service_tier_defers_to_served_tier( assert cost == pytest.approx(expected_priority) -def test_completion_cost_non_string_usage_service_tier_prices_standard(): +def test_completion_cost_non_string_usage_service_tier_prices_standard(_local_model_cost_map): """ Regression: a non-string ``service_tier`` on the usage object must not crash cost tracking. @@ -2729,8 +2655,6 @@ def test_completion_cost_non_string_usage_service_tier_prices_standard(): """ from litellm import completion_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-usage-non-string-tier-cost-model" litellm.register_model( @@ -2764,7 +2688,7 @@ def test_completion_cost_non_string_usage_service_tier_prices_standard(): assert cost == pytest.approx(expected_standard) -def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(): +def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(_local_model_cost_map): """ Regression for the cache/tier interaction in the Anthropic geo/speed path. @@ -2780,8 +2704,6 @@ def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(): ) from litellm.types.utils import PromptTokensDetailsWrapper, Usage - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-priority-cache-fast-model" litellm.register_model( @@ -2837,7 +2759,7 @@ def _register_anthropic_geo_cache_model(model: str) -> None: ) -def test_anthropic_geo_multiplier_applies_to_cache_tokens(monkeypatch): +def test_anthropic_geo_multiplier_applies_to_cache_tokens(_local_model_cost_map, monkeypatch): """ Regression: the regional (geo) uplift must scale cache read and cache write cost too, not just non-cache input and output. @@ -2853,7 +2775,6 @@ def test_anthropic_geo_multiplier_applies_to_cache_tokens(monkeypatch): from litellm.types.utils import PromptTokensDetailsWrapper, Usage monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-geo-cache-model" _register_anthropic_geo_cache_model(model) @@ -2882,7 +2803,7 @@ def test_anthropic_geo_multiplier_applies_to_cache_tokens(monkeypatch): assert geo_completion_cost == pytest.approx(base_completion_cost * 1.1) -def test_anthropic_geo_and_fast_multipliers_compose(monkeypatch): +def test_anthropic_geo_and_fast_multipliers_compose(_local_model_cost_map, monkeypatch): """ The ``fast`` speed multiplier stays cache-exclusive (the old explicit ``fast/`` entries kept base cache rates) while the geo multiplier scales the @@ -2895,7 +2816,6 @@ def test_anthropic_geo_and_fast_multipliers_compose(monkeypatch): from litellm.types.utils import PromptTokensDetailsWrapper, Usage monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") model = "claude-test-geo-fast-cache-model" _register_anthropic_geo_cache_model(model) @@ -3100,7 +3020,7 @@ def test_gemini_implicit_caching_cost_calculation(): ) -def test_additional_costs_only_for_azure_ai(): +def test_additional_costs_only_for_azure_ai(_local_model_cost_map): """ Test that _get_additional_costs is only called for azure_ai provider. @@ -3111,8 +3031,6 @@ def test_additional_costs_only_for_azure_ai(): """ from litellm.cost_calculator import _get_additional_costs - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") # Non-azure_ai providers should return None result = _get_additional_costs( @@ -3140,7 +3058,7 @@ def test_additional_costs_only_for_azure_ai(): assert result is None, "Vertex AI should have no additional costs" -def test_openrouter_gemini_3_1_flash_lite_preview_pricing(): +def test_openrouter_gemini_3_1_flash_lite_preview_pricing(_local_model_cost_map): """ Test that openrouter/google/gemini-3.1-flash-lite-preview has a pricing entry. @@ -3150,8 +3068,6 @@ def test_openrouter_gemini_3_1_flash_lite_preview_pricing(): model_prices_and_context_window.json when other Gemini 3.x variants were present. This caused ValueError: This model isn't mapped yet during router pre-call checks. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_name = "openrouter/google/gemini-3.1-flash-lite-preview" model_info = litellm.model_cost.get(model_name) @@ -3164,9 +3080,7 @@ def test_openrouter_gemini_3_1_flash_lite_preview_pricing(): assert model_info["max_output_tokens"] == 65536 -def test_gemini_3_1_flash_lite_pricing(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") +def test_gemini_3_1_flash_lite_pricing(_local_model_cost_map): for model_name in ( "gemini-3.1-flash-lite", @@ -3489,7 +3403,7 @@ def test_custom_pricing_without_cache_keys_preserves_legacy_behavior(): assert cost == pytest.approx(expected) -def test_openrouter_gemini_3_1_flash_lite_stable_pricing(): +def test_openrouter_gemini_3_1_flash_lite_stable_pricing(_local_model_cost_map): """ Test that openrouter/google/gemini-3.1-flash-lite (stable, no -preview suffix) has a pricing entry. @@ -3505,8 +3419,6 @@ def test_openrouter_gemini_3_1_flash_lite_stable_pricing(): Pricing matches the existing -preview entry one-for-one (input $0.25/M, output $1.50/M, cache-read $0.025/M) — Google did not change costs at the GA cutover. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") model_name = "openrouter/google/gemini-3.1-flash-lite" model_info = litellm.model_cost.get(model_name) @@ -3520,7 +3432,7 @@ def test_openrouter_gemini_3_1_flash_lite_stable_pricing(): assert model_info["max_output_tokens"] == 65536 -def test_completion_cost_logs_reasoning_and_cache_breakdown(): +def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_map): """ completion_cost must surface explicit reasoning and cache-read costs into the cost_breakdown stored on the logging object, so they end up in the spend logs @@ -3531,8 +3443,6 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(): from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import Choices, CompletionTokensDetailsWrapper, Message - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") logging_obj = Logging( model="gemini-2.5-flash", @@ -3750,13 +3660,11 @@ def test_combine_usage_objects_sums_mirrored_cache_write_fields_once(): assert combined_pair.prompt_tokens_details.cache_creation_tokens == 100 -def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(): +def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_cost_map): """Regression: an Anthropic /v1/messages response reports cache reads as top-level cache_read_input_tokens with input_tokens excluding them. Reading that usage as Responses API usage dropped the cache tokens and billed the whole prompt at the uncached input rate, overstating spend on cache hits.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") response = { "id": "msg_1", @@ -3774,4 +3682,4 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(): custom_llm_provider="openai", ) - assert cost == pytest.approx(3 * 5e-6 + 4014 * 5e-7 + 5 * 3e-5, rel=1e-9) + assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9) diff --git a/tests/test_litellm/test_count_tokens_public_api.py b/tests/test_litellm/test_count_tokens_public_api.py index 1e2cf83dec0..86c33c3e8f7 100644 --- a/tests/test_litellm/test_count_tokens_public_api.py +++ b/tests/test_litellm/test_count_tokens_public_api.py @@ -4,10 +4,8 @@ Tests for litellm.acount_tokens() public API. import asyncio import os -import sys from unittest.mock import AsyncMock, patch -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.types.utils import TokenCountResponse @@ -144,20 +142,16 @@ def test_acount_tokens_api_error_falls_back(): assert result.total_tokens > 0 -def test_acount_tokens_no_api_key_falls_back(): +def test_acount_tokens_no_api_key_falls_back(monkeypatch): """Test that missing API key falls back to local counting.""" - env_backup = os.environ.pop("OPENAI_API_KEY", None) - try: - result = asyncio.run( - litellm.acount_tokens( - model="openai/gpt-4o", - messages=[{"role": "user", "content": "Hello"}], - ) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + result = asyncio.run( + litellm.acount_tokens( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello"}], ) + ) - # Should fall back to local tokenizer since no API key - assert result.total_tokens > 0 - assert result.tokenizer_type == "local_tokenizer" - finally: - if env_backup: - os.environ["OPENAI_API_KEY"] = env_backup + # Should fall back to local tokenizer since no API key + assert result.total_tokens > 0 + assert result.tokenizer_type == "local_tokenizer" diff --git a/tests/test_litellm/test_deepseek_model_metadata.py b/tests/test_litellm/test_deepseek_model_metadata.py index 4900af5d97d..b9eb33f0972 100644 --- a/tests/test_litellm/test_deepseek_model_metadata.py +++ b/tests/test_litellm/test_deepseek_model_metadata.py @@ -11,11 +11,7 @@ field set to ``True``. import json import os -import sys -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.utils import ( diff --git a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py index 67d6b9e76cf..276f54c116a 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py +++ b/tests/test_litellm/test_gemini_3_1_flash_lite_image_pricing.py @@ -1,147 +1,284 @@ -"""Pricing entry for ``gemini-3.1-flash-lite-image`` (Google's Nano Banana 2 Lite). - -Google publishes: $0.25/1M input, $1.50/1M text output, and $30/1M image-output -tokens for the Lite image model (https://cloud.google.com/vertex-ai/generative-ai/pricing). -A 1K image is ~1120 output image tokens => ~$0.0336 / image. - -Without this entry, ``completion_cost`` raises "model isn't mapped yet" and Vertex -generateContent pass-through cost tracking silently logs $0. These tests pin the -values in both the primary price map and the ``litellm/`` backup, and verify -``get_model_info`` / ``completion_cost`` surface them. -""" - import json -import os +from pathlib import Path + +import pytest import litellm from litellm import completion_cost -from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage +from litellm.cost_calculator import cost_per_token +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as gemini_image_generation_cost_calculator, +) +from litellm.llms.vertex_ai.image_generation.cost_calculator import ( + cost_calculator as vertex_image_generation_cost_calculator, +) +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) -VARIANTS = [ - "gemini-3.1-flash-lite-image", - "gemini/gemini-3.1-flash-lite-image", - "vertex_ai/gemini-3.1-flash-lite-image", -] +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" -EXPECTED = { - "input_cost_per_token": 2.5e-07, - "output_cost_per_token": 1.5e-06, - "output_cost_per_image_token": 3e-05, +UNPREFIXED = "gemini-3.1-flash-lite-image" +GEMINI = "gemini/gemini-3.1-flash-lite-image" +VERTEX = "vertex_ai/gemini-3.1-flash-lite-image" +ALL_KEYS = (UNPREFIXED, GEMINI, VERTEX) + +INPUT_COST = 2.5e-07 +INPUT_COST_BATCHES = 1.25e-07 +OUTPUT_TEXT_COST = 1.5e-06 +OUTPUT_TEXT_COST_BATCHES = 7.5e-07 +OUTPUT_IMAGE_TOKEN_COST = 3e-05 +OUTPUT_COST_PER_1K_IMAGE = 0.0336 +INPUT_COST_PER_IMAGE = 0.00028 +CACHE_READ_COST = 2.5e-08 +MAX_INPUT_TOKENS = 65536 +MAX_OUTPUT_TOKENS = 4096 +TOKENS_PER_1K_IMAGE = 1120 + +SHARED_FIELDS = { "mode": "image_generation", -} - -EXPECTED_CAPABILITIES = { - "max_output_tokens": 4096, - "max_tokens": 4096, + "input_cost_per_token": INPUT_COST, + "input_cost_per_token_batches": INPUT_COST_BATCHES, + "input_cost_per_image": INPUT_COST_PER_IMAGE, + "output_cost_per_token": OUTPUT_TEXT_COST, + "output_cost_per_token_batches": OUTPUT_TEXT_COST_BATCHES, + "output_cost_per_image": OUTPUT_COST_PER_1K_IMAGE, + "output_cost_per_image_token": OUTPUT_IMAGE_TOKEN_COST, + "max_input_tokens": MAX_INPUT_TOKENS, + "max_output_tokens": MAX_OUTPUT_TOKENS, + "max_tokens": MAX_OUTPUT_TOKENS, + "supported_endpoints": ["/v1/chat/completions", "/v1/completions", "/v1/batch"], + "supported_output_modalities": ["text", "image"], + "supports_reasoning": False, "supports_response_schema": False, - "supports_reasoning": True, + "supports_system_messages": True, + "supports_vision": True, } -EXPECTED_PER_ROUTE = { - "gemini-3.1-flash-lite-image": { - "supports_prompt_caching": True, - "supports_function_calling": False, - }, - "vertex_ai/gemini-3.1-flash-lite-image": { - "supports_prompt_caching": True, - "supports_function_calling": False, - }, - "gemini/gemini-3.1-flash-lite-image": { - "supports_prompt_caching": False, +VERTEX_ROUTE_FIELDS = { + "litellm_provider": "vertex_ai-language-models", + "cache_read_input_token_cost": CACHE_READ_COST, + "supported_modalities": ["text", "image", "video"], + "supports_function_calling": False, + "supports_pdf_input": True, + "supports_prompt_caching": True, + "supports_video_input": True, +} + +PER_ROUTE_FIELDS = { + UNPREFIXED: VERTEX_ROUTE_FIELDS, + VERTEX: VERTEX_ROUTE_FIELDS, + GEMINI: { + "litellm_provider": "gemini", + "supported_modalities": ["text", "image"], "supports_function_calling": True, - "input_cost_per_token_batches": 1.25e-07, - "output_cost_per_token_batches": 7.5e-07, + "supports_prompt_caching": False, + "rpm": 1000, + "tpm": 4000000, }, } +GROUNDING_FIELDS = ( + "supports_web_search", + "search_context_cost_per_query", + "web_search_billing_unit", +) -def _load_json(path: str) -> dict: + +def _load(path: Path) -> dict: with open(path, encoding="utf-8") as f: return json.load(f) -def _backup_path() -> str: - return os.path.join( - os.path.dirname(litellm.__file__), - "model_prices_and_context_window_backup.json", +@pytest.fixture +def local_model_cost_map(monkeypatch): + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", ALL_KEYS) +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_published_prices_are_registered(model: str, path: Path): + info = _load(path).get(model) + assert info is not None, f"{model} missing from {path.name}" + for field, value in SHARED_FIELDS.items(): + assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" + + +@pytest.mark.parametrize("model", ALL_KEYS) +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_per_route_capabilities_match_model_cards(model: str, path: Path): + info = _load(path)[model] + for field, value in PER_ROUTE_FIELDS[model].items(): + assert info[field] == value, f"{model} {field} in {path.name}: {info.get(field)} != {value}" + + +@pytest.mark.parametrize("model", ALL_KEYS) +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_grounding_fields_absent(model: str, path: Path): + info = _load(path)[model] + for field in GROUNDING_FIELDS: + assert field not in info, f"{model} should not define {field}" + + +@pytest.mark.parametrize("path", (MAIN_PATH, BACKUP_PATH), ids=("main", "backup")) +def test_ai_studio_route_has_no_implicit_cache_price(path: Path): + assert "cache_read_input_token_cost" not in _load(path)[GEMINI] + + +@pytest.mark.parametrize("model", ALL_KEYS) +def test_backup_matches_main(model: str): + assert _load(BACKUP_PATH).get(model) == _load(MAIN_PATH).get(model) + + +def test_one_k_image_price_matches_official_token_math(): + assert TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST == pytest.approx(OUTPUT_COST_PER_1K_IMAGE) + assert TOKENS_PER_1K_IMAGE * INPUT_COST == pytest.approx(INPUT_COST_PER_IMAGE) + + +def test_gemini_prefix_routes_to_gemini(): + routed_model, provider, _, _ = get_llm_provider(model=GEMINI) + assert routed_model == UNPREFIXED + assert provider == "gemini" + + +def test_vertex_prefix_routes_to_vertex(): + routed_model, provider, _, _ = get_llm_provider(model=VERTEX) + assert routed_model == UNPREFIXED + assert provider == "vertex_ai" + + +def test_get_model_info_reports_published_costs(local_model_cost_map): + info = litellm.get_model_info(UNPREFIXED) + assert info["input_cost_per_token"] == INPUT_COST + assert info["output_cost_per_token"] == OUTPUT_TEXT_COST + assert info["cache_read_input_token_cost"] == CACHE_READ_COST + + +@pytest.mark.parametrize("model", ALL_KEYS) +def test_reasoning_params_are_not_offered_on_an_image_endpoint(model: str, local_model_cost_map): + assert litellm.supports_reasoning(model) is False + + +def test_text_token_cost(local_model_cost_map): + prompt_cost, text_completion_cost = cost_per_token( + model=GEMINI, prompt_tokens=1000, completion_tokens=500 + ) + assert prompt_cost == pytest.approx(1000 * INPUT_COST) + assert text_completion_cost == pytest.approx(500 * OUTPUT_TEXT_COST) + + +def test_completion_cost_bills_one_k_image(local_model_cost_map): + response = ModelResponse() + response.model = UNPREFIXED + response.usage = Usage( + prompt_tokens=7, + completion_tokens=TOKENS_PER_1K_IMAGE, + total_tokens=7 + TOKENS_PER_1K_IMAGE, + completion_tokens_details=CompletionTokensDetailsWrapper( + image_tokens=TOKENS_PER_1K_IMAGE, text_tokens=0 + ), + ) + billed = completion_cost( + completion_response=response, + model=UNPREFIXED, + custom_llm_provider="vertex_ai", + ) + expected = TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + 7 * INPUT_COST + assert billed == pytest.approx(expected) + + +def test_image_tokens_are_not_billed_as_text(local_model_cost_map): + usage = Usage( + completion_tokens=1345, + prompt_tokens=10, + total_tokens=1355, + completion_tokens_details=CompletionTokensDetailsWrapper( + accepted_prediction_tokens=None, + audio_tokens=None, + reasoning_tokens=225, + rejected_prediction_tokens=None, + text_tokens=0, + image_tokens=TOKENS_PER_1K_IMAGE, + ), + prompt_tokens_details=PromptTokensDetailsWrapper( + audio_tokens=None, cached_tokens=None, text_tokens=10, image_tokens=None + ), + ) + + _, image_completion_cost = generic_cost_per_token( + model=UNPREFIXED, + usage=usage, + custom_llm_provider="vertex_ai", + ) + + expected_completion_cost = ( + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + 225 * OUTPUT_TEXT_COST + ) + bugged_text_only_cost = 1345 * OUTPUT_TEXT_COST + assert image_completion_cost > bugged_text_only_cost * 2 + assert image_completion_cost == pytest.approx(expected_completion_cost) + + +def _one_k_image_response() -> ImageResponse: + return ImageResponse( + data=[ImageObject(b64_json="img1")], + usage=ImageUsage( + input_tokens=50 + TOKENS_PER_1K_IMAGE, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=50, + image_tokens=TOKENS_PER_1K_IMAGE, + ), + output_tokens=TOKENS_PER_1K_IMAGE, + total_tokens=50 + TOKENS_PER_1K_IMAGE + TOKENS_PER_1K_IMAGE, + ), ) -def _main_path() -> str: - return os.path.join( - os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json" +def test_gemini_image_generation_uses_token_pricing(local_model_cost_map): + cost = gemini_image_generation_cost_calculator( + model=GEMINI, image_response=_one_k_image_response() ) + expected = ( + 50 + TOKENS_PER_1K_IMAGE + ) * INPUT_COST + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + assert cost == pytest.approx(expected) + assert cost != OUTPUT_COST_PER_1K_IMAGE -class TestGeminiFlashLiteImagePricingData: - """Both price maps must carry Google's published Nano Banana 2 Lite costs.""" - - def test_present_in_both_maps(self): - main = _load_json(_main_path()) - backup = _load_json(_backup_path()) - for key in VARIANTS: - for label, data in (("main", main), ("backup", backup)): - assert key in data, f"{key} missing from {label} JSON" - entry = data[key] - for field, value in EXPECTED.items(): - assert entry[field] == value, f"{key} {field} in {label}: {entry.get(field)} != {value}" - - def test_capabilities_match_model_cards(self): - main = _load_json(_main_path()) - backup = _load_json(_backup_path()) - for key in VARIANTS: - expected = {**EXPECTED_CAPABILITIES, **EXPECTED_PER_ROUTE[key]} - for label, data in (("main", main), ("backup", backup)): - entry = data[key] - for field, value in expected.items(): - assert entry[field] == value, f"{key} {field} in {label}: {entry.get(field)} != {value}" - - def test_grounding_fields_absent(self): - """Grounding with Google Search is unsupported on Lite, so no search pricing.""" - for path in (_main_path(), _backup_path()): - data = _load_json(path) - for key in VARIANTS: - for field in ( - "supports_web_search", - "search_context_cost_per_query", - "web_search_billing_unit", - ): - assert field not in data[key], f"{key} should not define {field}" - - def test_image_output_pricing_consistent(self): - """1120 image-output tokens * output_cost_per_image_token == output_cost_per_image.""" - backup = _load_json(_backup_path()) - entry = backup["gemini-3.1-flash-lite-image"] - assert round(1120 * entry["output_cost_per_image_token"], 6) == entry["output_cost_per_image"] +def test_vertex_image_generation_uses_token_pricing(local_model_cost_map): + cost = vertex_image_generation_cost_calculator( + model=UNPREFIXED, image_response=_one_k_image_response() + ) + expected = ( + 50 + TOKENS_PER_1K_IMAGE + ) * INPUT_COST + TOKENS_PER_1K_IMAGE * OUTPUT_IMAGE_TOKEN_COST + assert cost == pytest.approx(expected) -class TestGeminiFlashLiteImageModelInfo: - """``get_model_info`` and ``completion_cost`` must report the new costs.""" - - def test_get_model_info_and_cost(self): - original = litellm.model_cost - try: - litellm.model_cost = _load_json(_backup_path()) - info = litellm.get_model_info("gemini-3.1-flash-lite-image") - assert info["input_cost_per_token"] == EXPECTED["input_cost_per_token"] - assert info["output_cost_per_token"] == EXPECTED["output_cost_per_token"] - - resp = ModelResponse() - resp.model = "gemini-3.1-flash-lite-image" - resp.usage = Usage( - prompt_tokens=7, - completion_tokens=1120, - total_tokens=1127, - completion_tokens_details=CompletionTokensDetailsWrapper( - image_tokens=1120, text_tokens=0 - ), - ) - cost = completion_cost( - completion_response=resp, - model="gemini-3.1-flash-lite-image", - custom_llm_provider="vertex_ai", - ) - expected_cost = 1120 * 3e-05 + 7 * 2.5e-07 - assert abs(cost - expected_cost) < 1e-6, f"unexpected cost {cost}" - finally: - litellm.model_cost = original +def test_vertex_image_generation_falls_back_to_flat_image_price(local_model_cost_map): + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + cost = vertex_image_generation_cost_calculator( + model=UNPREFIXED, image_response=image_response + ) + assert cost == pytest.approx(2 * OUTPUT_COST_PER_1K_IMAGE) diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py index c371f7442be..d3ec0673fe3 100644 --- a/tests/test_litellm/test_gpt_image_cost_calculator.py +++ b/tests/test_litellm/test_gpt_image_cost_calculator.py @@ -10,10 +10,7 @@ gpt-image-1 uses token-based pricing: - Image Output: $40.00/1M tokens """ -import os -import sys -sys.path.insert(0, os.path.abspath("../..")) import pytest diff --git a/tests/test_litellm/test_gpt_realtime_mode.py b/tests/test_litellm/test_gpt_realtime_mode.py index 4413cbc12ef..ed593228621 100644 --- a/tests/test_litellm/test_gpt_realtime_mode.py +++ b/tests/test_litellm/test_gpt_realtime_mode.py @@ -10,6 +10,7 @@ from litellm.types.utils import ModelInfoBase REALTIME_ONLY_GPT_MODELS = ( "azure/gpt-realtime-2025-08-28", "azure/gpt-realtime-1.5-2026-02-23", + "azure/gpt-realtime-mini", "azure/gpt-realtime-mini-2025-10-06", "gpt-realtime", "gpt-realtime-1.5", diff --git a/tests/test_litellm/test_lazy_imports.py b/tests/test_litellm/test_lazy_imports.py index f7c9cfa3074..2b16a812611 100644 --- a/tests/test_litellm/test_lazy_imports.py +++ b/tests/test_litellm/test_lazy_imports.py @@ -1,11 +1,9 @@ """Simple tests for lazy import functionality.""" -import os import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm._lazy_imports import ( diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 8551085cbd6..db8dfaa3ad6 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,7 +1,6 @@ import ast import asyncio import json -import os import re import sys from pathlib import Path @@ -9,9 +8,7 @@ from typing import List import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system-path import logging -import sys import litellm from litellm._logging import ( diff --git a/tests/test_litellm/test_lowest_latency_zero_tokens.py b/tests/test_litellm/test_lowest_latency_zero_tokens.py index b9fc9b00cc7..ff60744e9ee 100644 --- a/tests/test_litellm/test_lowest_latency_zero_tokens.py +++ b/tests/test_litellm/test_lowest_latency_zero_tokens.py @@ -1,14 +1,9 @@ #### What this tests #### # This tests the router's handling of zero completion tokens in lowest latency routing -import os -import sys import time import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.caching.caching import DualCache diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 4b223a3a900..c05f25430c2 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2,16 +2,12 @@ import contextlib import copy import json import os -import sys import httpx import pytest import respx from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import urllib.parse from unittest.mock import MagicMock, patch @@ -2754,3 +2750,159 @@ def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6() {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} ] assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"} + + +_SUBSCRIPTION_OAUTH_CREDENTIAL = "Bearer sk-ant-oat01-fake-subscription-token-for-testing-0123456789" + + +def _scoped_headers_for_oauth_request(): + from litellm.types.utils import ProviderSpecificHeader + + return [ + ProviderSpecificHeader( + custom_llm_provider="anthropic,bedrock,vertex_ai", + extra_headers={"anthropic-version": "2023-06-01"}, + ), + ProviderSpecificHeader( + custom_llm_provider="anthropic", + extra_headers={"authorization": _SUBSCRIPTION_OAUTH_CREDENTIAL}, + ), + ] + + +def _run_anthropic_hop_with_shared_headers(shared_headers): + litellm.completion( + model="anthropic/claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "Say OK"}], + extra_headers=shared_headers, + provider_specific_header=_scoped_headers_for_oauth_request(), + api_key="sk-fake-anthropic-key", + mock_response="OK", + ) + + +def test_completion_does_not_mutate_caller_supplied_headers(): + shared_headers = {"x-tenant": "acme"} + + _run_anthropic_hop_with_shared_headers(shared_headers) + + assert shared_headers == {"x-tenant": "acme"} + + +def test_anthropic_oauth_credential_does_not_persist_into_next_provider_hop(): + shared_headers = {"x-tenant": "acme"} + + _run_anthropic_hop_with_shared_headers(shared_headers) + + leaked = [name for name, value in shared_headers.items() if value == _SUBSCRIPTION_OAUTH_CREDENTIAL] + assert leaked == [] + assert "anthropic-version" not in shared_headers + + +STREAM_COST_MODEL = "gpt-4o" +STREAMED_USAGE = {"prompt_tokens": 137, "completion_tokens": 42, "total_tokens": 179} + + +def _text_chunk(content, finish_reason=None, usage=None): + chunk = { + "id": "chatcmpl-stream-cost", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": STREAM_COST_MODEL, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": content}, + "finish_reason": finish_reason, + } + ], + } + if usage is not None: + chunk["usage"] = usage + return chunk + + +def _priced_at(prompt_tokens, completion_tokens): + prices = litellm.model_cost[STREAM_COST_MODEL] + return ( + prompt_tokens * prices["input_cost_per_token"] + + completion_tokens * prices["output_cost_per_token"] + ) + + +@pytest.fixture +def local_cost_map(monkeypatch): + """The prices these tests assert are the checked-in ones. Setting the environment + variable alone does not reload the map, so pin the map itself.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + +def test_a_streamed_response_bills_the_usage_the_provider_reported(local_cost_map): + rebuilt = litellm.stream_chunk_builder( + chunks=[ + _text_chunk("Hello"), + _text_chunk(" there"), + _text_chunk(None, finish_reason="stop", usage=STREAMED_USAGE), + ], + messages=[{"role": "user", "content": "hi"}], + ) + + assert rebuilt.choices[0].message.content == "Hello there" + assert rebuilt.usage.prompt_tokens == STREAMED_USAGE["prompt_tokens"] + assert rebuilt.usage.completion_tokens == STREAMED_USAGE["completion_tokens"] + + cost = litellm.completion_cost(completion_response=rebuilt, model=STREAM_COST_MODEL) + + assert cost == pytest.approx(_priced_at(137, 42)) + assert cost == pytest.approx(0.0007625) + + +def test_streaming_and_not_streaming_bill_the_same_usage_the_same(local_cost_map): + rebuilt = litellm.stream_chunk_builder( + chunks=[ + _text_chunk("Hello"), + _text_chunk(" there"), + _text_chunk(None, finish_reason="stop", usage=STREAMED_USAGE), + ], + messages=[{"role": "user", "content": "hi"}], + ) + whole = litellm.ModelResponse( + id="chatcmpl-stream-cost", + model=STREAM_COST_MODEL, + object="chat.completion", + created=1700000000, + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello there"}, + "finish_reason": "stop", + } + ], + usage=STREAMED_USAGE, + ) + + assert litellm.completion_cost( + completion_response=rebuilt, model=STREAM_COST_MODEL + ) == pytest.approx(litellm.completion_cost(completion_response=whole, model=STREAM_COST_MODEL)) + + +def test_a_stream_that_reported_no_usage_is_still_billed(local_cost_map): + rebuilt = litellm.stream_chunk_builder( + chunks=[ + _text_chunk("Hello"), + _text_chunk(" there"), + _text_chunk(None, finish_reason="stop"), + ], + messages=[{"role": "user", "content": "hi"}], + ) + + assert rebuilt.usage.prompt_tokens > 0 + assert rebuilt.usage.completion_tokens > 0 + + cost = litellm.completion_cost(completion_response=rebuilt, model=STREAM_COST_MODEL) + + assert cost > 0 + assert cost == pytest.approx( + _priced_at(rebuilt.usage.prompt_tokens, rebuilt.usage.completion_tokens) + ) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index cb7023e6c12..6114d1d8aba 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -11,6 +11,7 @@ import pytest REPO_ROOT = Path(__file__).parents[2] GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" @@ -118,6 +119,31 @@ def test_schema_accepts_cache_creation_cost_inside_a_pricing_tier(committed_sche assert validator.is_valid({"some-model": entry}) +def find_duplicate_keys(path: Path) -> list[str]: + duplicates: list[str] = [] + + def record_duplicates(pairs): + seen: set[str] = set() + for key, _ in pairs: + if key in seen: + duplicates.append(key) + seen.add(key) + return dict(pairs) + + json.loads(path.read_text(), object_pairs_hook=record_duplicates) + return duplicates + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_price_map_has_no_duplicate_keys(path: Path): + assert find_duplicate_keys(path) == [], ( + f"{path.name} defines the same key twice; JSON parsers keep only the last " + "occurrence, so the earlier entry's fields are silently dropped. This is what " + "a clean text merge of two branches that both added a model looks like: " + "deduplicate the keys into one entry" + ) + + DATED_VARIANT = re.compile(r"^(.*?)-(\d{4}-\d{2}-\d{2})$") SERVICE_TIER_SUFFIXES = ("_flex", "_priority") diff --git a/tests/test_litellm/test_mutation_report.py b/tests/test_litellm/test_mutation_report.py new file mode 100644 index 00000000000..60b29ef2628 --- /dev/null +++ b/tests/test_litellm/test_mutation_report.py @@ -0,0 +1,139 @@ +"""Tests for scripts/mutation_report.py. + +The report is the only thing anyone reads after a mutation run, so the one thing it +must never do is describe a run that produced nothing as a run that killed everything. +`render` decides that wording and `get_survivors` supplies the evidence for it, so both +are tested directly. +""" + +import importlib.util +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_MODULE_PATH = _REPO_ROOT / "scripts" / "mutation_report.py" +_spec = importlib.util.spec_from_file_location("mutation_report", _MODULE_PATH) +report = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = report +_spec.loader.exec_module(report) + +_CONFIG = {"paths_to_mutate": ["litellm/proxy/management_endpoints/"], "tests_dir": ["tests/"]} + + +def test_a_run_that_reported_nothing_is_not_a_clean_sweep(): + rendered = report.render(_CONFIG, report.MutmutResults(survivors=(), reported=0), None) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + + +def test_a_run_that_killed_every_mutant_says_so(): + rendered = report.render( + _CONFIG, report.MutmutResults(survivors=(), reported=0), {"killed": 48, "survived": 0} + ) + + assert "caught every mutation" in rendered + assert "not a passing score" not in rendered + + +def test_stats_counting_survivors_results_never_listed_is_not_a_clean_sweep(): + rendered = report.render( + _CONFIG, report.MutmutResults(survivors=(), reported=0), {"killed": 48, "survived": 3} + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + assert "3 surviving mutant(s)" in rendered + + +def test_mutants_that_never_reached_the_tests_are_not_a_clean_sweep(): + rendered = report.render( + _CONFIG, + report.MutmutResults(survivors=(), reported=0), + {"killed": 48, "survived": 0, "no_tests": 4, "timeout": 1}, + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + assert "4 no tests" in rendered + assert "1 timeout" in rendered + + +def test_a_status_the_reporter_has_never_met_still_blocks_a_clean_sweep(): + rendered = report.render( + _CONFIG, + report.MutmutResults(survivors=(), reported=0), + {"killed": 48, "survived": 0, "check_was_interrupted_by_user": 2}, + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + assert "2 check was interrupted by user" in rendered + + +def test_no_survivors_without_a_kill_is_not_a_clean_sweep(): + rendered = report.render( + _CONFIG, report.MutmutResults(survivors=(), reported=48), {"killed": 0, "survived": 0} + ) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + + +def test_no_survivors_and_no_stats_cannot_claim_a_sweep(): + """`mutmut results` never lists killed mutants, so with the stats file missing an + empty survivor list is equally consistent with a perfect run and a dead one.""" + rendered = report.render(_CONFIG, report.MutmutResults(survivors=(), reported=48), None) + + assert "not a passing score" in rendered + assert "caught every mutation" not in rendered + + +def test_survivors_are_read_out_of_the_verdicts_they_came_with(monkeypatch): + class _Proc: + stdout = ( + "litellm.proxy.management_endpoints.key_management_endpoints.x_1: killed\n" + "litellm.proxy.management_endpoints.key_management_endpoints.x_2: survived\n" + "litellm.proxy.management_endpoints.key_management_endpoints.x_3: no tests\n" + "not a verdict line at all\n" + ) + + monkeypatch.setattr(report.subprocess, "run", lambda *a, **k: _Proc()) + + results = report.get_survivors() + + assert results.survivors == ( + "litellm.proxy.management_endpoints.key_management_endpoints.x_2", + ) + assert results.reported == 3 + + +def test_every_multi_word_verdict_mutmut_can_emit_still_counts(monkeypatch): + class _Proc: + stdout = "".join( + f"litellm.proxy.management_endpoints.key_management_endpoints.x_{i}: {verdict}\n" + for i, verdict in enumerate( + ( + "no tests", + "not checked", + "caught by type check", + "check was interrupted by user", + ) + ) + ) + + monkeypatch.setattr(report.subprocess, "run", lambda *a, **k: _Proc()) + + results = report.get_survivors() + + assert results.survivors == () + assert results.reported == 4 + + +def test_an_empty_mutmut_results_reports_nothing_rather_than_zero_survivors(monkeypatch): + class _Proc: + stdout = "" + + monkeypatch.setattr(report.subprocess, "run", lambda *a, **k: _Proc()) + + assert report.get_survivors() == report.MutmutResults(survivors=(), reported=0) diff --git a/tests/test_litellm/test_project_alias_tracking.py b/tests/test_litellm/test_project_alias_tracking.py index d18989d543f..476dfba0a27 100644 --- a/tests/test_litellm/test_project_alias_tracking.py +++ b/tests/test_litellm/test_project_alias_tracking.py @@ -5,12 +5,9 @@ Verifies that project_alias flows from UserAPIKeyAuth through the metadata pipel to StandardLoggingMetadata, mirroring how team_alias already works. """ -import os -import sys import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.proxy._types import LiteLLM_VerificationTokenView, UserAPIKeyAuth diff --git a/tests/test_litellm/test_redact_string_in_error_paths.py b/tests/test_litellm/test_redact_string_in_error_paths.py index 1c4d91397d1..6404db91acf 100644 --- a/tests/test_litellm/test_redact_string_in_error_paths.py +++ b/tests/test_litellm/test_redact_string_in_error_paths.py @@ -9,14 +9,11 @@ Covers actual execution of redaction in: """ import logging -import os -import sys import traceback from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string diff --git a/tests/test_litellm/test_register_model_custom_pricing.py b/tests/test_litellm/test_register_model_custom_pricing.py index ba82bfaadc6..e3f6a1a0f40 100644 --- a/tests/test_litellm/test_register_model_custom_pricing.py +++ b/tests/test_litellm/test_register_model_custom_pricing.py @@ -11,13 +11,9 @@ calculations for DB-sourced models with prompt caching pricing. import copy import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.main import _build_custom_pricing_entry @@ -318,7 +314,7 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp litellm.model_cost.pop(model_key, None) -def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(): +def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypatch): """Registering a custom override under a key shape that ``get_model_info`` cannot resolve (e.g. a triple provider prefix like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6``; a double @@ -338,7 +334,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(): from litellm.types.utils import PromptTokensDetailsWrapper, Usage original_model_cost = litellm.model_cost - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") builtin_key = "us.anthropic.claude-sonnet-4-6" diff --git a/tests/test_litellm/test_responses_api_bridge_non_stream.py b/tests/test_litellm/test_responses_api_bridge_non_stream.py index 08d55ee8290..617b2cfc031 100644 --- a/tests/test_litellm/test_responses_api_bridge_non_stream.py +++ b/tests/test_litellm/test_responses_api_bridge_non_stream.py @@ -1,11 +1,8 @@ -import os -import sys from typing import Final, Optional from unittest.mock import Mock import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.completion_extras.litellm_responses_transformation.handler import ( ResponsesToCompletionBridgeHandler, diff --git a/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py b/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py index aa057b7bc73..9d0daa52645 100644 --- a/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py +++ b/tests/test_litellm/test_retrieve_batch_bedrock_dispatch.py @@ -14,13 +14,10 @@ here is purely the dispatch logic that lives in ``main.py``. from __future__ import annotations -import os -import sys from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm # noqa: E402 import openai diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 58a500def8e..56e00ecdad6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -3,15 +3,11 @@ import copy import json import logging import os -import sys import threading from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm diff --git a/tests/test_litellm/test_router_exception_redaction.py b/tests/test_litellm/test_router_exception_redaction.py index 2066352e2ce..6754775db22 100644 --- a/tests/test_litellm/test_router_exception_redaction.py +++ b/tests/test_litellm/test_router_exception_redaction.py @@ -115,14 +115,9 @@ def _router_with_credentialed_fallback() -> Router: @pytest.fixture(autouse=True) -def _reset_expose_flag(): +def _reset_expose_flag(monkeypatch: pytest.MonkeyPatch) -> None: """Each test starts with the flag in its default (on) state.""" - original = litellm.expose_router_debug_in_errors - litellm.expose_router_debug_in_errors = True - try: - yield - finally: - litellm.expose_router_debug_in_errors = original + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) def test_flag_defaults_on(): @@ -133,8 +128,8 @@ def test_flag_defaults_on(): @pytest.mark.asyncio -async def test_flag_off_does_not_leak_received_model_group(): - litellm.expose_router_debug_in_errors = False +async def test_flag_off_does_not_leak_received_model_group(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", False) router = _router_with_rate_limit_failure() with pytest.raises(litellm.RateLimitError) as excinfo: await router.acompletion( @@ -148,8 +143,8 @@ async def test_flag_off_does_not_leak_received_model_group(): @pytest.mark.asyncio -async def test_flag_on_shows_received_model_group(): - litellm.expose_router_debug_in_errors = True +async def test_flag_on_shows_received_model_group(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) router = _router_with_rate_limit_failure() with pytest.raises(litellm.RateLimitError) as excinfo: await router.acompletion( @@ -166,8 +161,8 @@ async def test_flag_on_shows_received_model_group(): @pytest.mark.asyncio -async def test_flag_off_does_not_leak_context_window_fallback_hint(): - litellm.expose_router_debug_in_errors = False +async def test_flag_off_does_not_leak_context_window_fallback_hint(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", False) router = _router_with_context_window_failure() with pytest.raises(litellm.ContextWindowExceededError) as excinfo: await router.acompletion( @@ -181,8 +176,8 @@ async def test_flag_off_does_not_leak_context_window_fallback_hint(): @pytest.mark.asyncio -async def test_flag_on_shows_context_window_fallback_hint(): - litellm.expose_router_debug_in_errors = True +async def test_flag_on_shows_context_window_fallback_hint(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) router = _router_with_context_window_failure() with pytest.raises(litellm.ContextWindowExceededError) as excinfo: await router.acompletion( @@ -201,8 +196,8 @@ async def test_flag_on_shows_context_window_fallback_hint(): @pytest.mark.asyncio -async def test_flag_off_does_not_leak_when_no_fallback_group_found(): - litellm.expose_router_debug_in_errors = False +async def test_flag_off_does_not_leak_when_no_fallback_group_found(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", False) router = Router( model_list=[ { @@ -232,8 +227,8 @@ async def test_flag_off_does_not_leak_when_no_fallback_group_found(): @pytest.mark.asyncio -async def test_flag_on_shows_when_no_fallback_group_found(): - litellm.expose_router_debug_in_errors = True +async def test_flag_on_shows_when_no_fallback_group_found(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) router = Router( model_list=[ { @@ -284,8 +279,8 @@ def _router_with_plain_deployment() -> Router: @pytest.mark.asyncio -async def test_flag_off_does_not_leak_deployment_timeout_debug(): - litellm.expose_router_debug_in_errors = False +async def test_flag_off_does_not_leak_deployment_timeout_debug(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", False) router = _router_with_plain_deployment() with pytest.raises(litellm.Timeout) as excinfo: await router.acompletion( @@ -299,8 +294,8 @@ async def test_flag_off_does_not_leak_deployment_timeout_debug(): @pytest.mark.asyncio -async def test_flag_on_shows_deployment_timeout_debug(): - litellm.expose_router_debug_in_errors = True +async def test_flag_on_shows_deployment_timeout_debug(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) router = _router_with_plain_deployment() with pytest.raises(litellm.Timeout) as excinfo: await router.acompletion( @@ -325,8 +320,8 @@ def _content_policy_error() -> litellm.ContentPolicyViolationError: @pytest.mark.asyncio -async def test_flag_off_does_not_leak_content_policy_fallback_hint(): - litellm.expose_router_debug_in_errors = False +async def test_flag_off_does_not_leak_content_policy_fallback_hint(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", False) router = _router_with_plain_deployment() with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: await router.acompletion( @@ -340,8 +335,8 @@ async def test_flag_off_does_not_leak_content_policy_fallback_hint(): @pytest.mark.asyncio -async def test_flag_on_shows_content_policy_fallback_hint(): - litellm.expose_router_debug_in_errors = True +async def test_flag_on_shows_content_policy_fallback_hint(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) router = _router_with_plain_deployment() with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: await router.acompletion( @@ -358,8 +353,8 @@ async def test_flag_on_shows_content_policy_fallback_hint(): @pytest.mark.asyncio -async def test_flag_off_hides_fallback_credentials(): - litellm.expose_router_debug_in_errors = False +async def test_flag_off_hides_fallback_credentials(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", False) router = _router_with_credentialed_fallback() with pytest.raises(litellm.RateLimitError) as excinfo: await router.acompletion( @@ -372,8 +367,8 @@ async def test_flag_off_hides_fallback_credentials(): @pytest.mark.asyncio -async def test_flag_on_masks_fallback_credentials(): - litellm.expose_router_debug_in_errors = True +async def test_flag_on_masks_fallback_credentials(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) router = _router_with_credentialed_fallback() with pytest.raises(litellm.RateLimitError) as excinfo: await router.acompletion( @@ -389,14 +384,14 @@ async def test_flag_on_masks_fallback_credentials(): @pytest.mark.asyncio -async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(): +async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(monkeypatch: pytest.MonkeyPatch): """If the fallback attempt itself raises an exception whose message embeds a raw provider credential (e.g. a provider SDK echoing back the api_key it was called with), that string is re-embedded via `Error doing the fallback: ...` on the terminal raise. The router must scrub known secret patterns from it. The primary fails with a benign rate-limit; the fallback deployment fails with an exception whose text contains the secret.""" - litellm.expose_router_debug_in_errors = True + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) inner_secret = "sk-INNERFALLBACKEXCEPTIONSECRET1234" router = Router( model_list=[ diff --git a/tests/test_litellm/test_router_google_genai.py b/tests/test_litellm/test_router_google_genai.py index 81dd7bbdc40..8a90173bb7f 100644 --- a/tests/test_litellm/test_router_google_genai.py +++ b/tests/test_litellm/test_router_google_genai.py @@ -3,15 +3,10 @@ Test to verify the new Google GenAI router methods """ import asyncio -import os -import sys from unittest.mock import AsyncMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.types.utils import ModelResponse diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 4fdb5faf305..b580b03574e 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -8,13 +8,13 @@ should still use the built-in pricing. """ import copy +import logging import os -import sys +import re from unittest.mock import patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path import litellm from litellm import Router @@ -1892,7 +1892,7 @@ def test_a_reservation_without_a_declared_id_is_refused(): duplicate is permanent.""" anonymous = {k: v for k, v in _PTU_MODEL_INFO.items() if k != "id"} - with pytest.raises(ValueError, match="model_info.id is required"): + with pytest.raises(ValueError, match=re.escape("model_info.id is required")): _ptu_router(model_info=anonymous) @@ -1976,7 +1976,7 @@ def test_a_bare_yaml_date_bound_does_not_escape_the_id_rule(): windowed = {k: v for k, v in _PTU_MODEL_INFO.items() if k != "id"} - with pytest.raises(ValueError, match="model_info.id is required"): + with pytest.raises(ValueError, match=re.escape("model_info.id is required")): _ptu_router(model_info={**windowed, "ptu_effective_to": _dt.date(2027, 1, 1)}) @@ -2006,3 +2006,127 @@ def test_a_falsy_id_is_still_scanned_for_collisions(): }, ] ) + + +# --- a reservation declared while the feature is off says so ------------------------ + + +def _ptu_warnings(caplog): + return tuple( + record.getMessage() + for record in caplog.records + if record.name == "LiteLLM Router" and record.levelno == logging.WARNING and "PTU" in record.getMessage() + ) + + +def test_a_reservation_declared_while_the_feature_is_off_is_warned_about(caplog): + """The deployment serves and bills per token, so without this the operator believes they + reserved capacity and sees no signal anywhere that nothing accrues.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(ptu_enabled=False) + + warnings = _ptu_warnings(caplog) + + assert len(warnings) == 1 + assert "gpt-4o-ptu" in warnings[0] + assert "LITELLM_ENABLE_PTU_COST_ATTRIBUTION" in warnings[0] + + +def test_a_reservation_is_not_warned_about_while_the_feature_is_on(caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router() + + assert _ptu_warnings(caplog) == () + + +def test_a_deployment_carrying_no_ptu_field_is_not_warned_about(caplog): + """Most of every config.yaml, so warning here would fire on proxies that never asked.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(model_info={"team_id": "team-alpha"}, ptu_enabled=False) + + assert _ptu_warnings(caplog) == () + + +def test_a_half_written_reservation_is_warned_about(caplog): + """A count with no rate is not a chargeable reservation, but the operator still meant to + declare one, so what they wrote is what decides whether they hear about it.""" + half_written = {k: v for k, v in _PTU_MODEL_INFO.items() if k != "cost_per_ptu_per_hour"} + + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(model_info=half_written, ptu_enabled=False) + + assert len(_ptu_warnings(caplog)) == 1 + + +@pytest.mark.parametrize( + "typo", + [ + {"ptu_count": 0}, + {"ptu_count": 0, "cost_per_ptu_per_hour": 0, "ptu_effective_from": None}, + ], + ids=["count out of range", "every value still a zero placeholder"], +) +def test_a_reservation_dropped_by_a_typo_is_warned_about(caplog, typo): + """An out-of-range value fails ModelInfo before the flag is ever consulted, so the + deployment stops serving on a proxy that never enabled PTU. The warning is what tells the + operator which feature the entry that vanished belonged to. + + Built the way proxy_server builds it, since dropping rather than raising is what + ``ignore_invalid_deployments`` does and config.yaml is loaded with it on. + """ + with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": ""}, clear=False): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + router = Router( + ignore_invalid_deployments=True, + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://e.azure.com"}, + "model_info": {**_PTU_MODEL_INFO, **typo}, + } + ], + ) + + assert router.model_list == [] + assert len(_ptu_warnings(caplog)) == 1 + + +def test_a_db_backed_reservation_is_not_warned_about(caplog): + """/model/new already answered the caller with a 400, so repeating it on every reload + would report the operator's own rejected write back to them as a standing problem.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + _ptu_router(model_info={**_PTU_MODEL_INFO, "db_model": True}, ptu_enabled=False) + + assert _ptu_warnings(caplog) == () + + +def test_every_declaring_deployment_is_named(caplog): + """One line naming all of them, so a reload does not bury the config in repeats.""" + with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": ""}, clear=False): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + Router( + model_list=[ + { + "model_name": "azure-ptu-east", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://e.azure.com"}, + "model_info": dict(_PTU_MODEL_INFO), + }, + { + "model_name": "azure-ptu-west", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://w.azure.com"}, + "model_info": {**_PTU_MODEL_INFO, "id": "ptu-alpha-westus"}, + }, + { + "model_name": "plain-gpt-4o", + "litellm_params": {"model": "azure/gpt-4o", "api_key": "k", "api_base": "https://p.azure.com"}, + "model_info": {"id": "plain"}, + }, + ] + ) + + warnings = _ptu_warnings(caplog) + + assert len(warnings) == 1 + assert "azure-ptu-east" in warnings[0] + assert "azure-ptu-west" in warnings[0] + assert "plain-gpt-4o" not in warnings[0] diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 3fc6bc71b84..1b98b8c1ae8 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -20,15 +20,12 @@ This file pins both halves of the fix. """ import json -import os -import sys from dataclasses import dataclass from unittest.mock import AsyncMock, MagicMock import pytest from pydantic import ValidationError -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.types.router import RetryPolicy, UpdateRouterConfig diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 0188d87dfdb..7d50694c805 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -147,6 +147,71 @@ def test_filter_redacts_extra_fields(): assert record.region == "us-east-1" +def test_filter_preserves_uvicorn_color_message_args(): + """Regression test: uvicorn's startup banner logs a plain message plus a + colorized `extra={"color_message": ...}` copy of the same "%s://%s:%d" template, + both meant to be filled in from record.args. uvicorn's own ColourizedFormatter + re-substitutes color_message against record.args when writing to a TTY, instead + of using the already-formatted record.msg. + + Before this fix, the filter cleared record.args after substituting only + record.msg, so color_message was rendered with args=None and the raw + "%s://%s:%d" placeholders were printed instead of the real host/port. + """ + from uvicorn.logging import DefaultFormatter + + addr_format = "%s://%s:%d" + plain_message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" + color_message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" + + logger = logging.getLogger("uvicorn.error") + saved_handlers, saved_level = logger.handlers[:], logger.level + buf = StringIO() + handler = logging.StreamHandler(buf) + formatter = DefaultFormatter("%(levelprefix)s %(message)s") + formatter.use_colors = True + handler.setFormatter(formatter) + logger.handlers = [handler] + logger.setLevel(logging.INFO) + try: + logger.info( + plain_message, + "http", + "0.0.0.0", + 4000, + extra={"color_message": color_message}, + ) + output = buf.getvalue() + finally: + logger.handlers = saved_handlers + logger.setLevel(saved_level) + + assert "%s" not in output and "%d" not in output, f"unsubstituted placeholders leaked: {output!r}" + assert "http://0.0.0.0:4000" in output + + +def test_filter_redacts_secrets_substituted_into_color_message(): + """The color_message substitution runs before the extra-field redaction + loop, so a secret arriving through record.args lands in color_message and + must still be scrubbed. Substituting after that loop would ship the secret + to any colorized handler.""" + record = logging.LogRecord( + name="uvicorn.error", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="connecting with %s", + args=(SECRET,), + exc_info=None, + ) + record.color_message = "connecting with %s" + + _secret_filter.filter(record) + + assert SECRET not in record.color_message + assert "REDACTED" in record.color_message + + def test_disable_redaction_passes_secrets_through(): """When LITELLM_DISABLE_REDACT_SECRETS=true, secrets pass through.""" with patch("litellm._logging._ENABLE_SECRET_REDACTION", False): diff --git a/tests/test_litellm/test_shared_session_integration.py b/tests/test_litellm/test_shared_session_integration.py index 4ce704f88cb..fab356db3b6 100644 --- a/tests/test_litellm/test_shared_session_integration.py +++ b/tests/test_litellm/test_shared_session_integration.py @@ -2,14 +2,11 @@ Integration tests for shared session functionality in main.py """ -import os -import sys from unittest.mock import MagicMock, patch import pytest # Add the litellm directory to the path -sys.path.insert(0, os.path.abspath("../../..")) import litellm diff --git a/tests/test_litellm/test_streaming_connection_cleanup.py b/tests/test_litellm/test_streaming_connection_cleanup.py index 5a81a3ffb17..39fcee8d44d 100644 --- a/tests/test_litellm/test_streaming_connection_cleanup.py +++ b/tests/test_litellm/test_streaming_connection_cleanup.py @@ -3,15 +3,12 @@ Regression tests for streaming connection pool leak fix. """ import asyncio -import os -import sys from unittest.mock import MagicMock, patch import anyio import httpx import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.custom_httpx.aiohttp_transport import ( diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 43100b10aeb..d655eb96a02 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,16 +1,12 @@ import json import logging import os -import sys from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from jsonschema import validate -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm._logging import ( @@ -672,8 +668,8 @@ def test_all_model_configs(): ) == {"max_output_tokens": 10} -def test_anthropic_web_search_in_model_info(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_anthropic_web_search_in_model_info(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") supported_models = [ @@ -1193,11 +1189,11 @@ def test_max_tokens_consistency(): raise AssertionError(error_msg) -def test_get_model_info_gemini(): +def test_get_model_info_gemini(monkeypatch): """ Tests if ALL gemini models have 'tpm' and 'rpm' in the model info """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model_map = litellm.model_cost @@ -1252,8 +1248,8 @@ def test_get_model_info_bedrock_double_provider_prefix_resolves(local_model_cost assert info["key"] == "us.anthropic.claude-sonnet-4-6" -def test_openai_models_in_model_info(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" +def test_openai_models_in_model_info(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") model_map = litellm.model_cost @@ -1408,7 +1404,7 @@ for commitment in BEDROCK_COMMITMENTS: print("block_list", block_list) -def test_supports_computer_use_utility(): +def test_supports_computer_use_utility(monkeypatch): """ Tests the litellm.utils.supports_computer_use utility function. """ @@ -1420,7 +1416,7 @@ def test_supports_computer_use_utility(): original_env_var = os.getenv("LITELLM_LOCAL_MODEL_COST_MAP") original_model_cost = getattr(litellm, "model_cost", None) - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") # Load with local/backup try: @@ -1438,7 +1434,7 @@ def test_supports_computer_use_utility(): if original_env_var is None: del os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env_var + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", original_env_var) if original_model_cost is not None: litellm.model_cost = original_model_cost @@ -1446,13 +1442,13 @@ def test_supports_computer_use_utility(): delattr(litellm, "model_cost") -def test_get_model_info_shows_supports_computer_use(): +def test_get_model_info_shows_supports_computer_use(monkeypatch): """ Tests if 'supports_computer_use' is correctly retrieved by get_model_info. We'll use 'claude-4-sonnet-20250514' as it's configured in the backup JSON to have supports_computer_use: True. """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") # Ensure litellm.model_cost is loaded, relying on the backup mechanism if primary fails # as per previous debugging. litellm.model_cost = litellm.get_model_cost_map(url="") @@ -4391,11 +4387,13 @@ class TestVertexEmbeddingEncodingFormat: "vertex_ai/gemini-3-pro-image-preview", "vertex_ai/gemini-3.1-flash-image", "vertex_ai/gemini-3.1-flash-image-preview", + "vertex_ai/gemini-3.1-flash-lite-image", "gemini/gemini-2.5-flash-image", "gemini/gemini-3-pro-image", "gemini/gemini-3-pro-image-preview", "gemini/gemini-3.1-flash-image", "gemini/gemini-3.1-flash-image-preview", + "gemini/gemini-3.1-flash-lite-image", ], ) def test_gemini_image_models_do_not_support_reasoning( diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 117ca72c34f..fb167a8624e 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -2,14 +2,10 @@ import asyncio import io import json import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.cost_calculator import default_video_cost_calculator @@ -242,7 +238,6 @@ class TestVideoGeneration: def test_video_generation_cost_calculation(self): """Test video generation cost calculation.""" import json - import os # Try to load the local model cost map, skip if not found cost_map_path = "model_prices_and_context_window.json" diff --git a/tests/test_litellm/test_xai_responses_auto_routing.py b/tests/test_litellm/test_xai_responses_auto_routing.py index a4d72bb97d9..5b1944dcb8b 100644 --- a/tests/test_litellm/test_xai_responses_auto_routing.py +++ b/tests/test_litellm/test_xai_responses_auto_routing.py @@ -2,11 +2,8 @@ Test automatic routing to xAI Responses API when tools are present """ -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.abspath("../..")) import pytest import litellm diff --git a/tests/test_litellm/types/llms/test_types_llms_openai.py b/tests/test_litellm/types/llms/test_types_llms_openai.py index 569743269a5..e5e5c0183a0 100644 --- a/tests/test_litellm/types/llms/test_types_llms_openai.py +++ b/tests/test_litellm/types/llms/test_types_llms_openai.py @@ -1,12 +1,9 @@ import asyncio -import os -import sys from typing import Optional from unittest.mock import AsyncMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import json import litellm diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 672aa84cc73..c081b9e8e0d 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -1,10 +1,7 @@ -import os -import sys from typing import Final import pytest -sys.path.insert(0, os.path.abspath("../..")) from litellm.types.utils import HiddenParams, all_litellm_params diff --git a/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py b/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py index bfb084e7dd2..4044e3dcc0e 100644 --- a/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py +++ b/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py @@ -1,12 +1,7 @@ -import os -import sys from unittest.mock import Mock, patch import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig diff --git a/tests/test_litellm/vector_stores/test_vector_store_registry.py b/tests/test_litellm/vector_stores/test_vector_store_registry.py index 85ff8a1bcae..f19c3706845 100644 --- a/tests/test_litellm/vector_stores/test_vector_store_registry.py +++ b/tests/test_litellm/vector_stores/test_vector_store_registry.py @@ -1,6 +1,4 @@ import json -import os -import sys from unittest.mock import patch import httpx @@ -8,9 +6,6 @@ import pytest import respx from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path from datetime import datetime, timezone from unittest.mock import MagicMock diff --git a/tests/test_litellm/videos/test_main.py b/tests/test_litellm/videos/test_main.py index 38667e93eee..22e1e5c05eb 100644 --- a/tests/test_litellm/videos/test_main.py +++ b/tests/test_litellm/videos/test_main.py @@ -30,8 +30,6 @@ helper runs for real against genuinely-encoded ids, so the provider assertions reflect production. """ -import os -import sys from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Dict @@ -39,7 +37,6 @@ from unittest.mock import MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler diff --git a/tests/test_litellm/videos/test_utils.py b/tests/test_litellm/videos/test_utils.py index 09975829531..57fb549c23d 100644 --- a/tests/test_litellm/videos/test_utils.py +++ b/tests/test_litellm/videos/test_utils.py @@ -9,12 +9,9 @@ runs for real, so the "litellm-internal params get stripped" assertions reflect production. Every test asserts the exact resulting dict, never "ran without error". """ -import os -import sys from unittest.mock import MagicMock -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.videos.utils import VideoGenerationRequestUtils diff --git a/tests/test_new_vector_store_endpoints.py b/tests/test_new_vector_store_endpoints.py index 4748d8e9947..c44723937ac 100644 --- a/tests/test_new_vector_store_endpoints.py +++ b/tests/test_new_vector_store_endpoints.py @@ -4,13 +4,10 @@ Tests both basic functionality and complex scenarios including target_model_name """ import asyncio -import os -import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest -sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_ratelimit.py b/tests/test_ratelimit.py index 121dfbd99b7..7959f182a3a 100644 --- a/tests/test_ratelimit.py +++ b/tests/test_ratelimit.py @@ -4,14 +4,10 @@ import os import pytest import random from typing import Any -import sys from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../") -) # Adds the parent directory to the system path import litellm from pydantic import BaseModel diff --git a/tests/unified_google_tests/base_google_test.py b/tests/unified_google_tests/base_google_test.py index c4d8bb0d5aa..b7134962a0c 100644 --- a/tests/unified_google_tests/base_google_test.py +++ b/tests/unified_google_tests/base_google_test.py @@ -1,14 +1,10 @@ import asyncio import json -import sys import os import tempfile from typing import Any, AsyncIterator, Dict, List, Optional, Union import pytest -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.google_genai import ( diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index d2c6830c273..a4df8d03605 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -4,7 +4,6 @@ import asyncio import importlib import os import socket -import sys import threading import time from pathlib import Path @@ -16,9 +15,6 @@ from dotenv import load_dotenv load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm # noqa: E402,F401 from tests._vcr_conftest_common import ( # noqa: E402,F401 @@ -146,9 +142,6 @@ def setup_and_teardown(request): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path if "google_genai_proxy_url" not in request.fixturenames: diff --git a/tests/unified_google_tests/test_google_ai_studio.py b/tests/unified_google_tests/test_google_ai_studio.py index 3e40fa41089..6d4c3725080 100644 --- a/tests/unified_google_tests/test_google_ai_studio.py +++ b/tests/unified_google_tests/test_google_ai_studio.py @@ -1,11 +1,6 @@ from base_google_genai_proxy_sdk_test import BaseGoogleGenAIProxySDKTest from base_google_test import BaseGoogleGenAITest -import sys -import os -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import pytest import litellm import unittest.mock diff --git a/tests/unified_google_tests/test_vertex_anthropic.py b/tests/unified_google_tests/test_vertex_anthropic.py index 71dad3a5cf9..f11ee28aacb 100644 --- a/tests/unified_google_tests/test_vertex_anthropic.py +++ b/tests/unified_google_tests/test_vertex_anthropic.py @@ -1,15 +1,10 @@ import asyncio import json -import sys -import os from typing import Any, AsyncIterator, Dict, List, Optional, Union import pytest from unittest.mock import MagicMock, AsyncMock, patch import httpx -sys.path.insert( - 0, os.path.abspath("../../..") -) # Adds the parent directory to the system path import litellm from litellm.google_genai import agenerate_content, agenerate_content_stream diff --git a/tests/vector_store_tests/base_vector_store_test.py b/tests/vector_store_tests/base_vector_store_test.py index 4093ea7b43b..926fe98b6ec 100644 --- a/tests/vector_store_tests/base_vector_store_test.py +++ b/tests/vector_store_tests/base_vector_store_test.py @@ -1,17 +1,12 @@ import httpx import json import pytest -import sys from typing import Any, Dict, List from unittest.mock import MagicMock, Mock, patch -import os from litellm._uuid import uuid import time import base64 -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm from abc import ABC, abstractmethod from litellm.integrations.custom_logger import CustomLogger diff --git a/tests/vector_store_tests/conftest.py b/tests/vector_store_tests/conftest.py index 41da685895b..8c1e70b14bc 100644 --- a/tests/vector_store_tests/conftest.py +++ b/tests/vector_store_tests/conftest.py @@ -2,13 +2,9 @@ import importlib import os -import sys import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path import litellm @@ -18,9 +14,6 @@ def setup_and_teardown(): This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. """ curr_dir = os.getcwd() # Get the current working directory - sys.path.insert( - 0, os.path.abspath("../..") - ) # Adds the project directory to the system path from litellm import Router @@ -28,8 +21,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/vector_store_tests/rag/base_rag_tests.py b/tests/vector_store_tests/rag/base_rag_tests.py index 2c5a2540a7e..caeb7651085 100644 --- a/tests/vector_store_tests/rag/base_rag_tests.py +++ b/tests/vector_store_tests/rag/base_rag_tests.py @@ -4,15 +4,12 @@ Base RAG test class that enforces common tests across all providers. Providers should inherit from BaseRAGTest and implement the abstract methods. """ -import os -import sys import uuid from abc import ABC, abstractmethod from typing import Any, Dict, Optional import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.types.rag import ( diff --git a/tests/vector_store_tests/rag/test_rag_bedrock.py b/tests/vector_store_tests/rag/test_rag_bedrock.py index 7e788ed32f1..90cf4a3a44e 100644 --- a/tests/vector_store_tests/rag/test_rag_bedrock.py +++ b/tests/vector_store_tests/rag/test_rag_bedrock.py @@ -11,12 +11,10 @@ Optional (for using existing KB instead of auto-creating): """ import os -import sys from typing import Any, Dict, Optional import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.types.rag import RAGIngestOptions, BedrockVectorStoreOptions diff --git a/tests/vector_store_tests/rag/test_rag_openai.py b/tests/vector_store_tests/rag/test_rag_openai.py index d948e86fcf4..368e4e471b1 100644 --- a/tests/vector_store_tests/rag/test_rag_openai.py +++ b/tests/vector_store_tests/rag/test_rag_openai.py @@ -2,13 +2,10 @@ OpenAI RAG ingestion tests. """ -import os -import sys from typing import Any, Dict, Optional import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.types.rag import RAGIngestOptions, OpenAIVectorStoreOptions diff --git a/tests/vector_store_tests/rag/test_rag_s3_vectors.py b/tests/vector_store_tests/rag/test_rag_s3_vectors.py index cd8a362a7bf..d950bc0f644 100644 --- a/tests/vector_store_tests/rag/test_rag_s3_vectors.py +++ b/tests/vector_store_tests/rag/test_rag_s3_vectors.py @@ -11,12 +11,10 @@ Optional: """ import os -import sys from typing import Any, Dict, Optional import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.types.rag import RAGIngestOptions diff --git a/tests/vector_store_tests/rag/test_rag_vertex_ai.py b/tests/vector_store_tests/rag/test_rag_vertex_ai.py index c99840bb0fe..ae5891ed3ff 100644 --- a/tests/vector_store_tests/rag/test_rag_vertex_ai.py +++ b/tests/vector_store_tests/rag/test_rag_vertex_ai.py @@ -17,12 +17,10 @@ Environment variables: """ import os -import sys from typing import Any, Dict, Optional import pytest -sys.path.insert(0, os.path.abspath("../../..")) import litellm from litellm.types.rag import RAGIngestOptions diff --git a/tests/vector_store_tests/test_gemini_vector_store.py b/tests/vector_store_tests/test_gemini_vector_store.py index 8e30c94de51..2aa2c1741a8 100644 --- a/tests/vector_store_tests/test_gemini_vector_store.py +++ b/tests/vector_store_tests/test_gemini_vector_store.py @@ -3,9 +3,7 @@ Minimal Gemini File Search vector store tests. """ import os -import sys -sys.path.insert(0, os.path.abspath("../..")) from base_vector_store_test import BaseVectorStoreTest diff --git a/tests/vector_store_tests/test_ragflow_vector_store.py b/tests/vector_store_tests/test_ragflow_vector_store.py index 46751b64cce..0af821da98a 100644 --- a/tests/vector_store_tests/test_ragflow_vector_store.py +++ b/tests/vector_store_tests/test_ragflow_vector_store.py @@ -3,13 +3,11 @@ Test RAGFlow Vector Store helper functions and transformation. """ import os -import sys import json import pytest from unittest.mock import Mock, patch, MagicMock import httpx -sys.path.insert(0, os.path.abspath("../..")) import litellm from tests.vector_store_tests.base_vector_store_test import BaseVectorStoreTest diff --git a/tests/windows_tests/test_litellm_on_windows.py b/tests/windows_tests/test_litellm_on_windows.py index 8810cc78929..0a6058d6784 100644 --- a/tests/windows_tests/test_litellm_on_windows.py +++ b/tests/windows_tests/test_litellm_on_windows.py @@ -1,16 +1,11 @@ import asyncio -import os import subprocess -import sys import time import traceback import platform import pytest -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path def test_using_litellm_on_windows(): diff --git a/ui/litellm-dashboard/public/assets/logos/scx_ai.svg b/ui/litellm-dashboard/public/assets/logos/scx_ai.svg new file mode 100644 index 00000000000..545176a945b --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/scx_ai.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index f2dbd866c2d..dfc737ddd45 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -62,6 +62,17 @@ describe("provider_info_helpers", () => { expect(result.logo).toBe(providerLogoMap[Providers.Groq]); }); + it("should map scx-ai slug and SCX_AI enum key to the SCX.ai display name and logo", () => { + const fromSlug = getProviderLogoAndName("scx-ai"); + expect(fromSlug.displayName).toBe(Providers.SCX_AI); + expect(fromSlug.logo).toBe(providerLogoMap[Providers.SCX_AI]); + expect(fromSlug.logo).toBeTruthy(); + + const fromEnumKey = getProviderLogoAndName("SCX_AI"); + expect(fromEnumKey.displayName).toBe(Providers.SCX_AI); + expect(fromEnumKey.logo).toBe(providerLogoMap[Providers.SCX_AI]); + }); + it("should map bedrock_mantle slug to Bedrock Mantle display name and logo", () => { const result = getProviderLogoAndName("bedrock_mantle"); expect(result.displayName).toBe(Providers.BedrockMantle); @@ -180,6 +191,10 @@ describe("provider_info_helpers", () => { expect(getPlaceholder(Providers.Vertex_AI)).toBe("gemini-pro"); }); + it("should return an scx-ai model placeholder for SCX_AI provider", () => { + expect(getPlaceholder(Providers.SCX_AI)).toBe("scx-ai/GLM-5.2"); + }); + it("should return claude-3-opus placeholder for Anthropic provider", () => { expect(getPlaceholder(Providers.Anthropic)).toBe("claude-3-opus"); }); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 519438622f3..c97bb8ede3d 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -50,6 +50,7 @@ import replicateLogo from "../../public/assets/logos/replicate.svg"; import runwayLogo from "../../public/assets/logos/runway.png"; import sambanovaLogo from "../../public/assets/logos/sambanova.svg"; import sapLogo from "../../public/assets/logos/sap.png"; +import scxAiLogo from "../../public/assets/logos/scx_ai.svg"; import snowflakeLogo from "../../public/assets/logos/snowflake.svg"; import sonioxLogo from "../../public/assets/logos/soniox.svg"; import togetheraiLogo from "../../public/assets/logos/togetherai.svg"; @@ -153,6 +154,7 @@ export enum Providers { SAGEMAKER_LEGACY = "Sagemaker", Sambanova = "Sambanova", SAP = "SAP Generative AI Hub", + SCX_AI = "SCX.ai", Snowflake = "Snowflake", Soniox = "Soniox", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", @@ -264,6 +266,7 @@ export const provider_map: Record = { SageMaker: "sagemaker_chat", Sambanova: "sambanova", SAP: "sap", + SCX_AI: "scx-ai", Snowflake: "snowflake", Soniox: "soniox", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", @@ -356,6 +359,7 @@ export const providerLogoMap: Partial> = { [Providers.SAGEMAKER_LEGACY]: bedrockLogo.src, [Providers.Sambanova]: sambanovaLogo.src, [Providers.SAP]: sapLogo.src, + [Providers.SCX_AI]: scxAiLogo.src, [Providers.Snowflake]: snowflakeLogo.src, [Providers.Soniox]: sonioxLogo.src, [Providers.TEXT_COMPLETION_CODESTRAL]: mistralLogo.src, @@ -421,6 +425,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Oracle]: "oci/xai.grok-4", [Providers.RunwayML]: "runwayml/gen4_turbo", [Providers.SageMaker]: "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b", + [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Snowflake]: "snowflake/mistral-7b", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", diff --git a/uv.lock b/uv.lock index 6b18be68c92..628483c0117 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-17T21:26:36.028845Z" +exclude-newer = "2026-08-19T15:53:37.294198Z" exclude-newer-span = "P3D" [manifest] @@ -4661,12 +4661,12 @@ proxy-dev = [ [[package]] name = "litellm-enterprise" -version = "0.1.58" +version = "0.1.59" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.88" +version = "0.4.89" source = { editable = "litellm-proxy-extras" } [[package]]