From 38d0333fcc6e623339601f9b8b34da86821fbf12 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:39:54 +0000 Subject: [PATCH 01/96] fix(ui): drop the decision models sublabel from the Add Model provider dropdown (#45755) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../panels/AddModelPanel.integration.test.tsx | 8 ------ .../src/components/add_model/AddModelForm.tsx | 5 ++-- .../src/lib/decisionModels.test.ts | 26 +------------------ .../src/lib/decisionModels.ts | 14 +--------- 4 files changed, 4 insertions(+), 49 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 9f9f9668760..05560c9e7cd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -563,14 +563,6 @@ describe("AddModelPanel decision models", () => { const decisionNotice = () => screen.queryByRole("note", { name: "Decision model notice" }); - it("finds a provider by the name of one of its decision models", async () => { - const { user } = await setup(); - await user.type(screen.getByRole("combobox", { name: /provider/i }), "jev"); - - expect(await screen.findByRole("option", { name: /TypeSafe/ })).toBeInTheDocument(); - expect(screen.queryByRole("option", { name: /Anthropic/ })).not.toBeInTheDocument(); - }); - it("points a decision-only provider at the decision routes, the docs, and the playground", async () => { const { user } = await setup(); await user.type(screen.getByRole("combobox", { name: /provider/i }), "TypeSafe"); diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index e8b7ca62c3d..2a93b642d6e 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -39,7 +39,7 @@ import ProviderSpecificFields from "./provider_specific_fields"; import { authTypesFor } from "./provider_auth_types"; import { TEST_MODES } from "./add_model_modes"; import DecisionModelNote from "./DecisionModelNote"; -import { decisionModelsSublabel, type DecisionCatalog } from "@/lib/decisionModels"; +import { type DecisionCatalog } from "@/lib/decisionModels"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { credentialsKeys } from "@/app/(dashboard)/hooks/credentials/useCredentials"; import { extractProxyErrorMessage } from "@/lib/http/client"; @@ -173,10 +173,9 @@ const AddModelForm: React.FC = ({ sortedProviderMetadata.map((providerInfo) => ({ label: providerInfo.provider_display_name, value: providerInfo.provider, - sublabel: decisionModelsSublabel(decisionCatalog, providerInfo.litellm_provider), icon: , })), - [sortedProviderMetadata, decisionCatalog], + [sortedProviderMetadata], ); const pickedLitellmProvider = sortedProviderMetadata.find( diff --git a/ui/litellm-dashboard/src/lib/decisionModels.test.ts b/ui/litellm-dashboard/src/lib/decisionModels.test.ts index c0f3842993c..0876d88b261 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.test.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.test.ts @@ -1,11 +1,5 @@ import { describe, expect, it } from "vitest"; -import { - EMPTY_DECISION_CATALOG, - buildDecisionCatalog, - decisionModelsSublabel, - isDecisionMode, - isDecisionSelection, -} from "./decisionModels"; +import { EMPTY_DECISION_CATALOG, buildDecisionCatalog, isDecisionMode, isDecisionSelection } from "./decisionModels"; const COST_MAP = { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, @@ -46,12 +40,6 @@ describe("buildDecisionCatalog", () => { ); }); - it("strips the provider prefix from the names shown per provider", () => { - expect(catalog.providers.get("typesafe")?.names).toEqual(["jev-latest", "jev-preview"]); - expect(catalog.providers.get("perplexity")?.names).toEqual(["pplx-decider-v1-27b"]); - expect(catalog.providers.get("openai")?.names).toEqual(["gpt-6-luna"]); - }); - it("marks a provider decision-only when every one of its models is a decision model", () => { expect(catalog.providers.get("typesafe")?.decisionOnly).toBe(true); expect(catalog.providers.get("somevendor")?.decisionOnly).toBe(true); @@ -70,18 +58,6 @@ describe("buildDecisionCatalog", () => { }); }); -describe("decisionModelsSublabel", () => { - const catalog = buildDecisionCatalog(COST_MAP); - - it("lists the provider's decision models", () => { - expect(decisionModelsSublabel(catalog, "typesafe")).toBe("Decision models: jev-latest, jev-preview"); - }); - - it("is undefined for a provider with no decision models", () => { - expect(decisionModelsSublabel(catalog, "anthropic")).toBeUndefined(); - }); -}); - describe("isDecisionSelection", () => { const catalog = buildDecisionCatalog(COST_MAP); diff --git a/ui/litellm-dashboard/src/lib/decisionModels.ts b/ui/litellm-dashboard/src/lib/decisionModels.ts index 635d005b2df..51de8d62ace 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.ts @@ -15,7 +15,6 @@ const costMapEntrySchema = z.object({ type CostMapEntry = z.infer; interface ProviderDecisionModels { - readonly names: readonly string[]; readonly decisionOnly: boolean; } @@ -31,9 +30,6 @@ export const isDecisionMode = (mode: string | null | undefined): boolean => mode const isDecisionEntry = (entry: CostMapEntry): boolean => isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint)); -const withoutProviderPrefix = (key: string, provider: string): string => - key.startsWith(`${provider}/`) ? key.slice(provider.length + 1) : key; - const parsedEntries = (costMap: unknown): ReadonlyArray => { const record = z.record(z.string(), z.unknown()).safeParse(costMap); if (!record.success) { @@ -52,20 +48,12 @@ export function buildDecisionCatalog(costMap: unknown): DecisionCatalog { const providers = new Map( [...decisionProviders].map((provider) => { const providerEntries = entries.filter(([, entry]) => entry.litellm_provider === provider); - const names = decisionEntries - .filter(([, entry]) => entry.litellm_provider === provider) - .map(([key]) => withoutProviderPrefix(key, provider)); - return [provider, { names, decisionOnly: providerEntries.every(([, entry]) => isDecisionEntry(entry)) }] as const; + return [provider, { decisionOnly: providerEntries.every(([, entry]) => isDecisionEntry(entry)) }] as const; }), ); return { models: new Set(decisionEntries.map(([key]) => key)), providers }; } -export const decisionModelsSublabel = (catalog: DecisionCatalog, litellmProvider: string): string | undefined => { - const names = catalog.providers.get(litellmProvider)?.names ?? []; - return names.length > 0 ? `Decision models: ${names.join(", ")}` : undefined; -}; - export const isDecisionSelection = ( catalog: DecisionCatalog, litellmProvider: string | undefined, From 9b53ee878f8177a8631dcc8fce3fd5e4f8d558a7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 10 Oct 2026 11:15:19 -0700 Subject: [PATCH 02/96] perf(ci): isolate and shard integration suites (#44146) * perf(ci): distribute unit cases and reuse security sweep connections * revert(ci): reject first speed trial after new failures * perf(ci): avoid repeated test discovery and parsing * perf(ci): isolate parallel security workers * revert(ci): restore measured security execution isolation * perf(tests): reuse source scans and deduplicate cache cleanup * perf(ci): restore parallel security with safe log batching * test(ci): isolate fixture state and schema comparisons * test(ci): make schema and scheduler fixtures deterministic * test(ci): drain queued logging before fixture teardown * test(ci): preserve logging delivery without idle worker restarts * test(ci): keep guardrail sinks available during parallel bursts * test(ci): isolate full proxy lifecycle from gateway imports * test(ci): preserve first failure responses and sweep evidence * test(ci): restore documentation URL settings between cases * test(ci): isolate open MCP tools in private gateways * test(ci): group private MCP cases and handle exited workers * test(ci): retain proxy exits and MCP failure details * test(ci): repair merge interactions with main Key the schema shard's full clone on artifact-name, since main removed the unit-flag input. Give each owned Redis its own log file again. Hand the realtime transcription tests main added a logging object whose dispatch handlers are awaitable, so the per-test logging drain no longer waits on a MagicMock task forever * perf(ci): shard owned integration cases and close lifecycle leaks * fix(ci): qualify shard execution and harden interrupted cleanup * fix(ci): bound worker supervision and own fault traffic * perf(ci): reuse unparametrized fixtures across bounded case groups * fix(ci): preserve boot fault allowances and complete exit records * test(ci): tolerate startup retries and retain readiness details * perf(ci): balance integration shards by recorded duration * fix(ci): load migration database helpers only for workers --- .circleci/config.yml | 172 +++++++- .circleci/scripts/run_integration.sh | 14 +- .github/scripts/assert_ci_coverage.py | 8 +- .github/workflows/test-unit.yml | 8 +- .../generic_api/generic_api_callback.py | 38 +- scripts/check_schema_compatibility.py | 77 ++++ tests/integration/AGENTS.md | 2 +- tests/integration/README.md | 16 +- tests/integration/_support/async_server.py | 82 ++++ tests/integration/_support/database_relay.py | 68 +-- tests/integration/_support/ordering.py | 178 ++++++++ tests/integration/_support/process.py | 139 +++++-- tests/integration/_support/proxy.py | 4 +- tests/integration/_support/redis_process.py | 2 +- tests/integration/_support/runtime.py | 8 + tests/integration/_support/sharding.py | 113 +++++ tests/integration/_support/workers.py | 27 ++ .../test_multi_worker_slow_import_boot.py | 11 +- tests/integration/conftest.py | 93 ++++- .../database/test_harness_lifecycle.py | 389 ++++++++++++++++++ ...uto_router_databricks_classifier_writes.py | 42 +- .../integration/mcp/test_mcp_llm_endpoints.py | 53 +++ tests/integration/mcp/test_mcp_toolsets.py | 28 +- .../test_guardrail_timeout_all_providers.py | 48 ++- .../test_passthrough_upstream_error_chaos.py | 50 ++- .../test_straiker_v3_platform.py | 25 ++ .../providers/_gemini_tts_voice_support.py | 1 + .../test_bedrock_kimi_k3_cache_point_wire.py | 60 +-- ...che_control_tool_call_marks_owned_proxy.py | 21 +- .../test_device_code_login_guard_boot.py | 24 +- ...st_gemini_tts_voice_mapping_owned_proxy.py | 54 ++- tests/integration/run.py | 14 +- .../sdk/test_generic_api_callback.py | 109 +++++ tests/integration/security/_sinks.py | 4 + tests/integration/security/_sweeps.py | 18 +- tests/integration/security/conftest.py | 26 ++ .../security/test_sweep_sensitivity.py | 137 +++++- .../test_vertex_image_regional_uplift.py | 118 ++++-- tests/mcp_tests/test_proxy_mcp_e2e.py | 163 ++++++-- tests/proxy_migration_tests/conftest.py | 31 ++ .../test_request_log_indexes.py | 16 +- tests/unit/conftest.py | 30 +- .../unit/integration_support/test_ordering.py | 180 ++++++++ .../unit/integration_support/test_process.py | 53 +++ .../unit/integration_support/test_sharding.py | 117 ++++++ .../generic_api/test_generic_api_callback.py | 114 ++++- .../litellm_core_utils/test_aws_partition.py | 55 ++- .../test_realtime_streaming.py | 140 ++++--- .../test_vertex_ai_realtime_transformation.py | 8 +- tests/unit/proxy/conftest.py | 2 + .../health_endpoints/test_health_endpoints.py | 25 +- .../proxy/middleware/test_gzip_middleware.py | 8 + tests/unit/proxy/test_db_schema_changes.py | 139 ++----- tests/unit/proxy/test_proxy_utils.py | 65 +-- tests/unit/realtime_api/test_main.py | 16 +- tests/unit/responses/mcp/conftest.py | 8 + tests/unit/test_assert_ci_coverage.py | 47 +++ tests/unit/test_conftest_isolation.py | 133 ++++++ 58 files changed, 3077 insertions(+), 554 deletions(-) create mode 100644 scripts/check_schema_compatibility.py create mode 100644 tests/integration/_support/async_server.py create mode 100644 tests/integration/_support/ordering.py create mode 100644 tests/integration/_support/runtime.py create mode 100644 tests/integration/_support/sharding.py create mode 100644 tests/integration/_support/workers.py create mode 100644 tests/integration/database/test_harness_lifecycle.py create mode 100644 tests/integration/sdk/test_generic_api_callback.py create mode 100644 tests/integration/security/conftest.py create mode 100644 tests/proxy_migration_tests/conftest.py create mode 100644 tests/unit/integration_support/test_ordering.py create mode 100644 tests/unit/integration_support/test_sharding.py create mode 100644 tests/unit/responses/mcp/conftest.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 786a0720991..b22c2b4b7a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,7 +243,7 @@ commands: mkdir -p "$HOME/.local/bin" mv /tmp/codecov "$HOME/.local/bin/codecov" start_openai_record_replay_proxy: - description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." + description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." steps: - run: name: Start record/replay proxy @@ -2747,6 +2747,33 @@ jobs: - store_test_results: path: test-results/provider-replay-harness + integration_timing_snapshot: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: medium + working_directory: ~/project + steps: + - checkout + - run: + name: Prepare optional integration timing snapshots + command: mkdir -p /tmp/integration-timings/{extensions,providers,mcp} + - restore_cache: + keys: + - integration-timings-v1-extensions-{{ .Branch }}-{{ .Revision }} + - integration-timings-v1-extensions-{{ .Branch }}- + - restore_cache: + keys: + - integration-timings-v1-providers-{{ .Branch }}-{{ .Revision }} + - integration-timings-v1-providers-{{ .Branch }}- + - restore_cache: + keys: + - integration-timings-v1-mcp-{{ .Branch }}-{{ .Revision }} + - integration-timings-v1-mcp-{{ .Branch }}- + - persist_to_workspace: + root: /tmp + paths: + - integration-timings + integration_contracts: parameters: suite: @@ -2769,6 +2796,15 @@ jobs: parallelism: << parameters.parallelism >> steps: - setup_litellm_test_deps + - when: + condition: + or: + - equal: [extensions, << parameters.suite >>] + - equal: [providers, << parameters.suite >>] + - equal: [mcp, << parameters.suite >>] + steps: + - attach_workspace: + at: /tmp - when: condition: equal: [browser, << parameters.suite >>] @@ -2812,11 +2848,70 @@ jobs: docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true docker rm -f postgres-db redis-cache test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" + - when: + condition: + or: + - equal: [extensions, << parameters.suite >>] + - equal: [providers, << parameters.suite >>] + - equal: [mcp, << parameters.suite >>] + steps: + - run: + name: Prepare this shard's execution record + command: | + mkdir -p "/tmp/integration-shards/<< parameters.suite >>/$CIRCLE_NODE_INDEX" + cp test-results/integration-<< parameters.suite >>/execution.json "/tmp/integration-shards/<< parameters.suite >>/$CIRCLE_NODE_INDEX/execution.json" + if [ "<< parameters.suite >>" = mcp ]; then + mkdir -p "/tmp/integration-shards/mcp/$CIRCLE_NODE_INDEX/coverage" + cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" + fi + - persist_to_workspace: + root: /tmp/integration-shards + paths: + - << parameters.suite >> - store_test_results: path: test-results - store_artifacts: path: test-results + integration_shard_coverage: + parameters: + suite: + type: enum + enum: [extensions, providers, mcp] + shards: + type: integer + machine: + image: ubuntu-2204:2024.04.1 + resource_class: medium + working_directory: ~/project + steps: + - setup_litellm_test_deps + - attach_workspace: + at: /tmp/integration-shards + - run: + name: Verify every << parameters.suite >> case ran exactly once + command: | + .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >> \ + --shards << parameters.shards >> --timings-output /tmp/integration-timings/<< parameters.suite >>/timings.json + - save_cache: + key: integration-timings-v1-<< parameters.suite >>-{{ .Branch }}-{{ .Revision }} + paths: + - /tmp/integration-timings/<< parameters.suite >> + - when: + condition: + equal: [mcp, << parameters.suite >>] + steps: + - run: + name: Combine coverage from every MCP shard + command: | + mkdir -p test-results/integration-mcp/coverage + export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data" + .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp/*/coverage + .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt + .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html + - store_artifacts: + path: test-results/integration-mcp/coverage + routing_parity: parameters: suite: @@ -2898,6 +2993,9 @@ jobs: default: "" timeout_minutes: type: integer + workers: + type: integer + default: 1 machine: image: ubuntu-2204:2024.04.1 resource_class: large @@ -2925,12 +3023,16 @@ jobs: command: | mkdir -p test-results coverage_args=() + worker_args=() + if [ "<< parameters.workers >>" -gt 1 ]; then + worker_args=(-n << parameters.workers >> --dist=worksteal) + fi if [ -n "<< parameters.coverage_flag >>" ]; then coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml) fi timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \ << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \ - --junitxml=test-results/junit.xml "${coverage_args[@]}" + --junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}" - when: condition: << parameters.coverage_flag >> steps: @@ -3103,25 +3205,76 @@ workflows: - not: equal: ["", << pipeline.parameters.routing_parity_base >>] jobs: + - integration_timing_snapshot: + name: integration-timing-snapshot - integration_contracts: name: integration-<< matrix.suite >> matrix: parameters: - suite: [management, accounting, database, providers, sdk, cost, security, browser] + suite: [sdk, cost, browser] + - integration_contracts: + name: integration-security + suite: security + resource_class: xlarge + - integration_contracts: + name: integration-management + suite: management + parallelism: 8 + - integration_contracts: + name: integration-accounting + suite: accounting + parallelism: 8 + - integration_contracts: + name: integration-providers + suite: providers + parallelism: 16 + requires: + - integration-timing-snapshot + - integration_shard_coverage: + name: integration-providers-coverage + suite: providers + shards: 16 + requires: + - integration-providers + - integration_contracts: + name: integration-database + suite: database + parallelism: 2 - integration_contracts: name: integration-mcp suite: mcp resource_class: xlarge + parallelism: 2 + requires: + - integration-timing-snapshot + - integration_shard_coverage: + name: integration-mcp-coverage + suite: mcp + shards: 2 + requires: + - integration-mcp - integration_contracts: name: integration-extensions suite: extensions - parallelism: 4 + parallelism: 32 + requires: + - integration-timing-snapshot + - integration_shard_coverage: + name: integration-extensions-coverage + suite: extensions + shards: 32 + requires: + - integration-extensions - integration_contracts: - name: integration-<< matrix.suite >>-replica - matrix: - parameters: - suite: [management, database] - mode: [replica] + name: integration-management-replica + suite: management + mode: replica + parallelism: 8 + - integration_contracts: + name: integration-database-replica + suite: database + mode: replica + parallelism: 2 - postgres_suite: name: proxy-behavior test_path: tests/proxy_behavior @@ -3138,6 +3291,7 @@ workflows: test_path: tests/proxy_migration_tests seed: false timeout_minutes: 20 + workers: 2 - postgres_suite: name: roi-database test_path: tests/integration/database/test_roi_observed.py diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 3ac47c3e507..4bdef5eb23b 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -134,6 +134,9 @@ upstream_pid=$! if [ "$suite" = cost ]; then export INTEGRATION_WORKERS=8 fi +if [ "$suite" = security ]; then + export INTEGRATION_WORKERS=4 +fi if [ "$suite" = mcp ]; then export INTEGRATION_WORKERS=4 INTEGRATION_COVERAGE=1 fi @@ -214,7 +217,12 @@ if [ "$suite" = browser ]; then fi node_files=() -if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then +case_shards=1 +case_shard_index=0 +if [ "$suite" = extensions ] || [ "$suite" = providers ] || [ "$suite" = mcp ]; then + case_shards="${CIRCLE_NODE_TOTAL:-1}" + case_shard_index="${CIRCLE_NODE_INDEX:-0}" +elif [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then split="$(.venv/bin/python tests/integration/run.py "$suite" --list \ | circleci tests split --split-by=timings --timings-type=filename)" read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')" @@ -232,11 +240,13 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ LITELLM_LICENSE="${LITELLM_LICENSE:-}" \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ + INTEGRATION_CASE_SHARDS="$case_shards" INTEGRATION_CASE_SHARD_INDEX="$case_shard_index" \ LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \ INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \ INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \ - .venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}" + .venv/bin/python tests/integration/run.py "$suite" --results "$results" \ + --timings "/tmp/integration-timings/$suite/timings.json" "${node_files[@]}" if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then for covered_pid in "$proxy_pid" "$peer_pid"; do diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 04c801cf5cc..5c581c99d12 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -9,6 +9,8 @@ import sys import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass +from functools import lru_cache +from types import MappingProxyType from typing import Final import yaml @@ -316,6 +318,7 @@ def _keyword_terms( def _slices() -> tuple[Slice, ...]: if not CIRCLECI_CONFIG.exists(): return () + test_files: Final = frozenset(_test_files()) jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {}) return tuple( Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood) @@ -323,7 +326,7 @@ def _slices() -> tuple[Slice, ...]: for text in ("\n".join(_strings(body)),) if "pytest" in text for globs in (tuple(GLOB_CALL_RE.findall(text)),) - for named in (frozenset(TEST_TOKEN_RE.findall(text)) & frozenset(_test_files()),) + for named in (frozenset(TEST_TOKEN_RE.findall(text)) & test_files,) for required, excluded, understood in ( _keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2), ) @@ -363,6 +366,7 @@ def _workflow_named_tokens() -> frozenset[str]: def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]: + matchable_names: Final = lru_cache(maxsize=None)(_matchable_names) slices: Final = _slices() named_by_workflow: Final = _workflow_named_tokens() globbed: Final = tuple( @@ -376,7 +380,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]: for path in globbed if not allowlist.covers_test(path) and not any(_token_covers(token, path) for token in named_by_workflow) - and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices) + and not any(slice_.claims(path, matchable_names(path)) for slice_ in slices) ) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 42b418b3ee0..7f01b38ddc8 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -927,6 +927,7 @@ jobs: timeout-minutes: 3 with: persist-credentials: false + fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }} - name: Detect relevant changes id: changes @@ -959,11 +960,13 @@ jobs: with: artifact: ${{ needs.rust-bridge.outputs.artifact }} - - name: Install dependencies + - name: Install dependencies and check schema if: steps.changes.outputs.decision != 'skip' timeout-minutes: 8 env: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} + UNIT_SHARD: ${{ matrix.shard }} + SCHEMA_BASE_REF: origin/${{ github.event.repository.default_branch }} run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then @@ -975,6 +978,9 @@ jobs: uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" fi uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]' + if [ "$UNIT_SHARD" = "db-and-spend" ]; then + uv run --no-sync python scripts/check_schema_compatibility.py --base-ref "$SCHEMA_BASE_REF" + fi - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py index dfedc3a3cc9..47a472db420 100644 --- a/litellm/integrations/generic_api/generic_api_callback.py +++ b/litellm/integrations/generic_api/generic_api_callback.py @@ -10,7 +10,9 @@ import asyncio import json import os import re +import time import traceback +from collections.abc import Mapping from typing import Final, Literal import httpx @@ -21,6 +23,7 @@ from litellm._uuid import uuid from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, get_async_httpx_client, httpxSpecialProvider, ) @@ -104,6 +107,7 @@ class GenericAPILogger(CustomBatchLogger): max_retries: int = 0, retry_delay: float = 1.0, timeout: float | httpx.Timeout | None = None, + async_httpx_client: AsyncHTTPHandler | None = None, **kwargs, ): """ @@ -151,7 +155,11 @@ class GenericAPILogger(CustomBatchLogger): ######################################################### # Init httpx client ######################################################### - self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + self.async_httpx_client = ( + async_httpx_client + if async_httpx_client is not None + else get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + ) endpoint = endpoint or os.getenv("GENERIC_LOGGER_ENDPOINT") if endpoint is None: raise ValueError( @@ -191,7 +199,7 @@ class GenericAPILogger(CustomBatchLogger): self.flush_lock = asyncio.Lock() super().__init__(**kwargs, flush_lock=self.flush_lock) asyncio.create_task(self.periodic_flush()) - self.log_queue: list[dict | StandardLoggingPayload] = [] + self.log_queue: list[Mapping[str, object] | StandardLoggingPayload] = [] def _get_headers(self, headers: dict | None = None): """ @@ -343,7 +351,14 @@ class GenericAPILogger(CustomBatchLogger): except Exception as e: verbose_logger.exception("Generic API Logger Error - %s\n%s", e, traceback.format_exc()) - async def async_send_batch(self): + async def flush_queue(self) -> None: + await self.async_send_batch() + + async def async_send_batch(self) -> None: + async with self.flush_lock: + await self._send_batch() + + async def _send_batch(self) -> None: """ Sends the batch of messages to Generic API Endpoint @@ -352,18 +367,18 @@ class GenericAPILogger(CustomBatchLogger): - ndjson: Sends logs as newline-delimited JSON - single: Sends each log as individual HTTP request in parallel """ + batch: Final = tuple(self.log_queue) + if not batch: + return try: - if not self.log_queue: - return - verbose_logger.debug( - "Generic API Logger - about to flush %s events in '%s' format", len(self.log_queue), self.log_format + "Generic API Logger - about to flush %s events in '%s' format", len(batch), self.log_format ) if self.log_format == "single": # Send each log as individual HTTP request in parallel tasks: Final = [] - for log_entry in self.log_queue: + for log_entry in batch: task = self._post_with_retries(data=safe_dumps(log_entry)) tasks.append(task) @@ -384,9 +399,9 @@ class GenericAPILogger(CustomBatchLogger): else: # Format the payload based on log_format if self.log_format == "json_array": - data = safe_dumps(self.log_queue) + data = safe_dumps(batch) elif self.log_format == "ndjson": - data = "\n".join(safe_dumps(log) for log in self.log_queue) + data = "\n".join(safe_dumps(log) for log in batch) else: raise ValueError(f"Unknown log_format: {self.log_format}") @@ -403,7 +418,8 @@ class GenericAPILogger(CustomBatchLogger): except Exception as e: verbose_logger.exception("Generic API Logger Error sending batch - %s\n%s", e, traceback.format_exc()) finally: - self.log_queue.clear() + del self.log_queue[: len(batch)] + self.last_flush_time = time.time() def _get_v1_logging_payload(self, kwargs, response_obj, start_time, end_time) -> dict: """ diff --git a/scripts/check_schema_compatibility.py b/scripts/check_schema_compatibility.py new file mode 100644 index 00000000000..898c65dff2c --- /dev/null +++ b/scripts/check_schema_compatibility.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from collections.abc import Iterator, Mapping +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +Schema = Mapping[str, Mapping[str, str]] +MODELS: Final = re.compile(r"^[ \t]*model\s+(\w+)\s*\{(.*?)^[ \t]*\}", re.MULTILINE | re.DOTALL) + + +def parse_model_fields(schema: str) -> Schema: + def fields(body: str) -> Mapping[str, str]: + parts: Final = tuple(line.split() for line in body.splitlines()) + return MappingProxyType({part[0]: " ".join(part[1:]) for part in parts if len(part) >= 2}) + + return MappingProxyType({match[1]: fields(match[2]) for match in MODELS.finditer(schema)}) + + +def _field_changes(model: str, field: str, old_type: str, new_fields: Mapping[str, str]) -> Iterator[str]: + if field not in new_fields: + yield f"Breaking: Field {model}.{field} was removed" + return + new_type: Final = new_fields[field] + if old_type == new_type: + return + if "?" in old_type and "?" not in new_type: + yield f"Breaking: Field {model}.{field} changed from optional to required" + if not old_type.startswith(new_type.split("?")[0]): + yield f"Breaking: Field {model}.{field} changed type from {old_type} to {new_type}" + + +def _model_changes(model: str, old_fields: Mapping[str, str], new_schema: Schema) -> Iterator[str]: + if model not in new_schema: + yield f"Breaking: Model {model} was removed" + return + for field, old_type in old_fields.items(): + yield from _field_changes(model, field, old_type, new_schema[model]) + + +def check_breaking_changes(old_schema: Schema, new_schema: Schema) -> tuple[str, ...]: + def changes() -> Iterator[str]: + for model, fields in old_schema.items(): + yield from _model_changes(model, fields, new_schema) + + return tuple(changes()) + + +def main() -> int: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--base-ref", required=True) + options: Final = parser.parse_args() + base_ref: Final = TypeAdapter(str).validate_python(options.base_ref) + base: Final = subprocess.run( + ("git", "merge-base", "HEAD", base_ref), capture_output=True, text=True, check=True + ).stdout.strip() + previous: Final = subprocess.run( + ("git", "show", f"{base}:schema.prisma"), capture_output=True, text=True, check=True + ).stdout + changes: Final = check_breaking_changes( + parse_model_fields(previous), parse_model_fields(Path("schema.prisma").read_text()) + ) + if changes: + sys.stderr.write("\n".join(changes) + "\n") + return 1 + sys.stdout.write(f"Schema compatibility passed against merge base {base}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index 8df9c737ba0..5a09b66e7ed 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -24,4 +24,4 @@ function in a full stack is not By the domain a user would name: `pricing`, `spend`, `routing`, `mcp`. A file only needs to live in a directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no `covers` marker on new tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a -fix in the test and not a deletion. Needs no proxy, DB or Redis: `tests/unit` +fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit` diff --git a/tests/integration/README.md b/tests/integration/README.md index c53220cc895..8734e2e6237 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -10,13 +10,17 @@ Use `tests/integration/run.py management`, `accounting`, `database`, `providers` Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload -The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles the file order and the test order inside each file but keeps each file's tests together, so module fixtures are built once per file. Use `--seed` and `--order-seed` to reproduce a run. Actual installed Hypothesis version, settings, seeds and collected order are written beside the execution manifest +The generated lifecycle models use 20 examples, eight steps, generation and shrinking, with isolated resources per example. HTTP operation caps include generation and shrinking and exempt cleanup. Local qualification defaults to seed 4106601 and canonical order; CircleCI derives exploration and ordering seeds from the checked-out revision and workflow ID. The ordering seed shuffles files, scoped fixture configurations and cases within each configuration. Cases sharing a module or class fixture parameter stay together, so shuffling does not repeatedly rebuild the same configuration. Case partitions also group tests using the same unparametrized module or class fixtures. These groups split when they exceed a node's estimated time budget, limiting repeated setup without placing an oversized group on one node. A successful shard census saves every case's setup, call and teardown duration. The next CircleCI run restores that shared snapshot and balances measured cost. Missing or invalid timing data falls back to equal case weights. New cases receive the median known cost, and stale case identities cannot select or omit tests. Use `--timings ` to reproduce a measured partition Use `--seed` and `--order-seed` to reproduce a run. Actual installed Hypothesis version, settings, seeds and collected order are written beside the execution manifest Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change +Owned proxies disable `.env` loading by default. A contract testing dotenv behavior can explicitly set `PYTHON_DOTENV_DISABLED=0` in its owned environment. An explicit scratch writer database drops an inherited reader with a different database name; pass both database URLs when the contract needs a paired reader. Database relays close listeners, active connections and their event loops on teardown. Upstream startup failures reap the child process before returning the failure. Custom subprocesses, direct SQL and background tasks still need an explicit owner in the test + +Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline; application readiness and graceful teardown retain their separate longer deadlines. A fault that deliberately pauses Python before it can answer worker pings must supply a longer healthcheck through `extra_arguments`. Slow application startup is covered separately by the readiness deadline. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and prove that the intended worker had active traffic before killing it + The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection -There is no per-node manifest. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI +There is no hand-maintained list of test cases. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI Provider sentinels currently use the controlled server, not live recordings. The provider shard also runs the existing strict replay controls for changed requests, exhausted interactions, leftover interactions and no provider connection. Future recorded scenarios must use that replay-only implementation; missing recordings cannot fall back to a real provider. The observation endpoint is destructive and the current selection runs serially against one owned upstream @@ -30,15 +34,19 @@ Accounting cases compare persisted input and output cost components against lite Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior +CircleCI runs providers on 16 parallel nodes, partitioning cases while keeping scoped fixture configurations together. Each node owns its services. The dependent providers coverage job verifies all 16 execution records against the full case inventory and rejects missing, duplicate, failed or incomplete execution. Use `--shards 16 --shard-index ` to reproduce a node locally + Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. CircleCI reuses its existing Redis image for that extra process; it never pulls an image during tests The `messages_endpoint/` directory holds `/v1/messages` endpoint contracts: native-provider backends under `providers/` (`anthropic`, `bedrock`, `gemini`) and the translation bridges (`responses_bridge`, `chat_bridge`) at the top level. It runs in the providers shard; `run.py` selects test files recursively under each scheduled directory The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy or database. CircleCI starts a local Redis for this shard like the others, so SDK-side caching cases that need a real Redis server belong here too; a case that reaches the gateway belongs in one of the other shards -The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on parallel nodes, and each node starts its own database, Redis, upstream and proxy and runs its share of the group's files serially, split by recorded timings with `circleci tests split`. Tests keep the isolation of a serial run; they still must not assume a particular set of sibling files. `run.py --list` prints a group's files and `run.py ...` runs a subset of them +The extensions shard uses the built-in generic callback and guardrail transports. It checks callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers and A2A wire versions. CircleCI runs it on 32 parallel large nodes. Each node starts its own database, Redis, upstream and proxy and runs its share of cases serially. The runner assigns cases deterministically by estimated duration, keeping scoped fixture configurations together. Each node records the full collection inventory and its selected, passed and skipped nodes. The dependent coverage job rejects missing shards, differing inventories, omitted or duplicate cases and incomplete execution. Use `--shards 32 --shard-index ` to reproduce a node locally. More parallel nodes increase setup overhead and concurrent machine use. Tests keep the isolation of a serial run; they still must not assume a particular set of sibling files. `run.py --list` prints a group's files and `run.py ...` runs a subset of them -The mcp shard runs the MCP gateway against SDK peers owned by each test (`_support/mcp.py`): streamable HTTP, SSE and stdio peers, an OpenAPI-spec app, and an OAuth 2.1 authorization-server double. Every peer records the requests it receives so a test can assert what reached the peer, not only what the proxy answered. The shard runs with `INTEGRATION_WORKERS` set and with `INTEGRATION_COVERAGE=1`, which starts the proxy under `coverage run --parallel-mode` limited to the MCP modules and stores `coverage.txt` plus an HTML report with the job artifacts. A test that fails because the product is wrong is skipped with `pytest.skip("BUG: ")` so the skip list in `execution.json` is the open MCP bug list +The security shard uses four pytest workers on an xlarge machine, each with a private database and Redis. Route sweeps use a new HTTP client for every call and share only the TLS verification context, so authentication and response cookies cannot cross callers. The canary rig supplies owned OpenAI and Anthropic endpoints and synthetic credentials and rejects attempted external egress + +The two mcp nodes own separate services and use the same case census as providers and extensions. Their coverage data is combined into one MCP report after both nodes finish. Each node runs the MCP gateway against SDK peers owned by each test (`_support/mcp.py`): streamable HTTP, SSE and stdio peers, an OpenAPI-spec app, and an OAuth 2.1 authorization-server double. Every peer records the requests it receives so a test can assert what reached the peer, not only what the proxy answered. The shard runs with `INTEGRATION_WORKERS` set and with `INTEGRATION_COVERAGE=1`, which starts the proxy under `coverage run --parallel-mode` limited to the MCP modules and stores `coverage.txt` plus an HTML report with the job artifacts. A test that fails because the product is wrong is skipped with `pytest.skip("BUG: ")` so the skip list in `execution.json` is the open MCP bug list Browser contracts live in `tests/e2e/ui/tests/integrationCritical` and run only through `tests/e2e/ui/integration.config.ts`. The expected browser results are listed in `expected.json` in that directory and checked by `.circleci/scripts/verify_integration_browser.py`. The CircleCI browser shard builds the checked-out dashboard, starts the owned proxy with that build, and verifies one exact browser result without retries or skips. The default Playwright selection excludes this directory. The focused project flow asserts the submitted create and clear values, fresh SQL state and actual blocked/restored serving while preserving model restrictions diff --git a/tests/integration/_support/async_server.py b/tests/integration/_support/async_server.py new file mode 100644 index 00000000000..324cbaa5da4 --- /dev/null +++ b/tests/integration/_support/async_server.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from contextlib import suppress +from threading import Thread +from typing import Final + +Handler = Callable[[asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] + + +async def close_writer(writer: asyncio.StreamWriter) -> None: + writer.close() + with suppress(ConnectionError, OSError): + await writer.wait_closed() + + +class LoopbackServer: + def __init__(self, port: int, handler: Handler) -> None: + self.loop: Final = asyncio.new_event_loop() + self._port: Final = port + self._handler: Final = handler + self._server: asyncio.Server | None = None + self._thread: Final = Thread(target=self._run, name=f"integration-relay-{port}", daemon=True) + self._connections: tuple[asyncio.Task[None], ...] = () + + def start(self) -> None: + try: + self._thread.start() + asyncio.run_coroutine_threadsafe(self._bind(), self.loop).result(timeout=10) + except BaseException: + self.stop() + raise + + def stop(self) -> None: + if not self._thread.is_alive(): + if not self.loop.is_closed(): + self.loop.close() + return + try: + asyncio.run_coroutine_threadsafe(self._close(), self.loop).result(timeout=10) + finally: + self.loop.call_soon_threadsafe(self.loop.stop) + self._thread.join(10) + assert not self._thread.is_alive(), f"Database relay thread on port {self._port} did not stop" + + def _run(self) -> None: + asyncio.set_event_loop(self.loop) + try: + self.loop.run_forever() + finally: + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() + + async def _bind(self) -> None: + self._server = await asyncio.start_server(self._serve, "127.0.0.1", self._port) + + async def _serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task: Final = asyncio.current_task() + assert task is not None + self._connections = (*self._connections, task) + try: + await self._handler(reader, writer) + finally: + await close_writer(writer) + self._connections = tuple(connection for connection in self._connections if connection is not task) + + async def _close(self) -> None: + server: Final = self._server + if server is not None: + server.close() + connections: Final = self._connections + for task in connections: + task.cancel() + await asyncio.gather(*connections, return_exceptions=True) + current: Final = asyncio.current_task() + remaining: Final = tuple(task for task in asyncio.all_tasks(self.loop) if task is not current) + for task in remaining: + task.cancel() + await asyncio.gather(*remaining, return_exceptions=True) + if server is not None: + await server.wait_closed() diff --git a/tests/integration/_support/database_relay.py b/tests/integration/_support/database_relay.py index 20f23d0e010..754813256fe 100644 --- a/tests/integration/_support/database_relay.py +++ b/tests/integration/_support/database_relay.py @@ -9,6 +9,8 @@ from urllib.parse import urlsplit, urlunsplit from pydantic import TypeAdapter +from tests.integration._support.async_server import LoopbackServer, close_writer + PORT: Final = TypeAdapter(int) OUTAGE_SECONDS: Final = 10.0 @@ -25,32 +27,23 @@ class DatabaseRelay: self._upstream_host: Final = upstream_host self._upstream_port: Final = upstream_port self._trigger: Final = trigger - self._loop: Final = asyncio.new_event_loop() + self._server: Final = LoopbackServer(self.port, self._serve) + self._loop: Final = self._server.loop self._armed: Final = threading.Event() self.tripped: Final = threading.Event() self.refused = 0 self.reconnected: Final = threading.Event() self._tripped_at = 0.0 self._writers: tuple[asyncio.StreamWriter, ...] = () - self._ready: Final = threading.Event() - self._thread: Final = threading.Thread(target=self._run, daemon=True) def arm(self) -> None: self._armed.set() def start(self) -> None: - self._thread.start() - assert self._ready.wait(10), "Database relay did not start" + self._server.start() def stop(self) -> None: - self._loop.call_soon_threadsafe(self._loop.stop) - self._thread.join(10) - - def _run(self) -> None: - asyncio.set_event_loop(self._loop) - self._loop.run_until_complete(asyncio.start_server(self._serve, "127.0.0.1", self.port)) - self._ready.set() - self._loop.run_forever() + self._server.stop() def _drop_all(self) -> None: for writer in self._writers: @@ -80,7 +73,7 @@ class DatabaseRelay: except (ConnectionError, asyncio.IncompleteReadError): return finally: - writer.close() + await close_writer(writer) await asyncio.gather( forward(client_reader, server_writer, True), @@ -94,29 +87,21 @@ class HeldStatementRelay: self._upstream_host: Final = upstream_host self._upstream_port: Final = upstream_port self._trigger: Final = trigger - self._loop: Final = asyncio.new_event_loop() + self._server: Final = LoopbackServer(self.port, self._serve) + self._loop: Final = self._server.loop self._released: Final = asyncio.Event() self.held: Final = threading.Event() - self._ready: Final = threading.Event() - self._thread: Final = threading.Thread(target=self._run, daemon=True) def release(self) -> None: - self._loop.call_soon_threadsafe(self._released.set) + if not self._loop.is_closed(): + self._loop.call_soon_threadsafe(self._released.set) def start(self) -> None: - self._thread.start() - assert self._ready.wait(10), "Database relay did not start" + self._server.start() def stop(self) -> None: self.release() - self._loop.call_soon_threadsafe(self._loop.stop) - self._thread.join(10) - - def _run(self) -> None: - asyncio.set_event_loop(self._loop) - self._loop.run_until_complete(asyncio.start_server(self._serve, "127.0.0.1", self.port)) - self._ready.set() - self._loop.run_forever() + self._server.stop() def _holds(self, window: bytes) -> bool: return not self.held.is_set() and self._trigger in window @@ -138,7 +123,7 @@ class HeldStatementRelay: except (ConnectionError, asyncio.IncompleteReadError): return finally: - writer.close() + await close_writer(writer) await asyncio.gather( forward(client_reader, server_writer, True), @@ -163,11 +148,10 @@ class DroppedConnectionRelay: self._upstream_host: Final = upstream_host self._upstream_port: Final = upstream_port self._trigger: Final = trigger - self._loop: Final = asyncio.new_event_loop() + self._server: Final = LoopbackServer(self.port, self._serve) + self._loop: Final = self._server.loop self._armed: Final = threading.Event() self.dropped: Final = threading.Event() - self._ready: Final = threading.Event() - self._thread: Final = threading.Thread(target=self._run, daemon=True) def arm(self) -> None: self._armed.set() @@ -176,18 +160,10 @@ class DroppedConnectionRelay: self._armed.clear() def start(self) -> None: - self._thread.start() - assert self._ready.wait(10), "Database relay did not start" + self._server.start() def stop(self) -> None: - self._loop.call_soon_threadsafe(self._loop.stop) - self._thread.join(10) - - def _run(self) -> None: - asyncio.set_event_loop(self._loop) - self._loop.run_until_complete(asyncio.start_server(self._serve, "127.0.0.1", self.port)) - self._ready.set() - self._loop.run_forever() + self._server.stop() async def _serve(self, client_reader: asyncio.StreamReader, client_writer: asyncio.StreamWriter) -> None: server_reader, server_writer = await asyncio.open_connection(self._upstream_host, self._upstream_port) @@ -206,7 +182,7 @@ class DroppedConnectionRelay: except (ConnectionError, asyncio.IncompleteReadError): return finally: - writer.close() + await close_writer(writer) await asyncio.gather( forward(client_reader, server_writer, True), @@ -225,8 +201,8 @@ def database_relay(database_url: str, trigger: bytes) -> Generator[tuple[Databas parts: Final = urlsplit(database_url) assert parts.hostname is not None and parts.port is not None, database_url relay: Final = DatabaseRelay(parts.hostname, parts.port, trigger) - relay.start() try: + relay.start() yield relay, _relayed_url(database_url, relay.port) finally: relay.stop() @@ -237,8 +213,8 @@ def held_statement_relay(database_url: str, trigger: bytes) -> Generator[tuple[H parts: Final = urlsplit(database_url) assert parts.hostname is not None and parts.port is not None, database_url relay: Final = HeldStatementRelay(parts.hostname, parts.port, trigger) - relay.start() try: + relay.start() yield relay, _relayed_url(database_url, relay.port) finally: relay.stop() @@ -249,8 +225,8 @@ def dropped_connection_relay(database_url: str, trigger: bytes) -> Generator[tup parts: Final = urlsplit(database_url) assert parts.hostname is not None and parts.port is not None, database_url relay: Final = DroppedConnectionRelay(parts.hostname, parts.port, trigger) - relay.start() try: + relay.start() yield relay, _relayed_url(database_url, relay.port) finally: relay.stop() diff --git a/tests/integration/_support/ordering.py b/tests/integration/_support/ordering.py new file mode 100644 index 00000000000..7dbc85396ff --- /dev/null +++ b/tests/integration/_support/ordering.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from functools import reduce +from itertools import chain, groupby +from math import ceil, isfinite +from statistics import median +from types import MappingProxyType +from typing import Final + +import pytest +from _pytest.python import get_direct_param_fixture_func +from pydantic import BaseModel, ConfigDict, Field, FiniteFloat, TypeAdapter, ValidationError + +EMPTY_DURATIONS: Final[Mapping[str, float]] = MappingProxyType({}) + + +class CaseTiming(BaseModel): + model_config = ConfigDict(frozen=True, strict=True) + + nodeid: str = Field(min_length=1) + seconds: FiniteFloat = Field(ge=0) + + +TIMINGS: Final = TypeAdapter(tuple[CaseTiming, ...]) + + +def parse_timings(content: str) -> Mapping[str, float]: + try: + values: Final = TIMINGS.validate_json(content) + except ValidationError: + return EMPTY_DURATIONS + durations: Final = {value.nodeid: max(value.seconds, 0.001) for value in values} + if len(durations) != len(values) or not isfinite(sum(durations.values())): + return EMPTY_DURATIONS + return MappingProxyType(durations) + + +@dataclass(frozen=True, slots=True) +class ScopedParameter: + owner: str + name: str + value: object + + +@dataclass(frozen=True, slots=True) +class CollectedCase: + nodeid: str + parameters: tuple[ScopedParameter, ...] = () + fixtures: tuple[str, ...] = () + + +def _parameter(item: pytest.Function, name: str, value: object) -> ScopedParameter | None: + definitions: Final = item._fixtureinfo.name2fixturedefs.get(name) + scope: Final = item.callspec._arg2scope[name].value + if not definitions or scope == "function" or definitions[-1].func is get_direct_param_fixture_func: + return None + parent: Final = item.getparent(pytest.Class) if scope == "class" else None + owner: Final = parent.nodeid if parent is not None else definitions[-1].baseid + return ScopedParameter(f"{scope}:{owner}", name, value) + + +def collected_case(item: pytest.Item) -> CollectedCase: + if not isinstance(item, pytest.Function): + return CollectedCase(item.nodeid) + fixtures: Final = tuple( + f"{definitions[-1].scope}:{definitions[-1].baseid}:{name}" + for name, definitions in item._fixtureinfo.name2fixturedefs.items() + if definitions + and definitions[-1].scope in ("module", "class") + and definitions[-1].baseid + and definitions[-1].func is not get_direct_param_fixture_func + ) + parameters: Final = ( + tuple(_parameter(item, name, value) for name, value in item.callspec.params.items()) + if hasattr(item, "callspec") + else () + ) + scoped: Final = tuple(parameter for parameter in parameters if parameter is not None) + return CollectedCase( + item.nodeid, + tuple(sorted(scoped, key=lambda parameter: (parameter.owner, parameter.name))), + tuple(sorted(fixtures)), + ) + + +def _equal(first: object, second: object) -> bool: + try: + return bool(first == second) + except (ValueError, RuntimeError): + return first is second + + +def _same_parameters(first: CollectedCase, second: CollectedCase) -> bool: + if len(first.parameters) != len(second.parameters): + return False + if not first.parameters and first.fixtures != second.fixtures: + return False + return all( + left.owner == right.owner and left.name == right.name and _equal(left.value, right.value) + for left, right in zip(first.parameters, second.parameters, strict=True) + ) + + +def _file(node: CollectedCase) -> str: + return node.nodeid.split("::", 1)[0] + + +def _file_groups(cases: Iterable[CollectedCase]) -> tuple[tuple[str, str], ...]: + members: Final = tuple(cases) + return tuple( + (case.nodeid, min(other.nodeid for other in members if _same_parameters(case, other))) + if case.parameters or case.fixtures + else (case.nodeid, "") + for case in members + ) + + +def fixture_groups(cases: tuple[CollectedCase, ...]) -> Mapping[str, str]: + files: Final = groupby(sorted(cases, key=_file), key=_file) + return MappingProxyType(dict(chain.from_iterable(_file_groups(members) for _, members in files))) + + +def _allocate( + shards: tuple[tuple[str, ...], ...], group: tuple[str, ...], durations: Mapping[str, float] +) -> tuple[tuple[str, ...], ...]: + index: Final = min( + range(len(shards)), + key=lambda candidate: (sum(durations[nodeid] for nodeid in shards[candidate]), len(shards[candidate])), + ) + return tuple((*shard, *group) if candidate == index else shard for candidate, shard in enumerate(shards)) + + +def _add_case( + groups: tuple[tuple[str, ...], ...], case: CollectedCase, maximum: float, durations: Mapping[str, float] +) -> tuple[tuple[str, ...], ...]: + if not groups: + return ((case.nodeid,),) + if sum(durations[nodeid] for nodeid in groups[-1]) + durations[case.nodeid] > maximum: + return (*groups, (case.nodeid,)) + return (*groups[:-1], (*groups[-1], case.nodeid)) + + +def _partition_group( + cases: Iterable[CollectedCase], maximum: float, durations: Mapping[str, float] +) -> tuple[tuple[str, ...], ...]: + members: Final = tuple(sorted(cases, key=lambda case: case.nodeid)) + nodeids: Final = tuple(case.nodeid for case in members) + if members[0].parameters: + return (nodeids,) + return reduce(lambda groups, case: _add_case(groups, case, maximum, durations), members, ()) + + +def case_shards( + cases: tuple[CollectedCase, ...], count: int, durations: Mapping[str, float] = EMPTY_DURATIONS +) -> tuple[frozenset[str], ...]: + if count < 1: + raise ValueError("Integration shard count must be positive") + groups: Final = fixture_groups(cases) + known: Final = tuple(durations[case.nodeid] for case in cases if case.nodeid in durations) + default: Final = max(float(median(known)), 0.001) if known else 1.0 + weights: Final = MappingProxyType({case.nodeid: max(durations.get(case.nodeid, default), 0.001) for case in cases}) + + def key(case: CollectedCase) -> str: + return groups[case.nodeid] or case.nodeid + + maximum: Final = sum(weights.values()) / count if known else float(max(1, ceil(len(cases) / count))) + members: Final = tuple( + chain.from_iterable( + _partition_group(group, maximum, weights) for _, group in groupby(sorted(cases, key=key), key=key) + ) + ) + ordered: Final = tuple(sorted(members, key=lambda group: (-sum(weights[nodeid] for nodeid in group), group[0]))) + allocation: Final = reduce( + lambda shards, group: _allocate(shards, group, weights), ordered, tuple(() for _ in range(count)) + ) + return tuple(frozenset(shard) for shard in allocation) diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 23297b28584..c4c037603ff 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -1,4 +1,5 @@ import errno +import json import os import signal import socket @@ -14,6 +15,7 @@ from ipaddress import IPv4Address from pathlib import Path from types import MappingProxyType from typing import Final +from urllib.parse import urlsplit import httpx import psutil @@ -103,19 +105,25 @@ def owned_proxy( def _stop(process: subprocess.Popen[bytes]) -> None: - root_stopped: Final = stop_root_process(process) - residual: Final = group_members(process.pid) - if residual: - signal_group(process.pid, signal.SIGTERM) - psutil.wait_procs(residual, timeout=5) - remaining: Final = group_members(process.pid) - if remaining: + try: + root_stopped: Final = stop_root_process(process) + residual: Final = group_members(process.pid) + if residual: + signal_group(process.pid, signal.SIGTERM) + psutil.wait_procs(residual, timeout=5) + remaining: Final = group_members(process.pid) + if remaining: + signal_group(process.pid, signal.SIGKILL) + psutil.wait_procs(remaining, timeout=3) + process.wait(timeout=3) + survivors: Final = group_members(process.pid) + assert not survivors, "Owned proxy child survived cleanup" + assert root_stopped and not remaining, "Owned proxy required forced cleanup" + except BaseException: signal_group(process.pid, signal.SIGKILL) - psutil.wait_procs(remaining, timeout=3) - process.wait(timeout=3) - survivors: Final = group_members(process.pid) - assert not survivors, "Owned proxy child survived cleanup" - assert root_stopped and not remaining, "Owned proxy required forced cleanup" + process.wait(timeout=3) + psutil.wait_procs(group_members(process.pid), timeout=3) + raise _PORT_ATTEMPTS: Final = 3 @@ -133,11 +141,38 @@ class _Launch: process: subprocess.Popen[bytes] port: int log: Path + started: float + + +def _stop_launch(launch: _Launch) -> None: + started: Final = time.monotonic() + exit_before: Final = launch.process.poll() + exception_before: Final = sys.exc_info()[0] + try: + _stop(launch.process) + finally: + exception_at_record: Final = sys.exc_info()[0] + launch.log.with_suffix(".exit.json").write_text( + json.dumps( + { + "root_pid": launch.process.pid, + "exit_before_stop": exit_before, + "exit_after_stop": launch.process.poll(), + "exception_before_stop": exception_before.__name__ if exception_before else None, + "exception_at_record": exception_at_record.__name__ if exception_at_record else None, + "cleanup_seconds": time.monotonic() - started, + "process_lifetime_seconds": time.monotonic() - launch.started, + }, + indent=2, + ) + + "\n" + ) def _launch(command: tuple[str, ...], root: Path, environment: Mapping[str, str], output: Path) -> _Launch: port: Final = _free_port() log_path: Final = output / f"owned-proxy-{uuid.uuid4().hex}.log" + started: Final = time.monotonic() with log_path.open("w") as log: process: Final = subprocess.Popen( [*command, "--port", str(port)], @@ -147,7 +182,7 @@ def _launch(command: tuple[str, ...], root: Path, environment: Mapping[str, str] stderr=subprocess.STDOUT, start_new_session=True, ) - return _Launch(process, port, log_path) + return _Launch(process, port, log_path, started) def _lost_port_race(exit_code: int | None, log: Path) -> bool: @@ -178,11 +213,21 @@ def _launch_until_bound( "Owned proxy exited before readiness" ) except BaseException: - _stop(launch.process) + _stop_launch(launch) raise if exit_code is None: + launch.log.with_suffix(".startup.json").write_text( + json.dumps( + { + "root_pid": launch.process.pid, + "readiness_seconds": time.monotonic() - launch.started, + "logical_cpus": os.cpu_count(), + } + ) + + "\n" + ) return launch - _stop(launch.process) + _stop_launch(launch) return _launch_until_bound(command, root, environment, output, attempts - 1) @@ -193,17 +238,31 @@ def _proxy_root() -> Path: def _proxy_environment( gateway: Gateway, overrides: Mapping[str, str], remove_environment: tuple[str, ...] ) -> Mapping[str, str]: + inherited: Final = {**os.environ, **proxy_database_environment()} + writer: Final = overrides.get("DATABASE_URL") + reader: Final = inherited.get("DATABASE_URL_READ_REPLICA") + unpaired_reader: Final = ( + writer is not None + and reader is not None + and "DATABASE_URL_READ_REPLICA" not in overrides + and urlsplit(writer).path != urlsplit(reader).path + ) + removed: Final = frozenset((*remove_environment, *(("DATABASE_URL_READ_REPLICA",) if unpaired_reader else ()))) + configured_hooks: Final = overrides.get( + "LITELLM_WORKER_STARTUP_HOOKS", + inherited.get("LITELLM_WORKER_STARTUP_HOOKS", "") if "LITELLM_WORKER_STARTUP_HOOKS" not in removed else "", + ) return MappingProxyType( { - **{ - name: value - for name, value in {**os.environ, **proxy_database_environment()}.items() - if name not in remove_environment - }, + **{name: value for name, value in inherited.items() if name not in removed}, "LITELLM_MASTER_KEY": gateway.key, "LITELLM_SALT_KEY": os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt"), "STORE_MODEL_IN_DB": "True", + "PYTHON_DOTENV_DISABLED": "1", **overrides, + "LITELLM_WORKER_STARTUP_HOOKS": ",".join( + hook for hook in ("integration._support.runtime:configure_executor", configured_hooks) if hook + ), } ) @@ -261,7 +320,7 @@ def owned_proxy_process( "--num_workers", str(workers), "--timeout_worker_healthcheck", - str(int(graceful_stop_seconds())), + "5", *database_setup, *extra_arguments, ) @@ -273,7 +332,7 @@ def owned_proxy_process( ) as client: yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), process, launch.log) finally: - _stop(process) + _stop_launch(launch) @contextmanager @@ -306,7 +365,7 @@ def owned_gateway_image( ) as client: yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), launch.process, launch.log) finally: - _stop(launch.process) + _stop_launch(launch) def _is_ready(client: httpx.Client) -> bool: @@ -353,7 +412,7 @@ def refused_boot_log( assert launch.process.returncode != 0, f"Proxy exited 0 instead of refusing to boot:\n{launch.log.read_text()}" return launch.log.read_text() finally: - _stop(launch.process) + _stop_launch(launch) _UPSTREAM_READY_SECONDS: Final = 60 @@ -439,19 +498,23 @@ class UpstreamSlot: start_new_session=True, ) self.process = process - deadline: Final = time.monotonic() + _UPSTREAM_READY_SECONDS - while process.poll() is None: - try: - probe: Final = httpx.get( - f"{self.url}/health", timeout=2, trust_env=False, verify=self.certificate is None - ) - if probe.status_code == 200: - return - except httpx.TransportError: - pass - assert time.monotonic() < deadline, f"Owned upstream readiness deadline exceeded: {log_path}" - time.sleep(0.1) - raise AssertionError(f"Owned upstream exited before readiness: {log_path}") + try: + deadline: Final = time.monotonic() + _UPSTREAM_READY_SECONDS + while process.poll() is None: + try: + probe: Final = httpx.get( + f"{self.url}/health", timeout=2, trust_env=False, verify=self.certificate is None + ) + if probe.status_code == 200: + return + except httpx.TransportError: + pass + assert time.monotonic() < deadline, f"Owned upstream readiness deadline exceeded: {log_path}" + time.sleep(0.1) + raise AssertionError(f"Owned upstream exited before readiness: {log_path}") + except BaseException: + self.stop() + raise def stop(self) -> None: process: Final = self.process @@ -468,8 +531,8 @@ def _harness_root() -> Path: def owned_upstream(directory: Path, *, tls: bool = False) -> Generator[UpstreamSlot]: certificate: Final = self_signed_certificate(directory) if tls else None slot: Final = UpstreamSlot(directory, _free_port(), _harness_root(), certificate) - slot.start() try: + slot.start() yield slot finally: if slot.process is not None: diff --git a/tests/integration/_support/proxy.py b/tests/integration/_support/proxy.py index 610d4724349..e1b2efac36d 100644 --- a/tests/integration/_support/proxy.py +++ b/tests/integration/_support/proxy.py @@ -2,14 +2,14 @@ import signal import sys from types import FrameType -from litellm import run_server - def _exit_on_reraised_term(signum: int, frame: FrameType | None) -> None: sys.exit(0) def main() -> None: + from litellm import run_server + signal.signal(signal.SIGTERM, _exit_on_reraised_term) run_server() diff --git a/tests/integration/_support/redis_process.py b/tests/integration/_support/redis_process.py index 3a5bf8bec23..7966d105302 100644 --- a/tests/integration/_support/redis_process.py +++ b/tests/integration/_support/redis_process.py @@ -105,7 +105,7 @@ def owned_redis(directory: Path) -> Iterator[OwnedRedis]: prefix = ("docker", "exec", "redis-cache", "redis-server") output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) - with (output / "owned-redis-recovery.log").open("w") as log: + with (output / f"owned-redis-{uuid.uuid4().hex}.log").open("w") as log: pid_file: Final = str(directory / "owned-redis.pid") if binary else f"/tmp/integration-redis-{uuid.uuid4().hex}.pid" server: Final = OwnedRedis(host, port, (*prefix, "--port", str(port), "--set-proc-title", "no", "--pidfile", pid_file, "--bind", "0.0.0.0" if not binary else "127.0.0.1", "--protected-mode", "no", "--save", "", "--appendonly", "no"), log, pid_file) try: diff --git a/tests/integration/_support/runtime.py b/tests/integration/_support/runtime.py new file mode 100644 index 00000000000..a709ab3d70d --- /dev/null +++ b/tests/integration/_support/runtime.py @@ -0,0 +1,8 @@ +import asyncio +from concurrent.futures import ThreadPoolExecutor + + +def configure_executor() -> None: + asyncio.get_running_loop().set_default_executor( + ThreadPoolExecutor(max_workers=32, thread_name_prefix="integration-provider") + ) diff --git a/tests/integration/_support/sharding.py b/tests/integration/_support/sharding.py new file mode 100644 index 00000000000..5267985ad17 --- /dev/null +++ b/tests/integration/_support/sharding.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import argparse +import sys +from collections import Counter +from itertools import chain +from pathlib import Path +from typing import Final + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from tests.integration._support.ordering import TIMINGS, CaseTiming + + +class ShardingOptions(BaseModel): + model_config = ConfigDict(frozen=True) + + directory: Path + shards: int = Field(gt=0) + timings_output: Path | None = None + + +class ShardExecution(BaseModel): + model_config = ConfigDict(frozen=True, strict=True) + + inventory: tuple[str, ...] + collected: tuple[str, ...] + passed: tuple[str, ...] + skipped: tuple[str, ...] + shard_count: int + shard_index: int + complete: bool + exitstatus: int + timings: tuple[CaseTiming, ...] = () + + +def merged_timings(executions: tuple[ShardExecution, ...]) -> tuple[CaseTiming, ...]: + timings: Final = tuple(chain.from_iterable(execution.timings for execution in executions)) + measured: Final = Counter(timing.nodeid for timing in timings) + selected: Final = Counter(chain.from_iterable(execution.collected for execution in executions)) + if measured != selected or any(amount != 1 for amount in measured.values()): + return () + return tuple(sorted(timings, key=lambda timing: timing.nodeid)) + + +def shard_errors(executions: tuple[ShardExecution, ...], count: int) -> tuple[str, ...]: + if not executions: + return ("No integration shard execution records",) + inventory: Final = frozenset(executions[0].inventory) + selected: Final = Counter(chain.from_iterable(execution.collected for execution in executions)) + indices: Final = tuple(execution.shard_index for execution in executions) + return ( + *(("Missing or duplicate integration shards",) if sorted(indices) != list(range(count)) else ()), + *( + ("Integration shard counts disagree",) + if any(execution.shard_count != count for execution in executions) + else () + ), + *( + ("Integration collection inventories disagree",) + if any(frozenset(execution.inventory) != inventory for execution in executions) + else () + ), + *( + ("Integration inventory is empty or contains duplicates",) + if not inventory or any(len(execution.inventory) != len(inventory) for execution in executions) + else () + ), + *( + ("Integration selection omitted, duplicated, or added test cases",) + if frozenset(selected) != inventory or any(value != 1 for value in selected.values()) + else () + ), + *( + ("An integration shard failed or did not execute its complete selection",) + if any( + not execution.complete + or execution.exitstatus != 0 + or Counter(execution.collected) != Counter(execution.passed + execution.skipped) + for execution in executions + ) + else () + ), + ) + + +def main() -> int: + parser: Final = argparse.ArgumentParser() + parser.add_argument("directory", type=Path) + parser.add_argument("--shards", required=True, type=int) + parser.add_argument("--timings-output", type=Path) + options: Final = ShardingOptions.model_validate(vars(parser.parse_args())) + paths: Final = tuple(sorted(options.directory.glob("*/execution.json"))) + try: + executions: Final = tuple(ShardExecution.model_validate_json(path.read_text()) for path in paths) + except (OSError, ValidationError) as error: + sys.stderr.write(f"Invalid integration shard execution record: {error}\n") + return 1 + errors: Final = shard_errors(executions, options.shards) + if errors: + sys.stderr.write("\n".join(errors) + "\n") + return 1 + if options.timings_output is not None: + options.timings_output.parent.mkdir(parents=True, exist_ok=True) + options.timings_output.write_bytes(TIMINGS.dump_json(merged_timings(executions))) + sys.stdout.write( + f"Verified {len(executions[0].inventory)} integration cases across {options.shards} isolated shards\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/_support/workers.py b/tests/integration/_support/workers.py new file mode 100644 index 00000000000..85350caac9a --- /dev/null +++ b/tests/integration/_support/workers.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from pathlib import Path +from types import MappingProxyType + +from tests.integration._support.database import scratch_database +from tests.integration._support.redis_process import owned_redis + + +@contextmanager +def worker_services(directory: Path) -> Iterator[Mapping[str, str]]: + directory.mkdir(parents=True, exist_ok=True) + with ( + scratch_database() as database, + owned_redis(directory) as cache, + ): + yield MappingProxyType( + { + "DATABASE_URL": database, + "INTEGRATION_PROXY_DATABASE_URL": database, + "INTEGRATION_PROXY_READ_REPLICA_URL": "", + "REDIS_HOST": cache.host, + "REDIS_PORT": str(cache.port), + } + ) diff --git a/tests/integration/configuration/test_multi_worker_slow_import_boot.py b/tests/integration/configuration/test_multi_worker_slow_import_boot.py index a90160e5e80..fb544f37a66 100644 --- a/tests/integration/configuration/test_multi_worker_slow_import_boot.py +++ b/tests/integration/configuration/test_multi_worker_slow_import_boot.py @@ -33,13 +33,20 @@ def _slow_worker_environment(directory: Path) -> dict[str, str]: @pytest.mark.timeout(2 * graceful_stop_seconds() + 60) def test_owned_proxy_workers_outlast_the_deployments_healthcheck_default(gateway: Gateway, tmp_path: Path) -> None: - with owned_proxy_process(gateway, tmp_path, _slow_worker_environment(tmp_path), workers=WORKERS) as owned: + with owned_proxy_process( + gateway, + tmp_path, + _slow_worker_environment(tmp_path), + workers=WORKERS, + extra_arguments=("--timeout_worker_healthcheck", str(int(graceful_stop_seconds()))), + ) as owned: started: Final = eventually( lambda: STARTED_WORKER.findall(owned.log.read_text()), lambda pids: len(pids) >= WORKERS, seconds=graceful_stop_seconds(), ) - assert owned.gateway.request("GET", "/health/readiness").status_code == 200 + response: Final = owned.gateway.request("GET", "/health/readiness") + assert response.status_code == 200, response.text log: Final = owned.log.read_text() assert len(started) == WORKERS, log assert DIED_WORKER.findall(log) == [], log diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 116bc237018..abb4a94c880 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,28 +3,52 @@ from __future__ import annotations import hashlib import json import os -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from importlib.metadata import version from pathlib import Path -from typing import Final +from typing import Final, Protocol, runtime_checkable import httpx import pytest +from pydantic import TypeAdapter from redis import Redis from tests.integration._support.client import Gateway, eventually, gateway_from_environment from tests.integration._support.generation import LIFECYCLE_SETTINGS from tests.integration._support.manifest import OWNED_DIRECTORIES +from tests.integration._support.ordering import CaseTiming, case_shards, collected_case, fixture_groups, parse_timings from tests.integration._support.provider import SharedProvider, shared_provider from tests.integration._support.routing import RoutingPlugin from tests.integration.run import GITHUB_FILES COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() +INVENTORY: Final = pytest.StashKey[tuple[str, ...]]() REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]() +WORKER_INVENTORY: Final = TypeAdapter(tuple[str, ...]) +TIMING_PATH: Final[TypeAdapter[Path | None]] = TypeAdapter(Path | None) +INTEGER_OPTION: Final = TypeAdapter(int) + + +@runtime_checkable +class WorkerConfig(Protocol): + @property + def workeroutput(self) -> dict[str, object]: ... + + +class WorkerNode(Protocol): + @property + def workeroutput(self) -> Mapping[str, object]: ... + + +def _worker_output(config: object) -> dict[str, object] | None: + return config.workeroutput if isinstance(config, WorkerConfig) else None def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption("--integration-order-seed", type=int, default=0) + parser.addoption("--integration-shard-count", type=int, default=1) + parser.addoption("--integration-shard-index", type=int, default=0) + parser.addoption("--integration-timings", type=Path) def pytest_configure(config: pytest.Config) -> None: @@ -47,6 +71,21 @@ class IntegrationReportPlugin: def pytest_xdist_node_collection_finished(self, node: object, ids: Sequence[str]) -> None: self.config.stash[COLLECTED] = tuple(nodeid for nodeid in ids if _owned(nodeid)) + @pytest.hookimpl(optionalhook=True) + def pytest_testnodedown(self, node: WorkerNode, error: object) -> None: + if error is not None: + return + inventory: Final = WORKER_INVENTORY.validate_python(node.workeroutput.get("integration_inventory")) + previous: Final = self.config.stash.get(INVENTORY, inventory) + if inventory != previous: + raise pytest.UsageError("Integration worker collection inventories disagree") + self.config.stash[INVENTORY] = inventory + + def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None: + output: Final = _worker_output(self.config) + if output is not None: + output["integration_inventory"] = self.config.stash.get(INVENTORY, ()) + def _owned(nodeid: str) -> bool: parts: Final = Path(nodeid.split("::", 1)[0]).parts @@ -57,14 +96,26 @@ def _digest(seed: int, identity: str) -> bytes: return hashlib.sha256(f"{seed}:{identity}".encode()).digest() -def _order_key(seed: int, nodeid: str) -> tuple[bytes, bytes]: - return _digest(seed, nodeid.split("::", 1)[0]), _digest(seed, nodeid) +def _order_key(seed: int, nodeid: str, group: str = "") -> tuple[bytes, bytes, bytes]: + return _digest(seed, nodeid.split("::", 1)[0]), _digest(seed, group or nodeid), _digest(seed, nodeid) -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - order_seed: Final = config.getoption("integration_order_seed") +def _timing_content(path: Path | None) -> str: + if path is None: + return "[]" + try: + return path.read_text() + except (OSError, UnicodeError): + return "[]" + + +@pytest.hookimpl(wrapper=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> Iterator[None]: + yield + order_seed: Final = INTEGER_OPTION.validate_python(config.getoption("integration_order_seed")) if order_seed: - items.sort(key=lambda item: _order_key(order_seed, item.nodeid)) + groups: Final = fixture_groups(tuple(collected_case(item) for item in items)) + items.sort(key=lambda item: _order_key(order_seed, item.nodeid, groups[item.nodeid])) root: Final = Path(__file__).parent owned: Final = tuple( item @@ -78,7 +129,21 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item raise pytest.UsageError("Integration contracts are owned by CircleCI") for item in owned: item.add_marker(pytest.mark.integration) - config.stash[COLLECTED] = tuple(item.nodeid for item in owned) + config.stash[INVENTORY] = tuple(item.nodeid for item in owned) + count: Final = INTEGER_OPTION.validate_python(config.getoption("integration_shard_count")) + index: Final = INTEGER_OPTION.validate_python(config.getoption("integration_shard_index")) + if count < 1 or not 0 <= index < count: + raise pytest.UsageError("Integration shard index must be within the positive shard count") + if count > 1: + timing_path: Final = TIMING_PATH.validate_python(config.getoption("integration_timings")) + shards: Final = case_shards( + tuple(collected_case(item) for item in owned), count, parse_timings(_timing_content(timing_path)) + ) + selected: Final = tuple(item for item in items if item not in owned or item.nodeid in shards[index]) + deselected: Final = tuple(item for item in owned if item.nodeid not in shards[index]) + config.hook.pytest_deselected(items=deselected) + items[:] = selected + config.stash[COLLECTED] = tuple(item.nodeid for item in items if item in owned) def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: @@ -89,6 +154,10 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: return collected: Final = session.config.stash.get(COLLECTED, ()) reports: Final = tuple(report for report in session.config.stash[REPORTS] if report.nodeid in collected) + timings: Final = tuple( + CaseTiming(nodeid=nodeid, seconds=sum(report.duration for report in reports if report.nodeid == nodeid)) + for nodeid in collected + ) passed: Final = tuple(report.nodeid for report in reports if report.when == "call" and report.passed) skipped: Final = tuple(report.nodeid for report in reports if report.skipped) complete: Final = ( @@ -103,8 +172,12 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: json.dumps( { "collected": collected, + "inventory": session.config.stash.get(INVENTORY, collected), + "shard_count": session.config.getoption("integration_shard_count"), + "shard_index": session.config.getoption("integration_shard_index"), "passed": passed, "skipped": skipped, + "timings": tuple({"nodeid": timing.nodeid, "seconds": timing.seconds} for timing in timings), "complete": complete, "exitstatus": exitstatus, "hypothesis_version": version("hypothesis"), @@ -138,7 +211,9 @@ def shared_provider_server() -> Iterator[SharedProvider]: with shared_provider() as server: yield server late: Final = server.received() - assert late == (), f"the shared fake provider got {[item.target for item in late]} after {server.last_test} finished" + assert late == (), ( + f"the shared fake provider got {[item.target for item in late]} after {server.last_test} finished" + ) @pytest.fixture diff --git a/tests/integration/database/test_harness_lifecycle.py b/tests/integration/database/test_harness_lifecycle.py new file mode 100644 index 00000000000..63e3cba7363 --- /dev/null +++ b/tests/integration/database/test_harness_lifecycle.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import asyncio +import errno +import os +import re +import signal +import socket +import subprocess +import sys +import time +from collections import Counter +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from contextlib import AbstractContextManager, ExitStack +from pathlib import Path +from queue import SimpleQueue +from threading import Event +from types import FrameType +from typing import Final, Literal + +import httpx +import psutil +import psycopg +import pytest +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict + +from tests.integration._support import process as process_support +from tests.integration._support.async_server import LoopbackServer +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.database_relay import ( + DatabaseRelay, + DroppedConnectionRelay, + HeldStatementRelay, + database_relay, + dropped_connection_relay, + held_statement_relay, +) +from tests.integration._support.runtime import configure_executor + + +class ExitRecord(TypedDict): + root_pid: ReadOnly[int] + exit_after_stop: ReadOnly[int | None] + exception_at_record: ReadOnly[str | None] + + +Relay = DatabaseRelay | DroppedConnectionRelay | HeldStatementRelay +RelayFactory = Callable[[str, bytes], AbstractContextManager[tuple[Relay, str]]] +STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") + + +@pytest.mark.parametrize("factory", (database_relay, dropped_connection_relay, held_statement_relay)) +def test_relay_closes_active_connections_and_releases_its_listener(factory: RelayFactory) -> None: + with ExitStack() as cleanup: + with factory(os.environ["DATABASE_URL"], b"unissued-lifecycle-trigger") as (relay, url): + connection: Final = cleanup.enter_context(psycopg.connect(url, connect_timeout=2, autocommit=True)) + assert connection.execute("SELECT 1").fetchone() == (1,) + with pytest.raises(psycopg.OperationalError): + connection.execute("SELECT 1") + with socket.socket() as replacement: + replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + replacement.bind(("127.0.0.1", relay.port)) + replacement.listen() + assert relay._loop.is_closed() + + +async def _unused_peer(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + await reader.read() + + +def test_relay_bind_failure_closes_its_loop_and_allows_repeated_cleanup() -> None: + with socket.socket() as occupied: + occupied.bind(("127.0.0.1", 0)) + occupied.listen() + relay: Final = LoopbackServer(occupied.getsockname()[1], _unused_peer) + with pytest.raises(OSError, match=os.strerror(errno.EADDRINUSE).lower()): + relay.start() + relay.stop() + assert relay.loop.is_closed() + + +def test_upstream_readiness_failure_reaps_the_started_process(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(process_support, "_UPSTREAM_READY_SECONDS", 0) + children_before: Final = frozenset(child.pid for child in psutil.Process().children(recursive=True)) + slot: Final = process_support.UpstreamSlot(tmp_path, process_support._free_port(), process_support._harness_root()) + try: + with pytest.raises(AssertionError, match="readiness deadline exceeded"): + slot.start() + assert slot.process is None + assert frozenset(child.pid for child in psutil.Process().children(recursive=True)) == children_before + finally: + if slot.process is not None: + slot.stop() + + +class _InterruptedCleanup(BaseException): + pass + + +async def test_owned_runtime_starts_the_entire_held_provider_burst_on_a_small_runner() -> None: + arrived: Final[SimpleQueue[bool]] = SimpleQueue() + complete: Final = Event() + release: Final = Event() + + def held() -> None: + arrived.put(True) + if arrived.qsize() == 20: + complete.set() + assert release.wait(timeout=5), "Held executor job was never released" + + with ThreadPoolExecutor(max_workers=1) as initial: + asyncio.get_running_loop().set_default_executor(initial) + configure_executor() + async with asyncio.TaskGroup() as tasks: + burst: Final = tuple(tasks.create_task(asyncio.to_thread(held)) for _ in range(20)) + try: + assert await asyncio.wait_for(asyncio.to_thread(complete.wait, 2), timeout=3) + assert arrived.qsize() == len(burst) + finally: + release.set() + + +@pytest.mark.timeout(90) +def test_slow_worker_startup_survives_healthchecks_and_a_stalled_worker_is_replaced( + gateway: Gateway, tmp_path: Path +) -> None: + hook: Final = tmp_path / "slow_worker_startup.py" + hook.write_text("from threading import Event\n\ndef configure():\n Event().wait(6)\n") + overrides: Final = { + "PYTHONPATH": os.pathsep.join((str(tmp_path), os.environ.get("PYTHONPATH", ""))), + "LITELLM_WORKER_STARTUP_HOOKS": "slow_worker_startup:configure", + } + with process_support.owned_proxy_process(gateway, tmp_path, overrides, workers=2) as owned: + workers: Final = eventually( + lambda: tuple(int(pid) for pid in STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2 and owned.log.read_text().count("Application startup complete.") == 2, + seconds=30, + ) + victim: Final = psutil.Process(workers[0]) + victim.suspend() + try: + replacement: Final = eventually( + lambda: tuple(int(pid) for pid in STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 3 and owned.log.read_text().count("Application startup complete.") == 3, + seconds=30, + ) + assert replacement[-1] not in workers + assert not psutil.pid_exists(victim.pid), "Supervisor did not reap the stalled worker" + response: Final = owned.gateway.request("GET", "/health/readiness") + assert response.status_code == 200, response.text + finally: + try: + victim.kill() + except psutil.NoSuchProcess: + pass + + +def test_interrupted_cleanup_reaps_the_owned_root_and_its_child() -> None: + child_code: Final = ( + "import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); print('ready',flush=True); time.sleep(60)" + ) + parent_code: Final = ( + "import signal,subprocess,sys,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); " + f"child=subprocess.Popen((sys.executable,'-P','-c',{child_code!r}),stdout=subprocess.PIPE); " + "assert child.stdout.readline()==b'ready\\n'; print('ready',flush=True); time.sleep(60)" + ) + + def interrupt(signum: int, frame: FrameType | None) -> None: + raise _InterruptedCleanup + + with subprocess.Popen( + (sys.executable, "-P", "-c", parent_code), start_new_session=True, stdout=subprocess.PIPE + ) as process: + try: + assert process.stdout is not None + assert process.stdout.readline() == b"ready\n" + assert len(process_support.group_members(process.pid)) == 2 + previous_timer: Final = signal.getitimer(signal.ITIMER_REAL) + started: Final = time.monotonic() + previous_handler: Final = signal.signal(signal.SIGALRM, interrupt) + try: + signal.setitimer(signal.ITIMER_REAL, 0.2) + with pytest.raises(_InterruptedCleanup): + process_support._stop(process) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + remaining: Final = ( + max(0.001, previous_timer[0] - (time.monotonic() - started)) if previous_timer[0] else 0 + ) + signal.setitimer(signal.ITIMER_REAL, remaining, previous_timer[1]) + assert process.poll() is not None + eventually(lambda: process_support.group_members(process.pid), lambda members: not members, seconds=5) + finally: + process_support.signal_group(process.pid, signal.SIGKILL) + process.wait(timeout=3) + + +@pytest.mark.parametrize("explicit_dotenv", (False, True)) +def test_owned_process_ignores_ambient_dotenv_unless_its_contract_opts_in( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, explicit_dotenv: bool +) -> None: + monkeypatch.delenv("OWNED_DOTENV_SENTINEL", raising=False) + monkeypatch.delenv("PYTHON_DOTENV_DISABLED", raising=False) + (tmp_path / ".env").write_text("OWNED_DOTENV_SENTINEL=ambient-config\n") + with httpx.Client(base_url="http://127.0.0.1:1", trust_env=False) as client: + environment: Final = process_support._proxy_environment( + Gateway(client, "owned-test-key", "http://127.0.0.1:1"), + {"PYTHON_DOTENV_DISABLED": "0"} if explicit_dotenv else {}, + (), + ) + result: Final = subprocess.run( + ( + sys.executable, + "-P", + "-c", + "import os; from prisma import Prisma; Prisma(); print(os.getenv('OWNED_DOTENV_SENTINEL', 'absent'))", + ), + cwd=tmp_path, + env=dict(environment), + capture_output=True, + text=True, + timeout=10, + check=True, + ) + assert result.stdout.strip() == ("ambient-config" if explicit_dotenv else "absent") + + +def test_seeded_collection_reuses_scoped_fixtures_and_executes_every_case(tmp_path: Path) -> None: + events: Final = tmp_path / "events.txt" + probe: Final = tmp_path / "test_fixture_probe.py" + probe.write_text( + "import pytest\nfrom pathlib import Path\n" + f"EVENTS = Path({str(events)!r})\n" + "CONFIGS = ('D1',)*5 + ('D2',)*5 + ('D3',)*4 + ('D4',)*4\n" + "def record(value):\n" + " with EVENTS.open('a') as output:\n" + " output.write(value + '\\n')\n" + "@pytest.fixture(scope='module')\n" + "def rig(request):\n" + " record('start ' + request.param)\n" + " yield request.param\n" + " record('stop ' + request.param)\n" + "@pytest.mark.parametrize(('rig', 'number'), " + "tuple((value, index) for index, value in enumerate(CONFIGS)), indirect=['rig'], scope='module')\n" + "def test_route(rig, number):\n" + " assert rig == CONFIGS[number]\n" + " record('case ' + str(number))\n" + ) + result: Final = subprocess.run( + ( + sys.executable, + "-P", + "-m", + "pytest", + str(probe), + "-p", + "tests.integration.conftest", + "--integration-order-seed=1848224111", + "--timeout=15", + "-q", + ), + env={ + name: value + for name, value in os.environ.items() + if name not in ("INTEGRATION_RESULTS_DIR", "INTEGRATION_ROUTING") + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + observed: Final = tuple(events.read_text().splitlines()) + configurations: Final = Counter({"D1": 1, "D2": 1, "D3": 1, "D4": 1}) + assert Counter(event.removeprefix("start ") for event in observed if event.startswith("start ")) == configurations + assert Counter(event.removeprefix("stop ") for event in observed if event.startswith("stop ")) == configurations + assert sorted(int(event.removeprefix("case ")) for event in observed if event.startswith("case ")) == list( + range(18) + ) + + +def test_partitioned_collection_reuses_unparametrized_fixtures_and_executes_every_case(tmp_path: Path) -> None: + events: Final = tmp_path / "events.txt" + probe: Final = tmp_path / "test_fixture_probe.py" + probe.write_text( + "import pytest\nfrom pathlib import Path\n" + f"EVENTS = Path({str(events)!r})\n" + "def record(value):\n" + " with EVENTS.open('a') as output:\n" + " output.write(value + '\\n')\n" + "@pytest.fixture(scope='module')\n" + "def shared():\n" + " record('start shared')\n" + " yield 'shared'\n" + " record('stop shared')\n" + "@pytest.mark.parametrize('number', range(4))\n" + "def test_shared(shared, number):\n" + " assert shared == 'shared'\n" + " record('shared ' + str(number))\n" + "@pytest.mark.parametrize('number', range(8))\n" + "def test_plain(number):\n" + " record('plain ' + str(number))\n" + ) + (tmp_path / "partition_probe.py").write_text( + "from integration._support.ordering import case_shards, collected_case\n" + "def pytest_collection_modifyitems(config, items):\n" + " shards = case_shards(tuple(collected_case(item) for item in items), 3)\n" + " selected = shards[config.getoption('integration_shard_index')]\n" + " items[:] = tuple(item for item in items if item.nodeid in selected)\n" + ) + results: Final = tuple( + subprocess.run( + ( + sys.executable, + "-P", + "-m", + "pytest", + str(probe), + "-p", + "tests.integration.conftest", + "-p", + "partition_probe", + "--integration-shard-count=3", + f"--integration-shard-index={index}", + "--integration-order-seed=1848224111", + "--timeout=15", + "-q", + ), + env={ + **{ + name: value + for name, value in os.environ.items() + if name not in ("INTEGRATION_RESULTS_DIR", "INTEGRATION_ROUTING") + }, + "PYTHONPATH": os.pathsep.join((str(tmp_path), os.environ.get("PYTHONPATH", ""))), + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + for index in range(3) + ) + assert all(result.returncode == 0 for result in results), tuple(result.stdout + result.stderr for result in results) + observed: Final = tuple(events.read_text().splitlines()) + assert observed.count("start shared") == observed.count("stop shared") == 1 + assert sorted(event for event in observed if event.startswith("shared ")) == [ + f"shared {index}" for index in range(4) + ] + assert sorted(event for event in observed if event.startswith("plain ")) == [f"plain {index}" for index in range(8)] + + +@pytest.mark.parametrize("entrypoint", ("gateway", "refused")) +def test_owned_entrypoints_record_exit_and_reap_the_process_group( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, entrypoint: Literal["gateway", "refused"] +) -> None: + monkeypatch.setenv("INTEGRATION_RESULTS_DIR", str(tmp_path)) + config: Final = tmp_path / "config.yaml" + if entrypoint == "refused": + config.write_text("model_list: [\n") + refused: Final = process_support.refused_boot_log(gateway, tmp_path, {}, config=config) + assert refused + else: + rejected_config: Final = tmp_path / "rejected.yaml" + rejected_config.write_text("model_list: [\n") + assert process_support.refused_boot_log(gateway, tmp_path, {}, config=rejected_config) + config.write_text( + "model_list: []\ngeneral_settings:\n master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + ) + with process_support.owned_gateway_image(gateway, tmp_path, {}, config=config, workers=1) as owned: + response: Final = owned.gateway.request("GET", "/health/readiness") + assert response.status_code == 200, response.text + completed: Final = TypeAdapter(ExitRecord).validate_json(owned.log.with_suffix(".exit.json").read_text()) + assert completed["root_pid"] == owned.process.pid + assert completed["exit_after_stop"] == owned.process.returncode + records: Final = tuple( + TypeAdapter(ExitRecord).validate_json(path.read_text()) for path in tmp_path.glob("*.exit.json") + ) + assert records + if entrypoint == "refused": + assert len(records) == 1 and records[0]["exit_after_stop"] != 0 + for record in records: + assert record["exit_after_stop"] is not None + assert record["exception_at_record"] is None + assert not process_support.group_members(record["root_pid"]) diff --git a/tests/integration/management/test_auto_router_databricks_classifier_writes.py b/tests/integration/management/test_auto_router_databricks_classifier_writes.py index 6f541f5e622..e277eddf009 100644 --- a/tests/integration/management/test_auto_router_databricks_classifier_writes.py +++ b/tests/integration/management/test_auto_router_databricks_classifier_writes.py @@ -1,7 +1,9 @@ import asyncio import json +import threading import uuid from collections.abc import Callable, Mapping +from queue import SimpleQueue from typing import Final import httpx @@ -294,8 +296,17 @@ async def test_renaming_the_endpoint_under_a_burst_moves_the_classifier_and_logs gateway: Gateway, ) -> None: base_url: Final = str(gateway.client.base_url) + loop: Final = asyncio.get_running_loop() + classifying: Final = asyncio.Event() + release: Final = threading.Event() + + def classify(request: Request) -> Reply: + loop.call_soon_threadsafe(classifying.set) + assert release.wait(timeout=30), "The in-flight classifier calls were never released" + return _classifier(request) + with ( - wire_server(_classifier) as judge, + wire_server(classify) as judge, wire_server(_tier("simple answer")) as simple, wire_server(_tier("complex answer")) as complex_tier, gateway.scenario() as scenario, @@ -304,29 +315,40 @@ async def test_renaming_the_endpoint_under_a_burst_moves_the_classifier_and_logs complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url) classifier: Final = _classifier_config(judge.url) name, identity = _create_router(scenario, _router_config(classifier, simple_model, complex_model)) - burst: Final = asyncio.create_task(_burst(base_url, gateway.key, name, 24)) renamed: Final = _router_config({**classifier, "model": _SECOND_ENDPOINT}, simple_model, complex_model) - patched: Final = gateway.request( - "PATCH", - f"/model/{identity}/update", - {"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": renamed}}, - ) - assert patched.status_code == 200, patched.text - during: Final = await burst + async with asyncio.TaskGroup() as requests: + burst: Final = requests.create_task(_burst(base_url, gateway.key, name, 24)) + try: + await classifying.wait() + patched: Final = gateway.request( + "PATCH", + f"/model/{identity}/update", + {"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": renamed}}, + ) + assert patched.status_code == 200, patched.text + finally: + release.set() + during: Final = await burst assert [response.status_code for response in during] == [200] * 24, [response.text for response in during] assert set(_judge_targets(judge)) <= {f"/{_ENDPOINT}/invocations", f"/{_SECOND_ENDPOINT}/invocations"} assert _stored_classifier(gateway, identity)["model"] == _SECOND_ENDPOINT + convergence_calls: Final[SimpleQueue[str]] = SimpleQueue() + def classifier_target() -> tuple[str, ...]: chat: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(name)) assert chat.status_code == 200, chat.text + convergence_calls.put(string_value(chat.json()["id"])) return _judge_targets(judge) eventually(classifier_target, lambda targets: targets == (f"/{_SECOND_ENDPOINT}/invocations",), seconds=30) after: Final = await _burst(base_url, gateway.key, name, 6) assert [response.status_code for response in after] == [200] * 6, [response.text for response in after] assert set(_judge_targets(judge)) == {f"/{_SECOND_ENDPOINT}/invocations"} - identities: Final = tuple(string_value(response.json()["id"]) for response in (*during, *after)) + identities: Final = ( + *tuple(string_value(response.json()["id"]) for response in (*during, *after)), + *tuple(convergence_calls.get_nowait() for _ in range(convergence_calls.qsize())), + ) assert len(set(identities)) == len(identities), identities rows: Final = _rows(name, want=len(identities)) request_rows: Final = [row for row in rows if row["origin"] != "autorouter_classifier"] diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index 9e946dfc253..cd5670bcea9 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -18,6 +18,7 @@ from integration._support.mcp import ( McpPeer, ScriptedTool, mcp_peer, + official_client_outcomes, register_mcp, scripted_peer, text_result, @@ -26,11 +27,36 @@ from integration._support.mcp import ( from integration._support.mcp_grants import create_toolset from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.workers import worker_services from openai import AsyncOpenAI, OpenAI from openai.types.chat import ChatCompletionMessageParam from openai.types.responses.tool_param import Mcp from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.xdist_group("mcp_llm_private_gateway") + + +@pytest.fixture(scope="session") +def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: + directory: Final = tmp_path_factory.mktemp("mcp-llm-gateway") + with ( + worker_services(directory) as services, + gateway_from_environment() as shared, + owned_proxy(shared, directory, services) as private, + ): + yield private, services + + +@pytest.fixture +def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + private, services = _llm_gateway + with pytest.MonkeyPatch.context() as environment: + for name, value in services.items(): + environment.setenv(name, value) + yield private + + Surface = Literal["chat", "responses", "messages", "messages_bridge"] SURFACES: Final[tuple[Surface, ...]] = ("chat", "responses", "messages", "messages_bridge") ADD: Final = {"a": 2, "b": 3} @@ -491,6 +517,33 @@ def test_toolset_gateway_url_gives_a_key_of_an_ungranted_team_no_tools_and_never assert response.status_code in (200, 400, 401, 403), response.text +def test_an_open_server_in_another_gateway_cannot_widen_this_gateways_tools(gateway: Gateway) -> None: + with gateway_from_environment() as shared: + with ( + mcp_peer() as foreign, + shared.scenario() as foreign_scenario, + mcp_peer() as local, + gateway.scenario() as scenario, + ): + foreign_alias: Final = "foreign" + uuid.uuid4().hex[:8] + foreign_id: Final = register_mcp(foreign_scenario, foreign, foreign_alias) + created: Final = shared.post("/key/generate", {"object_permission": {"mcp_servers": [foreign_id]}}) + key: Final = created["key"] + assert isinstance(key, str) + foreign_scenario.cleanups.callback(shared.post, "/key/delete", {"keys": [key]}) + register_mcp(scenario, local, "open" + uuid.uuid4().hex[:8], allow_all_keys=True) + listed, called = official_client_outcomes(shared, key, "/mcp", f"{foreign_alias}-add", ADD) + assert set(listed.tools) == { + f"{foreign_alias}-add", f"{foreign_alias}-multiply", f"{foreign_alias}-fail" + }, listed.tools + assert called.ok and called.text == str(ADD["a"] + ADD["b"]), called + assert [call["body"]["params"]["name"] for call in _peer_add_calls(foreign)] == ["add"] + assert _peer_add_calls(local) == () + deleted: Final = shared.request("GET", "/key/info", params={"key": key}) + assert deleted.status_code == 200, deleted.text + assert deleted.json()["info"]["status"] == "deleted", deleted.text + + Bridge = Literal["chat", "responses", "messages"] BRIDGES: Final[tuple[Bridge, ...]] = ("chat", "responses") Client = Literal["sync", "async"] diff --git a/tests/integration/mcp/test_mcp_toolsets.py b/tests/integration/mcp/test_mcp_toolsets.py index b97fa6e72cb..7ff512a699b 100644 --- a/tests/integration/mcp/test_mcp_toolsets.py +++ b/tests/integration/mcp/test_mcp_toolsets.py @@ -1,5 +1,6 @@ import re import secrets +import signal import textwrap import uuid from collections.abc import Iterator @@ -10,6 +11,7 @@ from typing import Final import httpx import pytest import yaml +from pydantic import TypeAdapter from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value from integration._support.mcp import ( INITIALIZE, @@ -24,7 +26,7 @@ from integration._support.mcp import ( tool_calls, ) from integration._support.mcp_grants import create_toolset -from integration._support.process import owned_proxy +from integration._support.process import owned_proxy, owned_proxy_process from litellm.models.user import LiteLLM_UserTable from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth @@ -443,9 +445,11 @@ def test_require_key_mcp_access_defined_stops_key_inheritance_but_not_the_dashbo assert _listed_toolset_ids(strict, inheriting) == () assert _route_tools(strict, inheriting, granted_name).status == 403 assert _listed_toolset_ids(strict, own) == (granted_id,) - assert _route_tools(strict, own, granted_name).tools == (f"{alias}-add",) + own_tools: Final = _route_tools(strict, own, granted_name) + assert own_tools.tools == (f"{alias}-add",), (own_tools.status, own_tools.raw) assert _listed_toolset_ids(strict, member) == (granted_id,) - assert _route_tools(strict, member, granted_name).tools == (f"{alias}-add",) + member_tools: Final = _route_tools(strict, member, granted_name) + assert member_tools.tools == (f"{alias}-add",), (member_tools.status, member_tools.raw) peer.drain() called: Final = _route_call(strict, member, granted_name, f"{alias}-add") assert called.ok and called.text == "9", called.raw @@ -515,6 +519,24 @@ def test_a_member_of_two_teams_sees_the_union_and_each_route_stays_narrowed_to_i assert tool_calls(peer.drain()) == () +def test_owned_proxy_loss_retains_exit_status_and_first_http_failure(gateway: Gateway, tmp_path: Path) -> None: + with pytest.raises(httpx.TransportError) as failure: # noqa: PT012 # HTTP failure must propagate through proxy cleanup + with owned_proxy_process(gateway, tmp_path, {}) as owned: + response: Final = owned.gateway.client.get("/health/readiness") + assert response.status_code == 200, response.text + owned.process.kill() + owned.process.wait(timeout=3) + owned.gateway.client.get("/health/readiness") + record: Final = TypeAdapter(dict[str, int | float | str | None]).validate_json( + owned.log.with_suffix(".exit.json").read_text() + ) + assert record["root_pid"] == owned.process.pid + assert record["exit_before_stop"] == -signal.SIGKILL + assert record["exit_after_stop"] == -signal.SIGKILL + assert record["exception_before_stop"] == failure.type.__name__ + assert record["exception_at_record"] == failure.type.__name__ + + _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" _UNLISTED: Final = "" diff --git a/tests/integration/observability/test_guardrail_timeout_all_providers.py b/tests/integration/observability/test_guardrail_timeout_all_providers.py index 79051c6ec22..1f33bd19393 100644 --- a/tests/integration/observability/test_guardrail_timeout_all_providers.py +++ b/tests/integration/observability/test_guardrail_timeout_all_providers.py @@ -17,8 +17,10 @@ import threading import time from collections.abc import Iterator, Mapping from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack from dataclasses import dataclass, field from functools import partial +from http.client import HTTPResponse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from types import MappingProxyType @@ -211,8 +213,10 @@ class Sink: class Server(ThreadingHTTPServer): allow_reuse_address = True daemon_threads = True + request_queue_size: Final = socket.SOMAXCONN self.server = Server(("127.0.0.1", self.port), Handler) + self.port = self.server.server_port self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) self.thread.start() @@ -339,10 +343,7 @@ class Rig: @pytest.fixture(scope="module") def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: root: Final = tmp_path_factory.mktemp("guardrail-timeout") - with socket.socket() as reserve: - reserve.bind(("127.0.0.1", 0)) - port: Final = reserve.getsockname()[1] - sink: Final = Sink(port) + sink: Final = Sink(0) sink.start() with gateway_from_environment() as gateway, wire_server(_provider) as provider: config: Final = _rig_config(sink.url, root) @@ -446,3 +447,42 @@ def test_unset_timeout_waits_for_sink_response(rig: Rig, outcomes: Mapping[str, ) assert isinstance(outcome.response, httpx.Response), f"control-generic: client gave up: {outcome.response!r}" assert outcome.response.status_code in (200, 400, 500), outcome.response.text + + +def _connect_sink(port: int, _index: int) -> socket.socket | OSError: + try: + return socket.create_connection(("127.0.0.1", port), timeout=1) + except OSError as error: + return error + + +def _read_sink_probe(stream: socket.socket) -> tuple[int, bytes]: + stream.sendall(b"GET /token/capacity-probe/ HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + with HTTPResponse(stream) as response: + response.begin() + return response.status, response.read() + + +def test_owned_sink_records_every_connection_from_a_concurrent_probe_burst(rig: Rig) -> None: + server: Final = rig.sink.server + original_thread: Final = rig.sink.thread + assert server is not None and original_thread is not None + connection_count: Final = len(PROVIDERS) + 1 + server.shutdown() + original_thread.join(timeout=5) + try: + with ThreadPoolExecutor(max_workers=connection_count) as pool: + opened: Final = tuple(pool.map(partial(_connect_sink, rig.sink.port), range(connection_count))) + finally: + resumed: Final = threading.Thread(target=server.serve_forever, daemon=True) + rig.sink.thread = resumed + resumed.start() + with ExitStack() as clients: + connected: Final = tuple(stream for stream in opened if isinstance(stream, socket.socket)) + for stream in connected: + clients.enter_context(stream) + assert len(connected) == connection_count, tuple(type(value).__name__ for value in opened) + with ThreadPoolExecutor(max_workers=connection_count) as pool: + replies: Final = tuple(pool.map(_read_sink_probe, connected)) + assert replies == ((200, TOKEN_REPLY),) * connection_count + assert len(rig.sink.calls_for("capacity-probe")) == connection_count diff --git a/tests/integration/observability/test_passthrough_upstream_error_chaos.py b/tests/integration/observability/test_passthrough_upstream_error_chaos.py index 3dfdb630009..568a0990c56 100644 --- a/tests/integration/observability/test_passthrough_upstream_error_chaos.py +++ b/tests/integration/observability/test_passthrough_upstream_error_chaos.py @@ -2,6 +2,8 @@ import asyncio import json import re import signal +import threading +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import Final @@ -41,6 +43,22 @@ def _chaos_reply(request: Request) -> Reply: return Reply(status=404, body=_NOT_FOUND_BODY) +def _held_reply(release: threading.Event) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert release.wait(timeout=120), "Held passthrough peer was never released" + return _chaos_reply(request) + + return respond + + +def _held_connections(pid: int, port: int) -> int: + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + def _error_information(call_id: str) -> dict[str, JsonValue]: rows: Final = eventually( lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)), @@ -133,7 +151,8 @@ async def test_passthrough_upstream_outage_mid_burst_still_logs_errors_once(gate async def test_passthrough_worker_sigkill_leaves_sibling_serving_and_logging(gateway: Gateway, tmp_path: Path) -> None: config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) path: Final = tmp_path / "chaos-kill.yaml" - with wire_server(_chaos_reply) as wire: + release: Final = threading.Event() + with wire_server(_held_reply(release)) as wire: config["environment_variables"] = {"GEMINI_API_BASE": wire.url, "GEMINI_API_KEY": "scripted"} path.write_text(yaml.safe_dump(config)) with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned: @@ -143,17 +162,24 @@ async def test_passthrough_worker_sigkill_leaves_sibling_serving_and_logging(gat lambda pids: len(pids) == 2, seconds=30, ) - burst: Final = asyncio.create_task( - _fire_burst(str(candidate.client.base_url), candidate.key, 20, tolerate_transport_errors=True) - ) - await asyncio.to_thread(eventually, lambda: wire.received.qsize(), lambda size: size >= 5, 30) - victim: Final = psutil.Process(workers[0]) - victim.suspend() - victim_ports: Final = frozenset( - connection.raddr.port for connection in victim.net_connections(kind="tcp") if connection.raddr - ) - victim.send_signal(signal.SIGKILL) - served: Final = await burst + async with asyncio.TaskGroup() as tasks: + burst: Final = tasks.create_task( + _fire_burst(str(candidate.client.base_url), candidate.key, 20, tolerate_transport_errors=True) + ) + try: + await asyncio.to_thread(eventually, lambda: wire.received.qsize(), lambda size: size >= 20, 30) + peer_port: Final = int(wire.url.rsplit(":", 1)[1]) + held_by: Final = {pid: _held_connections(pid, peer_port) for pid in workers} + assert sum(held_by.values()) == 20 and min(held_by.values()) > 0, held_by + victim: Final = psutil.Process(max(workers, key=held_by.__getitem__)) + victim.suspend() + victim_ports: Final = frozenset( + connection.raddr.port for connection in victim.net_connections(kind="tcp") if connection.raddr + ) + victim.send_signal(signal.SIGKILL) + finally: + release.set() + served: Final = await burst for item in served: assert item.response.status_code in (200, 404, 500, 502), item.response.status_code follow_up: Final = candidate.request( diff --git a/tests/integration/observability/test_straiker_v3_platform.py b/tests/integration/observability/test_straiker_v3_platform.py index 5007b1a2ee6..45ae562f5f7 100644 --- a/tests/integration/observability/test_straiker_v3_platform.py +++ b/tests/integration/observability/test_straiker_v3_platform.py @@ -1083,6 +1083,31 @@ def _uvicorn_workers(parent: psutil.Process, *, exclude: int = 0) -> tuple[psuti return tuple(c for c in parent.children() if _is_live_worker(c, exclude)) +def test_an_unreaped_worker_does_not_hide_the_surviving_proxy_worker(rig: Rig) -> None: + parent: Final = psutil.Process(rig.owned.process.pid) + workers: Final = eventually(lambda: _uvicorn_workers(parent), lambda children: len(children) == 2) + victim: Final = workers[0] + survivor: Final = workers[1] + marker: Final = rig.marker() + os.kill(parent.pid, signal.SIGSTOP) + try: + os.kill(victim.pid, signal.SIGKILL) + assert eventually(victim.status, lambda status: status == psutil.STATUS_ZOMBIE) == psutil.STATUS_ZOMBIE + assert tuple(child.pid for child in _uvicorn_workers(parent)) == (survivor.pid,) + assert _uvicorn_workers(parent, exclude=survivor.pid) == () + with httpx.Client(base_url=rig._base(), timeout=15, trust_env=False) as client: + response: Final = client.post( + "/v1/chat/completions", + json={"model": rig.chat_model, "messages": _messages(marker)}, + headers={"Authorization": f"Bearer {rig.proxy.key}"}, + ) + assert response.status_code == 200, response.text + assert len(_v3_request_calls(rig, marker)) == 1 + finally: + os.kill(parent.pid, signal.SIGCONT) + assert len(eventually(lambda: _uvicorn_workers(parent, exclude=victim.pid), lambda children: len(children) == 2)) == 2 + + def test_burst_survives_one_worker_kill(rig: Rig) -> None: parent: Final = psutil.Process(rig.owned.process.pid) workers: Final = eventually(lambda: _uvicorn_workers(parent), lambda c: len(c) >= 2) diff --git a/tests/integration/providers/_gemini_tts_voice_support.py b/tests/integration/providers/_gemini_tts_voice_support.py index 7c453a63ef5..5c105cfda78 100644 --- a/tests/integration/providers/_gemini_tts_voice_support.py +++ b/tests/integration/providers/_gemini_tts_voice_support.py @@ -200,6 +200,7 @@ def anthropic_client(gateway: Gateway) -> anthropic.Anthropic: return anthropic.Anthropic( base_url=str(gateway.client.base_url), api_key=gateway.key, + auth_token=gateway.key, max_retries=0, http_client=httpx.Client(trust_env=False, timeout=60), ) diff --git a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py index 107839ed320..9bf526bf3d1 100644 --- a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py +++ b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py @@ -752,11 +752,15 @@ def _async_openai_client(rig: _Rig) -> openai.AsyncOpenAI: def _anthropic_client(rig: _Rig) -> anthropic.Anthropic: - return anthropic.Anthropic(base_url=_proxy_url(rig.gateway), api_key=rig.gateway.key, max_retries=0) + return anthropic.Anthropic( + base_url=_proxy_url(rig.gateway), api_key=rig.gateway.key, auth_token=rig.gateway.key, max_retries=0 + ) def _async_anthropic_client(rig: _Rig) -> anthropic.AsyncAnthropic: - return anthropic.AsyncAnthropic(base_url=_proxy_url(rig.gateway), api_key=rig.gateway.key, max_retries=0) + return anthropic.AsyncAnthropic( + base_url=_proxy_url(rig.gateway), api_key=rig.gateway.key, auth_token=rig.gateway.key, max_retries=0 + ) def _proxy_url(gateway: Gateway) -> str: @@ -812,18 +816,23 @@ _ANTHROPIC_SDK_TARGETS: Final = (("kimi-us", _KIMI_US), ("arn-yaml-flagged", _YA @pytest.mark.timeout(600) @pytest.mark.parametrize(("name", "model_id"), _ANTHROPIC_SDK_TARGETS, ids=("k03-kimi", "k03-flagged-arn")) def test_k03_anthropic_sdk_sync_messages_reaches_the_model_without_a_cache_point( - rig: _Rig, name: str, model_id: str + rig: _Rig, name: str, model_id: str, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setenv("ANTHROPIC_AUTH_TOKEN", "synthetic-unwanted-ambient-token") rig.wire.drain() - message: Final = _anthropic_client(rig).messages.create( - model=name, - max_tokens=16, - system=[{"type": "text", "text": _SYSTEM_TEXT, "cache_control": {"type": "ephemeral"}}], - messages=[ - {"role": "user", "content": [{"type": "text", "text": _prompt(), "cache_control": {"type": "ephemeral"}}]} - ], - extra_body=_EXTRA, - ) + with _anthropic_client(rig) as client: + message: Final = client.messages.create( + model=name, + max_tokens=16, + system=[{"type": "text", "text": _SYSTEM_TEXT, "cache_control": {"type": "ephemeral"}}], + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": _prompt(), "cache_control": {"type": "ephemeral"}}], + } + ], + extra_body=_EXTRA, + ) assert "".join(block.text for block in message.content if block.type == "text") == _ANSWER, message received: Final = _only_received(rig.wire) assert received.model == model_id and not received.streaming, received @@ -834,19 +843,24 @@ def test_k03_anthropic_sdk_sync_messages_reaches_the_model_without_a_cache_point @pytest.mark.timeout(600) @pytest.mark.parametrize(("name", "model_id"), _ANTHROPIC_SDK_TARGETS, ids=("k04-kimi", "k04-flagged-arn")) async def test_k04_anthropic_sdk_async_stream_reaches_the_model_without_a_cache_point( - rig: _Rig, name: str, model_id: str + rig: _Rig, name: str, model_id: str, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setenv("ANTHROPIC_AUTH_TOKEN", "synthetic-unwanted-ambient-token") rig.wire.drain() - async with _async_anthropic_client(rig).messages.stream( - model=name, - max_tokens=16, - system=[{"type": "text", "text": _SYSTEM_TEXT, "cache_control": {"type": "ephemeral"}}], - messages=[ - {"role": "user", "content": [{"type": "text", "text": _prompt(), "cache_control": {"type": "ephemeral"}}]} - ], - extra_body=_EXTRA, - ) as stream: - final: Final = await stream.get_final_message() + async with _async_anthropic_client(rig) as client: + async with client.messages.stream( + model=name, + max_tokens=16, + system=[{"type": "text", "text": _SYSTEM_TEXT, "cache_control": {"type": "ephemeral"}}], + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": _prompt(), "cache_control": {"type": "ephemeral"}}], + } + ], + extra_body=_EXTRA, + ) as stream: + final: Final = await stream.get_final_message() assert "".join(block.text for block in final.content if block.type == "text") == _ANSWER, final received: Final = _only_received(rig.wire) assert received.model == model_id and received.streaming, received diff --git a/tests/integration/providers/test_cache_control_tool_call_marks_owned_proxy.py b/tests/integration/providers/test_cache_control_tool_call_marks_owned_proxy.py index 0a941cff4b7..c945eaff38e 100644 --- a/tests/integration/providers/test_cache_control_tool_call_marks_owned_proxy.py +++ b/tests/integration/providers/test_cache_control_tool_call_marks_owned_proxy.py @@ -201,15 +201,18 @@ async def test_worker_sigkill_mid_burst_leaves_the_sibling_serving_capped_reques seconds=30, ) owned_url: Final = str(owned.gateway.client.base_url) - burst: Final = asyncio.create_task(_fire(owned_url, owned.gateway.key, tolerate_transport_errors=True)) - try: - await asyncio.to_thread(eventually, lambda: wire.received.qsize(), lambda size: size >= _BURST, 90) - held_by: Final = MappingProxyType({pid: _held_upstream_connections(pid, wire.url) for pid in workers}) - victim: Final = max(workers, key=held_by.__getitem__) - os.kill(victim, signal.SIGKILL) - finally: - release.set() - served: Final = await burst + async with asyncio.TaskGroup() as tasks: + burst: Final = tasks.create_task(_fire(owned_url, owned.gateway.key, tolerate_transport_errors=True)) + try: + await asyncio.to_thread(eventually, lambda: wire.received.qsize(), lambda size: size >= _BURST, 90) + held_by: Final = MappingProxyType( + {pid: _held_upstream_connections(pid, wire.url) for pid in workers} + ) + victim: Final = max(workers, key=held_by.__getitem__) + os.kill(victim, signal.SIGKILL) + finally: + release.set() + served: Final = await burst during: Final = wire.drain() after: Final = await _fire(owned_url, owned.gateway.key) after_received: Final = wire.drain() diff --git a/tests/integration/providers/test_device_code_login_guard_boot.py b/tests/integration/providers/test_device_code_login_guard_boot.py index 7bf15ac2e1c..ecec6402aee 100644 --- a/tests/integration/providers/test_device_code_login_guard_boot.py +++ b/tests/integration/providers/test_device_code_login_guard_boot.py @@ -370,8 +370,9 @@ async def _held_on_every_worker( control_port: int, held_markers: SimpleQueue[str], bursts: _Bursts, + tasks: asyncio.TaskGroup, ) -> tuple[_Bursts, Mapping[int, int]]: - started: Final = (*bursts, asyncio.create_task(_burst(_base_url(owned), key, (_CONTROL,) * _HELD_PER_BURST))) + started: Final = (*bursts, tasks.create_task(_burst(_base_url(owned), key, (_CONTROL,) * _HELD_PER_BURST))) expected: Final = _HELD_PER_BURST * len(started) await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == expected, 60) held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, control_port) for pid in workers}) @@ -379,7 +380,7 @@ async def _held_on_every_worker( if min(held_by.values()) >= _HELD_PER_WORKER: return started, held_by assert len(started) < _MOST_BURSTS, held_by - return await _held_on_every_worker(owned, key, workers, control_port, held_markers, started) + return await _held_on_every_worker(owned, key, workers, control_port, held_markers, started, tasks) async def test_worker_sigkill_mid_burst_leaves_the_sibling_refusing_logins_and_serving_the_control( @@ -412,13 +413,18 @@ async def test_worker_sigkill_mid_burst_leaves_the_sibling_refusing_logins_and_s control_port: Final = int(control_api.url.rsplit(":", 1)[1]) with owned.gateway.scenario() as scenario: key: Final = scenario.key() - bursts, held_by = await _held_on_every_worker(owned, key, workers, control_port, held_markers, ()) - victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) - victim: Final = psutil.Process(victim_pid) - victim.suspend() - victim.send_signal(signal.SIGKILL) - release.set() - answers: Final = await asyncio.gather(*bursts) + async with asyncio.TaskGroup() as tasks: + try: + bursts, held_by = await _held_on_every_worker( + owned, key, workers, control_port, held_markers, (), tasks + ) + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + victim: Final = psutil.Process(victim_pid) + victim.suspend() + victim.send_signal(signal.SIGKILL) + finally: + release.set() + answers: Final = await asyncio.gather(*bursts) served: Final = tuple(item for item in itertools.chain.from_iterable(answers) if item is not None) assert len(served) == held_by[survivor_pid], (held_by, len(served)) for item in served: diff --git a/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py b/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py index 3608e743506..8a977427741 100644 --- a/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py +++ b/tests/integration/providers/test_gemini_tts_voice_mapping_owned_proxy.py @@ -287,7 +287,7 @@ def _worker_pids(proxy: OwnedProxy) -> tuple[int, ...]: ) -def test_worker_sigkill_mid_speech_burst_leaves_the_sibling_serving(gateway: Gateway, tmp_path: Path) -> None: +async def test_worker_sigkill_mid_speech_burst_leaves_the_sibling_serving(gateway: Gateway, tmp_path: Path) -> None: release: Final = threading.Event() with wire_server(_held_peer(release)) as wire: config: Final = health_config(tmp_path) @@ -296,21 +296,22 @@ def test_worker_sigkill_mid_speech_burst_leaves_the_sibling_serving(gateway: Gat url: Final = str(proxy.gateway.client.base_url) with proxy.gateway.scenario() as scenario: model: Final = speech_deployment(scenario, wire.url) - loop: Final = asyncio.new_event_loop() - burst: Final = loop.run_in_executor( - None, lambda: asyncio.run(_fire(url, proxy.gateway.key, model, tolerate_transport_errors=True)) - ) - try: - eventually(lambda: wire.received.qsize(), lambda size: size >= BURST, 90) - held_by: Final = MappingProxyType({pid: _held_connections(pid, wire.url) for pid in workers}) - victim: Final = max(workers, key=held_by.__getitem__) - os.kill(victim, signal.SIGKILL) - finally: - release.set() - served: Final = loop.run_until_complete(burst) - loop.close() + async with asyncio.TaskGroup() as tasks: + burst: Final = tasks.create_task( + _fire(url, proxy.gateway.key, model, tolerate_transport_errors=True) + ) + try: + await asyncio.to_thread( + eventually, lambda: wire.received.qsize(), lambda size: size >= BURST, 90 + ) + held_by: Final = MappingProxyType({pid: _held_connections(pid, wire.url) for pid in workers}) + victim: Final = max(workers, key=held_by.__getitem__) + os.kill(victim, signal.SIGKILL) + finally: + release.set() + served: Final = await burst during: Final = wire.drain() - after: Final = asyncio.run(_fire(url, proxy.gateway.key, model)) + after: Final = await _fire(url, proxy.gateway.key, model) after_received: Final = wire.drain() assert sum(held_by.values()) == BURST, held_by assert held_by[victim] > 0, held_by @@ -321,7 +322,7 @@ def test_worker_sigkill_mid_speech_burst_leaves_the_sibling_serving(gateway: Gat _assert_served(after, after_received) -def test_proxy_restart_mid_speech_burst_serves_the_next_burst_after_the_reboot( +async def test_proxy_restart_mid_speech_burst_serves_the_next_burst_after_the_reboot( gateway: Gateway, tmp_path: Path ) -> None: release: Final = threading.Event() @@ -330,20 +331,17 @@ def test_proxy_restart_mid_speech_burst_serves_the_next_burst_after_the_reboot( model: Final = speech_deployment(scenario, wire.url) with owned_proxy_process(gateway, tmp_path, _overrides(wire.url), config=config, workers=2) as first: url: Final = str(first.gateway.client.base_url) - loop: Final = asyncio.new_event_loop() - burst: Final = loop.run_in_executor( - None, lambda: asyncio.run(_fire(url, first.gateway.key, model, tolerate_transport_errors=True)) - ) - try: - eventually(lambda: wire.received.qsize(), lambda size: size >= BURST, 90) - first.process.terminate() - finally: - release.set() - served: Final = loop.run_until_complete(burst) - loop.close() + async with asyncio.TaskGroup() as tasks: + burst: Final = tasks.create_task(_fire(url, first.gateway.key, model, tolerate_transport_errors=True)) + try: + await asyncio.to_thread(eventually, lambda: wire.received.qsize(), lambda size: size >= BURST, 90) + first.process.terminate() + finally: + release.set() + served: Final = await burst during: Final = wire.drain() with owned_proxy_process(gateway, tmp_path, _overrides(wire.url), config=config, workers=2) as second: - after: Final = asyncio.run(_fire(str(second.gateway.client.base_url), second.gateway.key, model)) + after: Final = await _fire(str(second.gateway.client.base_url), second.gateway.key, model) after_received: Final = wire.drain() completed: Final = tuple(item for item in served if item.status == 200) assert all(len(requests) == 1 for requests in _by_text(during).values()) diff --git a/tests/integration/run.py b/tests/integration/run.py index aa5ead066f2..f89bd15adf9 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -58,6 +58,9 @@ def main() -> int: parser.add_argument("--seed", type=int, default=int(os.environ.get("INTEGRATION_SEED", "4106601"))) parser.add_argument("--order-seed", type=int, default=int(os.environ.get("INTEGRATION_ORDER_SEED", "0"))) parser.add_argument("--workers", type=int, default=int(os.environ.get("INTEGRATION_WORKERS", "1"))) + parser.add_argument("--shards", type=int, default=int(os.environ.get("INTEGRATION_CASE_SHARDS", "1"))) + parser.add_argument("--shard-index", type=int, default=int(os.environ.get("INTEGRATION_CASE_SHARD_INDEX", "0"))) + parser.add_argument("--timings", type=Path) parser.add_argument("--list", action="store_true", help="print the group's test files and exit") parser.add_argument("files", nargs="*", help="run only these files, or pytest node ids inside them, of the group") options: Final = parser.parse_intermixed_args() @@ -69,7 +72,7 @@ def main() -> int: if str(path.relative_to(root)) not in GITHUB_FILES ) if options.list: - print("\n".join(group_files)) + sys.stdout.write("\n".join(group_files) + "\n") return 0 selection: Final = select(tuple(options.files), group_files) if selection.foreign: @@ -83,6 +86,8 @@ def main() -> int: "PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), str(root / "tests/e2e"))), "INTEGRATION_RESULTS_DIR": str(output), "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "LITELLM_MODE": "PRODUCTION", + "PYTHON_DOTENV_DISABLED": "1", } result: Final = subprocess.call( [ @@ -102,10 +107,15 @@ def main() -> int: "--tb=short", f"--hypothesis-seed={options.seed}", f"--integration-order-seed={options.order_seed}", + f"--integration-shard-count={options.shards}", + f"--integration-shard-index={options.shard_index}", + *(("--integration-timings", str(options.timings)) if options.timings is not None else ()), f"--junitxml={output / 'junit.xml'}", "-o", "junit_family=xunit1", *(("-n", str(options.workers)) if options.workers > 1 else ()), + *(("--dist=worksteal",) if options.group == "security" and options.workers > 1 else ()), + *(("--dist=loadgroup",) if options.group == "mcp" and options.workers > 1 else ()), ], cwd=root, env=environment, @@ -113,7 +123,7 @@ def main() -> int: if result != 0: return result evidence: Final = json.loads((output / "execution.json").read_text()) - empty: Final = uncollected(selection.nodes, frozenset(evidence["collected"])) + empty: Final = uncollected(selection.nodes, frozenset(evidence["inventory"])) if empty: sys.stderr.write(f"Selected integration files collected zero tests: {', '.join(empty)}\n") return 1 diff --git a/tests/integration/sdk/test_generic_api_callback.py b/tests/integration/sdk/test_generic_api_callback.py new file mode 100644 index 00000000000..08862a77ac3 --- /dev/null +++ b/tests/integration/sdk/test_generic_api_callback.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import asyncio +import threading +from collections.abc import Iterator, Sequence +from typing import Final, TypedDict + +import httpx +import pytest +from pydantic import TypeAdapter +from typing_extensions import ReadOnly + +from litellm.integrations.generic_api.generic_api_callback import LOG_FORMAT_TYPES, GenericAPILogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from tests.integration._support.wire import Reply as WireReply, Request as WireRequest, wire_server + + +class _QueuedLog(TypedDict): + id: ReadOnly[str] + + +_QUEUED_LOG: Final = TypeAdapter(_QueuedLog) +_LOG_BATCH: Final = TypeAdapter(tuple[_QueuedLog, ...]) + + +def _batch_events(body: bytes, log_format: LOG_FORMAT_TYPES) -> tuple[_QueuedLog, ...]: + match log_format: + case "json_array": + return _LOG_BATCH.validate_json(body) + case "ndjson": + return tuple(_QUEUED_LOG.validate_json(line) for line in body.splitlines()) + case "single": + return (_QUEUED_LOG.validate_json(body),) + + +def _received_log_ids(requests: Sequence[WireRequest], log_format: LOG_FORMAT_TYPES) -> Iterator[str]: + for request in requests: + for event in _batch_events(request.body, log_format): + yield event["id"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_format", ("json_array", "ndjson", "single")) +async def test_generic_api_keeps_failure_arriving_during_success_batch(log_format: LOG_FORMAT_TYPES) -> None: + loop: Final = asyncio.get_running_loop() + first_arrived: Final = asyncio.Event() + release: Final = threading.Event() + + def respond(request: WireRequest) -> WireReply: + events: Final = _batch_events(request.body, log_format) + if any(event["id"] == "before" for event in events): + loop.call_soon_threadsafe(first_arrived.set) + assert release.wait(5), "The first HTTP batch was never released" + return WireReply() + + with wire_server(respond) as wire: + handler: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport(trust_env=False)) + before_tasks: Final = frozenset(asyncio.all_tasks()) + logger: Final = GenericAPILogger( + endpoint=wire.url, + async_httpx_client=handler, + log_format=log_format, + batch_size=1000, + flush_interval=3600, + ) + owned_tasks: Final = frozenset(asyncio.all_tasks()) - before_tasks + try: + async with asyncio.TaskGroup() as flushes: + await logger.async_log_success_event({"standard_logging_object": {"id": "before"}}, None, None, None) + first_flush: Final = flushes.create_task(logger.flush_queue()) + try: + await first_arrived.wait() + await logger.async_log_failure_event({"standard_logging_object": {"id": "during"}}, None, None, None) + finally: + release.set() + await first_flush + await logger.flush_queue() + assert tuple(_received_log_ids(wire.drain(), log_format)) == ("before", "during") + finally: + release.set() + for task in owned_tasks: + task.cancel() + await asyncio.gather(*owned_tasks, return_exceptions=True) + await handler.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_format", ("json_array", "ndjson", "single")) +async def test_generic_api_concurrent_flushes_deliver_each_event_once(log_format: LOG_FORMAT_TYPES) -> None: + with wire_server(lambda request: WireReply()) as wire: + handler: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport(trust_env=False)) + before_tasks: Final = frozenset(asyncio.all_tasks()) + logger: Final = GenericAPILogger( + endpoint=wire.url, + async_httpx_client=handler, + log_format=log_format, + batch_size=1000, + flush_interval=3600, + ) + owned_tasks: Final = frozenset(asyncio.all_tasks()) - before_tasks + try: + await logger.async_log_success_event({"standard_logging_object": {"id": "once"}}, None, None, None) + await asyncio.gather(logger.flush_queue(), logger.async_send_batch()) + assert tuple(_received_log_ids(wire.drain(), log_format)) == ("once",) + finally: + for task in owned_tasks: + task.cancel() + await asyncio.gather(*owned_tasks, return_exceptions=True) + await handler.close() diff --git a/tests/integration/security/_sinks.py b/tests/integration/security/_sinks.py index 1d2bc236239..906384e8760 100644 --- a/tests/integration/security/_sinks.py +++ b/tests/integration/security/_sinks.py @@ -232,6 +232,10 @@ def canary_rig( **{name: "127.0.0.1,localhost" for name in ("NO_PROXY", "no_proxy")}, "OPENAI_BASE_URL": provider.url + "/v1", "OPENAI_API_BASE": provider.url + "/v1", + "ANTHROPIC_API_BASE": provider.url, + "ANTHROPIC_BASE_URL": provider.url, + "ANTHROPIC_API_KEY": b1.value, + "ANTHROPIC_AUTH_TOKEN": "", "GENERIC_LOGGER_ENDPOINT": sink.url, "GENERIC_LOGGER_HEADERS": f"Authorization=Bearer {token}", **(environment or {}), diff --git a/tests/integration/security/_sweeps.py b/tests/integration/security/_sweeps.py index 5b9fa9a613f..e103463cd1d 100644 --- a/tests/integration/security/_sweeps.py +++ b/tests/integration/security/_sweeps.py @@ -414,14 +414,17 @@ def sweep_routes( ) who: Final = callers if callers is not None else {"admin": gateway.key} base_url: Final = str(gateway.client.base_url) + verification: Final = httpx.create_ssl_context(trust_env=False) if httpx.URL(base_url).scheme == "http" else True def call(route: str, label: str, key: str, path: str) -> _RouteCall: location: Final = f"GET {path} as {label}" try: - with httpx.Client(base_url=base_url, timeout=_ROUTE_TIMEOUT, trust_env=False) as client: + with httpx.Client( + base_url=base_url, timeout=_ROUTE_TIMEOUT, trust_env=False, verify=verification + ) as client: response = client.get(path, headers={"Authorization": f"Bearer {key}"}) except httpx.HTTPError as error: - return _RouteCall((), (), None, f"{location}: {type(error).__name__}", location) + return _RouteCall((), (), None, f"{location}: {type(error).__name__}: {error}", location) headers = "\n".join(f"{name}: {value}" for name, value in response.headers.items()) found = _hits( "S2", f"{location} -> {response.status_code}", response.content + b"\n" + headers.encode(), canaries @@ -476,7 +479,7 @@ def sweep_routes( def record_route_sweep(routes: RouteSweep, node: str) -> None: - """Append the route sweep's errors and unfilled routes to the results directory, when set.""" + """Append a passed or failed S2 report to its results ledger, when set.""" destination: Final = os.environ.get("INTEGRATION_RESULTS_DIR") if not destination: return @@ -490,7 +493,12 @@ def record_route_sweep(routes: RouteSweep, node: str) -> None: "not_found": routes.not_found, "rejected": routes.rejected, } - with (Path(destination) / "security-route-sweep.jsonl").open("a") as report: + filename: Final = ( + "security-route-sweep-failures.jsonl" + if routes.unreachable or routes.rejected or routes.not_found + else "security-route-sweep.jsonl" + ) + with (Path(destination) / filename).open("a") as report: report.write(json.dumps(entry) + "\n") @@ -584,6 +592,8 @@ def sweep_all( """ redis: Final = sweep_redis(canaries) routes: Final = sweep_routes(gateway, canaries, ids, callers=callers, since=since) + if routes.unreachable or routes.rejected or routes.not_found: + record_route_sweep(routes, os.environ.get("PYTEST_CURRENT_TEST", "unknown")) assert not routes.unreachable, f"GET routes returned no response, so S2 did not check them: {routes.unreachable}" assert not routes.rejected, ( f"Scoped list routes rejected the scenario's query, so S2 saw no rows: {routes.rejected}" diff --git a/tests/integration/security/conftest.py b/tests/integration/security/conftest.py new file mode 100644 index 00000000000..b9f33c71f63 --- /dev/null +++ b/tests/integration/security/conftest.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.workers import worker_services + + +@pytest.fixture(scope="session", autouse=True) +def isolated_security_worker(worker_id: str, tmp_path_factory: pytest.TempPathFactory) -> Iterator[None]: + if worker_id == "master": + yield + return + directory: Final = tmp_path_factory.mktemp("security-worker") + results: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory))) / worker_id + results.mkdir(parents=True, exist_ok=True) + with pytest.MonkeyPatch.context() as environment: + environment.setenv("INTEGRATION_RESULTS_DIR", str(results)) + with worker_services(directory) as services: + for name, value in services.items(): + environment.setenv(name, value) + yield diff --git a/tests/integration/security/test_sweep_sensitivity.py b/tests/integration/security/test_sweep_sensitivity.py index 2a1340dc358..5c38069a796 100644 --- a/tests/integration/security/test_sweep_sensitivity.py +++ b/tests/integration/security/test_sweep_sensitivity.py @@ -9,17 +9,25 @@ from __future__ import annotations import base64 import gzip +import json import uuid +from collections import Counter from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path from typing import Final +import httpx import pytest -from integration._support.client import eventually, string_value +from redis import Redis +from integration._support.client import Gateway, eventually, string_value from integration.security._canary import DECODE_BUDGET_BYTES, MARKER, SLOTS, DecodeBudgetExceeded, canary, find_canary -from integration.security._sinks import CONFIG_MODEL, GENERIC_SINK, Rig, canary_rig, settle, team_caller -from integration._support.wire import Request +from integration.security._sinks import CONFIG_MODEL, GENERIC_SINK, Rig, canary_rig, chat_upstream, settle, team_caller +from integration._support.wire import Reply, Request, wire_server +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.redis_process import owned_redis +from tests.integration._support.tls import server_context, write_self_signed_cert +from tests.integration._support.workers import worker_services from integration.security._sweeps import ( ADMIN_ONLY_ALLOWANCES, ALLOWANCE_SLOT_FAMILIES, @@ -32,6 +40,7 @@ from integration.security._sweeps import ( scoped_queries, sweep_all, sweep_redis, + sweep_routes, sweep_sink, ) @@ -77,6 +86,42 @@ def test_rig_with_an_overridden_master_key_resolves_the_config_deployment(tmp_pa assert overridden.proxy.request("GET", "/model/info").status_code == 200 +def _skills_upstream(request: Request) -> Reply: + skill: Final = { + "id": "owned-skill", + "display_title": "Owned skill", + "source": "custom", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + } + if request.target.startswith("/v1/skills/"): + return Reply(body=json.dumps(skill).encode()) + if request.target.startswith("/v1/skills"): + return Reply(body=json.dumps({"data": [skill], "has_more": False}).encode()) + return chat_upstream(request) + + +def test_skills_reads_use_the_owned_provider_despite_ambient_credentials( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://ambient-provider.invalid") + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://ambient-provider.invalid") + monkeypatch.setenv("ANTHROPIC_API_KEY", "ambient-provider-key") + monkeypatch.setenv("ANTHROPIC_AUTH_TOKEN", "ambient-provider-token") + with canary_rig(tmp_path, upstream=_skills_upstream) as owned: + for path in ("/v1/skills", "/v1/skills/owned-skill"): + response: Final = owned.proxy.request("GET", path) + assert response.status_code == 200, response.text + requests: Final = owned.provider.requests() + assert tuple(request.target.split("?", 1)[0] for request in requests) == ( + "/v1/skills", + "/v1/skills/owned-skill", + ) + assert all(request.headers.get("x-api-key") == owned.canaries["B1"].value for request in requests) + assert all("authorization" not in request.headers for request in requests) + assert owned.egress() == () + + def test_route_allowances_match_only_their_exact_route_and_caller() -> None: routes: Final = get_routes() callers: Final = ("admin", "internal_user", "Admin", "admin ", "") @@ -178,3 +223,89 @@ def test_every_sweep_finds_the_stored_prompt_marker(rig: Rig, request: pytest.Fi listed = rig.proxy.request("GET", route + query) assert request_id in listed.text, f"{route}{query} does not list the scenario's row" assert report.credential_hits() == () + + +def test_security_worker_services_isolate_identical_database_and_cache_keys(tmp_path: Path) -> None: + with ( + worker_services(tmp_path / "first") as first, + worker_services(tmp_path / "second") as second, + ): + for environment, marker in ((first, "first"), (second, "second")): + write_rows( + "CREATE TABLE IF NOT EXISTS isolation_probe (id text PRIMARY KEY, marker text NOT NULL)", + (), + database_url=environment["DATABASE_URL"], + ) + write_rows( + "INSERT INTO isolation_probe VALUES (%s, %s) ON CONFLICT (id) DO UPDATE SET marker=EXCLUDED.marker", + ("same-id", marker), + database_url=environment["DATABASE_URL"], + ) + with Redis(host=environment["REDIS_HOST"], port=int(environment["REDIS_PORT"])) as cache: + cache.set("same-key", marker) + for environment, marker in ((first, "first"), (second, "second")): + assert read_rows( + "SELECT marker FROM isolation_probe WHERE id=%s", + ("same-id",), + database_url=environment["DATABASE_URL"], + ) == [{"marker": marker}] + with Redis( + host=environment["REDIS_HOST"], port=int(environment["REDIS_PORT"]), decode_responses=True + ) as cache: + assert cache.get("same-key") == marker + + +def test_route_sweep_keeps_connections_auth_and_response_cookies_isolated() -> None: + callers: Final = {"admin": "sk-sweep-admin", "internal_user": "sk-sweep-user"} + with wire_server(_response_with_cookie, keep_alive=True) as peer: + with httpx.Client(base_url=peer.url, trust_env=False) as client: + report: Final = sweep_routes(Gateway(client, callers["admin"], peer.url), (), {}, callers=callers) + requests: Final = peer.drain() + requested: Final = tuple(called.split(" ", 1) for called in report.called) + expected: Final = Counter((path, f"Bearer {callers[caller]}") for caller, path in requested) + assert report.called + assert len(requests) == len(report.called) + assert Counter((request.target, request.headers["authorization"]) for request in requests) == expected + assert all("cookie" not in request.headers for request in requests) + assert peer.connections() == len(requests) + assert report.errors == report.unreachable == () + + +def _response_with_cookie(_: Request) -> Reply: + return Reply(headers={"set-cookie": "session=another-caller; Path=/"}) + + +def test_route_sweep_rejects_an_untrusted_https_peer(tmp_path: Path) -> None: + cert, key = write_self_signed_cert(tmp_path) + with wire_server(_response_with_cookie, tls=server_context(cert, key), keep_alive=True) as peer: + with httpx.Client(base_url=peer.url, trust_env=False) as client: + report: Final = sweep_routes(Gateway(client, "sk-sweep-admin", peer.url), (), {}) + assert report.called + assert len(report.unreachable) == len(report.called) + assert peer.drain() == () + + +def test_route_sweep_preserves_truncated_response_errors_separately( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + monkeypatch.setenv("INTEGRATION_RESULTS_DIR", str(tmp_path)) + with ( + owned_redis(tmp_path) as cache, + wire_server(lambda _: Reply(chunks=(b"first", b"missing"), abort_after=1)) as peer, + ): + monkeypatch.setenv("REDIS_HOST", cache.host) + monkeypatch.setenv("REDIS_PORT", str(cache.port)) + with ( + httpx.Client(base_url=peer.url, trust_env=False) as client, + pytest.raises(AssertionError, match="GET routes returned no response"), + ): + sweep_all(Gateway(client, "sk-sweep-admin", peer.url), (), responses=(), sinks={}, ids={}) + received: Final = peer.drain() + + saved: Final = json.loads((tmp_path / "security-route-sweep-failures.jsonl").read_text()) + assert received + assert request.node.nodeid in saved["node"] + assert saved["called"] == len(received) + assert len(saved["unreachable"]) == len(received) + assert all(error.partition("RemoteProtocolError: ")[2] for error in saved["unreachable"]) + assert not (tmp_path / "security-route-sweep.jsonl").exists() diff --git a/tests/integration/spend/test_vertex_image_regional_uplift.py b/tests/integration/spend/test_vertex_image_regional_uplift.py index ae83a6bfb5f..a06c8842a45 100644 --- a/tests/integration/spend/test_vertex_image_regional_uplift.py +++ b/tests/integration/spend/test_vertex_image_regional_uplift.py @@ -7,8 +7,9 @@ import signal import threading import time import uuid -from collections.abc import Callable, Mapping +from collections.abc import AsyncIterator, Callable, Mapping from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager from dataclasses import dataclass from pathlib import Path from queue import SimpleQueue @@ -593,6 +594,20 @@ def _gated(release: threading.Event, held: SimpleQueue[str]) -> Callable[[Reques return respond +@asynccontextmanager +async def _gated_burst( + base_url: str, key: str, call_ids: tuple[str, ...], release: threading.Event +) -> AsyncIterator[asyncio.Task[tuple[_Served, ...]]]: + burst: Final = asyncio.create_task(_async_burst(base_url, key, call_ids)) + try: + yield burst + finally: + release.set() + if not burst.done(): + burst.cancel() + await asyncio.gather(burst, return_exceptions=True) + + @pytest.mark.timeout(420) async def test_worker_sigkill_mid_burst_leaves_the_sibling_billing_the_uplift(gateway: Gateway, tmp_path: Path) -> None: release: Final = threading.Event() @@ -605,37 +620,40 @@ async def test_worker_sigkill_mid_burst_leaves_the_sibling_billing_the_uplift(ga workers, _ = eventually( lambda: _worker_startups(owned.log), lambda found: len(found[0]) == 2 and found[1] == 2, seconds=240 ) - burst: Final = asyncio.create_task(_async_burst(base_url, gateway.key, call_ids)) - await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(call_ids), 120) - held_by: Final = {pid: _open_upstream_connections(pid, wire.url) for pid in workers} - assert sum(held_by.values()) == len(call_ids), held_by - victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) - psutil.Process(victim_pid).send_signal(signal.SIGKILL) - release.set() - served: Final = await burst - succeeded: Final = tuple(item.call_id for item in served if item.status == 200) - _artifact( - tmp_path, - "worker-sigkill.json", - { - "held_by_killed_worker": held_by[victim_pid], - "held_by_surviving_worker": held_by[survivor_pid], - "served": len(served), - "succeeded": len(succeeded), - "transport_errors": len(call_ids) - len(served), - }, - ) - assert len(succeeded) == held_by[survivor_pid], ( - held_by, - [(item.status, item.text[:80]) for item in served], - ) - follow_up_id, follow_up = _generate(owned.gateway, _CONFIG_MODEL) - _assert_images(follow_up, follow_up_id, _REGIONAL_TOKEN_COST) - rows: Final = await asyncio.to_thread(_spend_rows, succeeded, seconds=120) - assert rows == {call_id: pytest.approx(_REGIONAL_TOKEN_COST) for call_id in succeeded} - await asyncio.to_thread( - eventually, lambda: _worker_startups(owned.log), lambda found: len(found[0]) == 3 and found[1] == 3, 240 - ) + async with _gated_burst(base_url, gateway.key, call_ids, release) as burst: + await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(call_ids), 120) + held_by: Final = {pid: _open_upstream_connections(pid, wire.url) for pid in workers} + assert sum(held_by.values()) == len(call_ids), held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + psutil.Process(victim_pid).send_signal(signal.SIGKILL) + release.set() + served: Final = await burst + succeeded: Final = tuple(item.call_id for item in served if item.status == 200) + _artifact( + tmp_path, + "worker-sigkill.json", + { + "held_by_killed_worker": held_by[victim_pid], + "held_by_surviving_worker": held_by[survivor_pid], + "served": len(served), + "succeeded": len(succeeded), + "transport_errors": len(call_ids) - len(served), + }, + ) + assert len(succeeded) == held_by[survivor_pid], ( + held_by, + [(item.status, item.text[:80]) for item in served], + ) + follow_up_id, follow_up = _generate(owned.gateway, _CONFIG_MODEL) + _assert_images(follow_up, follow_up_id, _REGIONAL_TOKEN_COST) + rows: Final = await asyncio.to_thread(_spend_rows, succeeded, seconds=120) + assert rows == {call_id: pytest.approx(_REGIONAL_TOKEN_COST) for call_id in succeeded} + await asyncio.to_thread( + eventually, + lambda: _worker_startups(owned.log), + lambda found: len(found[0]) == 3 and found[1] == 3, + 240, + ) @pytest.mark.timeout(600) @@ -650,12 +668,12 @@ async def test_proxy_restart_mid_burst_bills_each_landed_call_once(gateway: Gate eventually( lambda: _worker_startups(owned.log), lambda found: len(found[0]) == 2 and found[1] == 2, seconds=240 ) - burst: Final = asyncio.create_task(_async_burst(base_url, gateway.key, call_ids)) - await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(call_ids), 120) - owned.process.terminate() - release.set() - served: Final = await burst - await asyncio.to_thread(owned.process.wait, 240) + async with _gated_burst(base_url, gateway.key, call_ids, release) as burst: + await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(call_ids), 120) + owned.process.terminate() + release.set() + served: Final = await burst + await asyncio.to_thread(owned.process.wait, 240) succeeded: Final = tuple(item.call_id for item in served if item.status == 200) with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as rebooted: follow_up_id, follow_up = _generate(rebooted.gateway, _CONFIG_MODEL) @@ -675,3 +693,27 @@ async def test_proxy_restart_mid_burst_bills_each_landed_call_once(gateway: Gate ) assert sorted(landed_ids) == sorted(succeeded), (succeeded, landed_ids) assert all(float(str(row["spend"])) == pytest.approx(_REGIONAL_TOKEN_COST) for row in landed), landed + + +async def test_gated_burst_releases_its_handlers_and_finishes_after_a_precondition_failure() -> None: + release: Final = threading.Event() + held: Final[SimpleQueue[str]] = SimpleQueue() + call_ids: Final = tuple(uuid.uuid4().hex for _ in range(2)) + + def respond(request: Request) -> Reply: + held.put(request.target) + assert release.wait(timeout=5), "The failed burst did not release its handlers" + return Reply() + + with wire_server(respond) as wire: + tasks_before: Final = asyncio.all_tasks() + + async def fail_precondition() -> None: + async with _gated_burst(wire.url, "owned-burst-key", call_ids, release): + await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(call_ids), 5) + raise AssertionError("injected precondition failure") + + with pytest.raises(AssertionError, match="injected precondition failure"): + await fail_precondition() + assert release.is_set() + assert asyncio.all_tasks() == tasks_before diff --git a/tests/mcp_tests/test_proxy_mcp_e2e.py b/tests/mcp_tests/test_proxy_mcp_e2e.py index f56b68922c5..19e02b8efcc 100644 --- a/tests/mcp_tests/test_proxy_mcp_e2e.py +++ b/tests/mcp_tests/test_proxy_mcp_e2e.py @@ -9,15 +9,15 @@ import tempfile import threading import time import typing -from collections.abc import Mapping -from contextlib import asynccontextmanager, contextmanager +from collections.abc import Callable, Mapping +from concurrent.futures import Future +from contextlib import ExitStack, asynccontextmanager, contextmanager from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Final import httpx -import httpx2 import pytest import uvicorn import yaml @@ -27,9 +27,6 @@ from mcp.shared._httpx_utils import create_mcp_http_client from mcp.types import CallToolResult from starlette.requests import Request -from tests._master_key import MASTER_KEY -from tests.integration._support.wire import Reply, Request as WireRequest, Wire, wire_server - from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_proxy_tool from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth @@ -40,6 +37,9 @@ from litellm.proxy.proxy_server import ( cleanup_router_config_variables, initialize, ) +from tests._master_key import MASTER_KEY +from tests.integration._support.wire import Reply, Wire, wire_server +from tests.integration._support.wire import Request as WireRequest CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml") MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py") @@ -84,9 +84,10 @@ async def test_schema_validation_stops_expensive_work_and_recovers(cancel: bool, existing_children: Final = frozenset(child.pid for child in psutil.Process().children()) warm_count: Final = min(concurrency, 4) - assert await asyncio.gather( - *(_tool_argument_validation_error({"type": "object"}, {}) for _ in range(warm_count)) - ) == [None] * warm_count + assert ( + await asyncio.gather(*(_tool_argument_validation_error({"type": "object"}, {}) for _ in range(warm_count))) + == [None] * warm_count + ) started: Final = time.monotonic() tasks: Final = tuple( asyncio.create_task( @@ -218,6 +219,7 @@ def test_proxy_schema_validation_resolves_only_local_references( if not external: assert called["content"][0]["text"] == "called" + @pytest.fixture(scope="session", autouse=True) def _clear_proxy_database_env() -> typing.Iterator[None]: """Ensure local proxy DB settings don't leak into tests.""" @@ -256,42 +258,104 @@ class ProxyRig: loop: asyncio.AbstractEventLoop +def _close_startup_loop(loop: asyncio.AbstractEventLoop) -> None: + if not loop.is_running() and not loop.is_closed(): + loop.close() + + +def _abort_proxy_startup(server: uvicorn.Server, thread: threading.Thread, loop: asyncio.AbstractEventLoop) -> None: + server.should_exit = True + if not thread.is_alive(): + return + + def cancel_owned_tasks() -> None: + for task in asyncio.all_tasks(loop): + task.cancel() + + loop.call_soon_threadsafe(cancel_owned_tasks) + thread.join(timeout=10) + assert not thread.is_alive(), "Proxy startup failure left its thread running" + + def _start_proxy_server( config_path: str, + *, + socket_factory: Callable[[], socket.socket] = socket.socket, ) -> tuple[ProxyRig, uvicorn.Server, threading.Thread, socket.socket]: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", 0)) - host, port = sock.getsockname() + with ExitStack() as startup: + sock: Final = socket_factory() + startup.callback(sock.close) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", 0)) + host, port = sock.getsockname() - config = uvicorn.Config(proxy_app, host=host, port=port, log_level="warning", lifespan="off") - server = uvicorn.Server(config) + config = uvicorn.Config(proxy_app, host=host, port=port, log_level="warning", lifespan="off") + server = uvicorn.Server(config) - loop = asyncio.new_event_loop() + loop: Final = asyncio.new_event_loop() + startup.callback(_close_startup_loop, loop) + failure: Final[Future[BaseException | None]] = Future() - async def _serve() -> None: - from litellm.proxy._experimental.mcp_server import server as mcp_server + async def _serve() -> None: + from litellm.proxy._experimental.mcp_server import server as mcp_server - await _initialize_proxy(config_path) - async with proxy_app.router.lifespan_context(proxy_app), mcp_server.lifespan(proxy_app): - await server.serve(sockets=[sock]) + await _initialize_proxy(config_path) + async with proxy_app.router.lifespan_context(proxy_app), mcp_server.lifespan(proxy_app): + await server.serve(sockets=[sock]) - def _run() -> None: - with asyncio.Runner(loop_factory=lambda: loop) as runner: - runner.run(_serve()) + def _run() -> None: + try: + with asyncio.Runner(loop_factory=lambda: loop) as runner: + runner.run(_serve()) + except BaseException as error: + failure.set_result(error) + else: + failure.set_result(None) - thread = threading.Thread(target=_run, daemon=True) - thread.start() + thread: Final = threading.Thread(target=_run, daemon=True) + startup.callback(_abort_proxy_startup, server, thread, loop) + thread.start() - start_time = time.time() - while not server.started: - if not thread.is_alive(): - raise RuntimeError("Proxy server failed to start") - if time.time() - start_time > PROXY_START_TIMEOUT: - raise TimeoutError("Proxy server did not start in time") - time.sleep(0.05) + start_time = time.time() + while not server.started: + if not thread.is_alive(): + raise RuntimeError("Proxy server failed to start") from failure.result() + if time.time() - start_time > PROXY_START_TIMEOUT: + raise TimeoutError("Proxy server did not start in time") + time.sleep(0.05) - return ProxyRig(f"http://{host}:{port}", config_path, loop), server, thread, sock + startup.pop_all() + return ProxyRig(f"http://{host}:{port}", config_path, loop), server, thread, sock + + +def _check_failed_startup(config: str) -> None: + threads: Final = frozenset(threading.enumerate()) + with socket.socket() as owned: + with pytest.raises(RuntimeError, match="Proxy server failed to start"): + _start_proxy_server(config, socket_factory=lambda: owned) + assert owned.fileno() == -1 + assert frozenset(threading.enumerate()) == threads + + +def test_proxy_startup_failure_closes_acquired_resources(tmp_path: Path) -> None: + config: Final = tmp_path / "invalid-config.yaml" + config.write_text("model_list: [\n") + result: Final = subprocess.run( + ( + sys.executable, + "-P", + "-c", + "import sys; from tests.mcp_tests.test_proxy_mcp_e2e import _check_failed_startup; " + "_check_failed_startup(sys.argv[1])", + str(config), + ), + env={**os.environ, "LITELLM_MODE": "PRODUCTION", "PYTHON_DOTENV_DISABLED": "1"}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stderr @contextmanager @@ -413,8 +477,12 @@ async def main(): asyncio.run(main()) """ completed = await asyncio.to_thread( - subprocess.run, [MCP_PEER_PYTHON, "-c", script, proxy_server_url], - capture_output=True, text=True, timeout=30, check=True, + subprocess.run, + [MCP_PEER_PYTHON, "-c", script, proxy_server_url], + capture_output=True, + text=True, + timeout=30, + check=True, ) results = json.loads(completed.stdout) assert [(item["type"], item["text"]) for item in results["math_stdio-add"]] == [("text", "7")] @@ -426,9 +494,16 @@ def test_initialize_keeps_legacy_negotiation(proxy_server_url: str, requested: s response = httpx.post( proxy_server_url + "/mcp", headers={"Authorization": PROXY_AUTHORIZATION_HEADER, "Accept": "application/json, text/event-stream"}, - json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { - "protocolVersion": requested, "capabilities": {}, "clientInfo": {"name": "legacy-test", "version": "1"}, - }}, + json={ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": requested, + "capabilities": {}, + "clientInfo": {"name": "legacy-test", "version": "1"}, + }, + }, timeout=10, ) assert response.status_code == 200 @@ -453,7 +528,9 @@ async def test_legacy_prompts_and_resources_round_trip(proxy_server_url: str) -> contents = await session.read_resource(status.uri) assert contents.contents[0].text == "ready" templates = await session.list_resource_templates() - greeting_template = next(template for template in templates.resource_templates if "greeting" in template.name) + greeting_template = next( + template for template in templates.resource_templates if "greeting" in template.name + ) contents = await session.read_resource(greeting_template.uri_template.replace("{name}", "Ada")) assert contents.contents[0].text == "Hello, Ada" @@ -828,7 +905,11 @@ class TestProxyMcpAuthorizationScope: listed = await _raw_rpc(proxy_server_url, "sk-none", "tools/list", {}) assert listed.status_code == 200, listed.text - assert {tool["name"] for tool in _rpc_result(listed)["tools"]} == {"search_tools", "get_tool_schema", "call_tool"} + assert {tool["name"] for tool in _rpc_result(listed)["tools"]} == { + "search_tools", + "get_tool_schema", + "call_tool", + } search = await raw_call("search_tools", {"query": "add"}) assert search["isError"] is False, search assert json.loads(search["content"][0]["text"]) == [] diff --git a/tests/proxy_migration_tests/conftest.py b/tests/proxy_migration_tests/conftest.py new file mode 100644 index 00000000000..de5ba62b1be --- /dev/null +++ b/tests/proxy_migration_tests/conftest.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator +from typing import Final +from urllib.parse import urlsplit + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def worker_database() -> Iterator[None]: + if not os.environ.get("PYTEST_XDIST_WORKER") or "DATABASE_URL" not in os.environ: + yield + return + from tests.integration._support.database import read_rows, scratch_database + + admin_url: Final = os.environ["DATABASE_URL"] + with scratch_database() as database, pytest.MonkeyPatch.context() as environment: + environment.setenv("DATABASE_URL", database) + environment.delenv("DIRECT_URL", raising=False) + environment.delenv("DATABASE_URL_READ_REPLICA", raising=False) + yield + assert ( + read_rows( + "SELECT datname FROM pg_database WHERE datname = %s", + (urlsplit(database).path.lstrip("/"),), + database_url=admin_url, + ) + == [] + ) diff --git a/tests/proxy_migration_tests/test_request_log_indexes.py b/tests/proxy_migration_tests/test_request_log_indexes.py index 23e4adce477..b351fc054dc 100644 --- a/tests/proxy_migration_tests/test_request_log_indexes.py +++ b/tests/proxy_migration_tests/test_request_log_indexes.py @@ -26,6 +26,7 @@ from litellm_proxy_extras.request_log_indexes import ( from litellm_proxy_extras.utils import ProxyExtrasDBManager from psycopg import sql from psycopg.abc import Params, QueryNoTemplate +from psycopg.conninfo import make_conninfo from psycopg.rows import class_row pytestmark = pytest.mark.timeout(900) @@ -608,19 +609,22 @@ def test_the_serving_proxy_setup_applies_the_inert_migrations_and_builds_no_inde def test_a_role_that_may_not_create_indexes_is_logged_and_left_for_the_next_job_run( scratch_database: str, caplog: pytest.LogCaptureFixture ) -> None: + role: Final = f"spend_logs_reader_{uuid.uuid4().hex[:8]}" with psycopg.connect(scratch_database, autocommit=True) as conn: conn.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC") - conn.execute("CREATE ROLE spend_logs_reader LOGIN PASSWORD 'reader'") - conn.execute("GRANT USAGE ON SCHEMA public TO spend_logs_reader") - conn.execute('GRANT SELECT ON "LiteLLM_SpendLogs" TO spend_logs_reader') - reader_url: Final = scratch_database.replace("postgres:postgres@", "spend_logs_reader:reader@", 1) + conn.execute(sql.SQL("CREATE ROLE {} LOGIN PASSWORD 'reader'").format(sql.Identifier(role))) + conn.execute(sql.SQL("GRANT USAGE ON SCHEMA public TO {}").format(sql.Identifier(role))) + conn.execute(sql.SQL('GRANT SELECT ON "LiteLLM_SpendLogs" TO {}').format(sql.Identifier(role))) + reader_url: Final = make_conninfo(scratch_database, user=role, password="reader") try: + with psycopg.connect(reader_url) as reader: + assert reader.execute("SELECT current_user").fetchone() == (role,) with caplog.at_level("WARNING", logger="litellm_proxy_extras"): assert ensure_request_log_indexes(reader_url, "public") is False finally: with psycopg.connect(scratch_database, autocommit=True) as conn: - conn.execute("DROP OWNED BY spend_logs_reader") - conn.execute("DROP ROLE spend_logs_reader") + conn.execute(sql.SQL("DROP OWNED BY {}").format(sql.Identifier(role))) + conn.execute(sql.SQL("DROP ROLE {}").format(sql.Identifier(role))) assert "leaving them for the next index build" in caplog.text assert _index_validity(scratch_database, "litellm_call_id_idx") == {} diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 951503aedd1..7efcc76073f 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -159,6 +159,31 @@ def _run_coroutine_if_needed(result: object) -> None: loop.create_task(coroutine) +async def _complete_test_logging() -> None: + await GLOBAL_LOGGING_WORKER.flush() + await GLOBAL_LOGGING_WORKER.stop() + + +def _flush_completed_test_logging() -> None: + queue: Final = GLOBAL_LOGGING_WORKER._queue + if queue is None or queue._unfinished_tasks == 0: + return + worker_loop: Final = GLOBAL_LOGGING_WORKER._bound_loop + if worker_loop is None or worker_loop.is_closed(): + _run_coroutine_if_needed(_complete_test_logging()) + return + if worker_loop.is_running(): + asyncio.run_coroutine_threadsafe(_complete_test_logging(), worker_loop).result() + return + worker_loop.run_until_complete(_complete_test_logging()) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call() -> Iterator[None]: + yield + _flush_completed_test_logging() + + def _close_handler_if_needed(handler: object) -> None: close: Final = getattr(handler, "close", None) if not callable(close): @@ -210,7 +235,8 @@ def isolate_host_environment(isolated_aws_config_files: tuple[Path, Path]) -> It environment.setenv("AWS_EC2_METADATA_DISABLED", "true") for name in AMBIENT_AWS_ENV_VARS: environment.delenv(name, raising=False) - environment.delenv("PROXY_BASE_URL", raising=False) + for name in ("PROXY_BASE_URL", "SERVER_ROOT_PATH", "SERVER_ROOT_PATHS", "LITELLM_LOG"): + environment.delenv(name, raising=False) environment.setenv("LITELLM_CLI_DISABLE_KEYRING", "1") yield @@ -260,6 +286,8 @@ def isolate_router_model_cost_state() -> Iterator[None]: for model_key, model_value in litellm_utils_module._runtime_registered_model_cost.items() } litellm_utils_module._invalidate_model_cost_lowercase_map() + litellm_router_module._live_routers.clear() + litellm_utils_module._runtime_registered_model_cost.clear() yield for router in tuple(litellm_router_module._live_routers): litellm_router_module._live_routers.discard(router) diff --git a/tests/unit/integration_support/test_ordering.py b/tests/unit/integration_support/test_ordering.py new file mode 100644 index 00000000000..98b08d0849c --- /dev/null +++ b/tests/unit/integration_support/test_ordering.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from itertools import groupby +from types import MappingProxyType +from typing import Final + +import pytest + +from tests.integration._support.ordering import ( + CollectedCase, + ScopedParameter, + case_shards, + fixture_groups, + parse_timings, +) +from tests.integration.conftest import _order_key + + +def test_seeded_order_reuses_each_scoped_configuration_without_dropping_cases() -> None: + values: Final = ( + "D1", + "D1", + "D1", + "D1", + "D1", + "D2", + "D2", + "D2", + "D2", + "D2", + "D3", + "D3", + "D3", + "D3", + "D4", + "D4", + "D4", + "D4", + ) + cases: Final = tuple( + CollectedCase( + f"tests/integration/security/test_slots.py::test_route[{value}-{index}]", + (ScopedParameter("module:slots", "rig", value),), + ) + for index, value in enumerate(values) + ) + groups: Final = fixture_groups(cases) + ordered: Final = tuple(sorted(cases, key=lambda case: _order_key(1848224111, case.nodeid, groups[case.nodeid]))) + configurations: Final = tuple(value for value, _ in groupby(case.parameters[0].value for case in ordered)) + assert len(configurations) == len(set(values)) + assert set(configurations) == set(values) + assert frozenset(case.nodeid for case in ordered) == frozenset(case.nodeid for case in cases) + assert len(ordered) == len(cases) + assert ordered == tuple(sorted(cases, key=lambda case: _order_key(1848224111, case.nodeid, groups[case.nodeid]))) + assert ordered != tuple(sorted(cases, key=lambda case: _order_key(984872314, case.nodeid, groups[case.nodeid]))) + + +def test_equal_unhashable_parameters_share_a_group_only_within_their_fixture_owner() -> None: + first: Final = CollectedCase("test_slots.py::test_first", (ScopedParameter("class:first", "rig", {"slot": "D1"}),)) + equal: Final = CollectedCase("test_slots.py::test_equal", (ScopedParameter("class:first", "rig", {"slot": "D1"}),)) + other_owner: Final = CollectedCase( + "test_slots.py::test_other", (ScopedParameter("class:second", "rig", {"slot": "D1"}),) + ) + other_file: Final = CollectedCase("test_other.py::test_first", first.parameters) + plain: Final = CollectedCase("test_slots.py::test_plain") + groups: Final = fixture_groups((first, equal, other_owner, other_file, plain)) + assert groups[first.nodeid] == groups[equal.nodeid] + assert groups[first.nodeid] != groups[other_owner.nodeid] + assert groups[first.nodeid] != groups[other_file.nodeid] + assert groups[plain.nodeid] == "" + + +def test_shards_balance_cases_without_splitting_a_scoped_configuration() -> None: + cases: Final = tuple( + CollectedCase( + f"test_slots.py::test_route[{index}]", + (ScopedParameter("module:slots", "rig", index // 4),), + ("module:slots:extra",) if index % 2 else (), + ) + for index in range(20) + ) + plain: Final = tuple(CollectedCase(f"test_slots.py::test_plain[{index}]") for index in range(7)) + shards: Final = case_shards(cases + plain, 3) + assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases + plain) + assert sum(len(shard) for shard in shards) == len(cases + plain) + assert max(map(len, shards)) - min(map(len, shards)) <= 1 + for _, group in groupby(cases, key=lambda case: case.parameters[0].value): + configuration: Final = frozenset(case.nodeid for case in group) + assert sum(configuration <= shard for shard in shards) == 1 + assert shards == case_shards(tuple(reversed(cases + plain)), 3) + + +def test_shards_reuse_unparametrized_fixtures_and_keep_different_results_separate() -> None: + shared: Final = tuple( + CollectedCase(f"test_sockets.py::test_override[{index}]", fixtures=("module:test_sockets.py:override",)) + for index in range(4) + ) + other: Final = CollectedCase("test_sockets.py::test_default", fixtures=("module:test_sockets.py:default",)) + plain: Final = tuple(CollectedCase(f"test_other.py::test_plain[{index}]") for index in range(7)) + shards: Final = case_shards((*shared, other, *plain), 3) + configuration: Final = frozenset(case.nodeid for case in shared) + assert sum(configuration <= shard for shard in shards) == 1 + assert sorted(map(len, shards)) == [4, 4, 4] + assert frozenset.union(*shards) == frozenset(case.nodeid for case in (*shared, other, *plain)) + groups: Final = fixture_groups((*shared, other)) + assert len({groups[case.nodeid] for case in shared}) == 1 + assert groups[other.nodeid] != groups[shared[0].nodeid] + assert shards == case_shards(tuple(reversed((*shared, other, *plain))), 3) + + +def test_an_oversized_unparametrized_fixture_is_split_into_bounded_groups() -> None: + cases: Final = tuple( + CollectedCase(f"test_sockets.py::test_route[{index:02}]", fixtures=("module:test_sockets.py:owned",)) + for index in range(20) + ) + shards: Final = case_shards(cases, 4) + assert tuple(map(len, shards)) == (5, 5, 5, 5) + assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases) + assert sum(len(shard) for shard in shards) == len(cases) + + +def test_measured_cost_splits_an_expensive_shared_fixture_without_losing_cases() -> None: + expensive: Final = tuple( + CollectedCase(f"test_tracing.py::test_route[{index}]", fixtures=("module:test_tracing.py:gateway",)) + for index in range(4) + ) + cheap: Final = tuple(CollectedCase(f"test_other.py::test_route[{index}]") for index in range(8)) + cases: Final = (*expensive, *cheap) + durations: Final = MappingProxyType({case.nodeid: 40.0 if case in expensive else 1.0 for case in cases}) + shards: Final = case_shards(cases, 4, durations) + assert sorted(sum(durations[nodeid] for nodeid in shard) for shard in shards) == [42.0] * 4 + assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases) + assert sum(len(shard) for shard in shards) == len(cases) + assert shards == case_shards(tuple(reversed(cases)), 4, durations) + + +@pytest.mark.parametrize( + "content", + ( + "invalid", + "{}", + '[{"nodeid":"case","seconds":-1}]', + '[{"nodeid":"case","seconds":"1"}]', + '[{"nodeid":"case","seconds":NaN}]', + '[{"nodeid":"case","seconds":Infinity}]', + '[{"nodeid":"case","seconds":1},{"nodeid":"case","seconds":2}]', + ), +) +def test_invalid_timing_cache_preserves_the_complete_cold_partition(content: str) -> None: + cases: Final = tuple(CollectedCase(f"test_owned.py::test_route[{index}]") for index in range(9)) + durations: Final = parse_timings(content) + assert not durations + assert case_shards(cases, 3, durations) == case_shards(cases, 3) + + +def test_timing_cache_keeps_zero_duration_skips_and_assigns_new_cases() -> None: + durations: Final = parse_timings('[{"nodeid":"skip","seconds":0},{"nodeid":"old","seconds":12}]') + cases: Final = (CollectedCase("skip"), CollectedCase("new")) + shards: Final = case_shards(cases, 2, durations) + assert durations["skip"] > 0 + assert frozenset.union(*shards) == frozenset({"skip", "new"}) + assert tuple(map(len, shards)) == (1, 1) + + +def test_measured_cost_keeps_parametrized_fixture_configurations_atomic() -> None: + shared: Final = tuple( + CollectedCase( + f"test_tracing.py::test_route[{index}]", + (ScopedParameter("module:test_tracing.py", "configuration", "shared"),), + ) + for index in range(3) + ) + plain: Final = tuple(CollectedCase(f"test_other.py::test_route[{index}]") for index in range(5)) + cases: Final = (*shared, *plain) + durations: Final = MappingProxyType({case.nodeid: 40.0 if case in shared else 1.0 for case in cases}) + shards: Final = case_shards(cases, 3, durations) + configuration: Final = frozenset(case.nodeid for case in shared) + assert sum(configuration <= shard for shard in shards) == 1 + assert frozenset.union(*shards) == frozenset(case.nodeid for case in cases) + assert sum(len(shard) for shard in shards) == len(cases) diff --git a/tests/unit/integration_support/test_process.py b/tests/unit/integration_support/test_process.py index 7896786fce8..50c750de759 100644 --- a/tests/unit/integration_support/test_process.py +++ b/tests/unit/integration_support/test_process.py @@ -9,8 +9,11 @@ from pathlib import Path from types import ModuleType from typing import Final +import httpx import pytest +from tests.integration._support.client import Gateway + TESTS_DIR: Final = Path(__file__).resolve().parents[2] UNRELATED_CRASH: Final = "Traceback (most recent call last):\nModuleNotFoundError: No module named 'litellm'\n" @@ -59,3 +62,53 @@ def test_lost_port_race_needs_the_process_to_have_exited( process_module: ModuleType, bind_error_line: str, tmp_path: Path ) -> None: assert not process_module._lost_port_race(None, _written_log(tmp_path, bind_error_line)) + + +@pytest.mark.parametrize( + ("writer", "explicit_reader", "expected_reader"), + ( + ("postgresql://writer@127.0.0.1/shared", None, "postgresql://reader@127.0.0.1/shared"), + ("postgresql://writer@127.0.0.1/scratch", None, None), + ( + "postgresql://writer@127.0.0.1/scratch", + "postgresql://reader@127.0.0.1/scratch", + "postgresql://reader@127.0.0.1/scratch", + ), + ), +) +def test_owned_writer_keeps_only_a_reader_paired_with_its_database( + process_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + writer: str, + explicit_reader: str | None, + expected_reader: str | None, +) -> None: + monkeypatch.setenv("INTEGRATION_PROXY_READ_REPLICA_URL", "postgresql://reader@127.0.0.1/shared") + overrides: Final = { + "DATABASE_URL": writer, + **({"DATABASE_URL_READ_REPLICA": explicit_reader} if explicit_reader else {}), + } + with httpx.Client(base_url="http://127.0.0.1:1", trust_env=False) as client: + environment: Final = process_module._proxy_environment( + Gateway(client, "owned-test-key", "http://127.0.0.1:1"), overrides, () + ) + assert environment["DATABASE_URL"] == writer + assert environment.get("DATABASE_URL_READ_REPLICA") == expected_reader + + +@pytest.mark.parametrize("explicit", (False, True)) +@pytest.mark.parametrize("remove", (False, True)) +def test_owned_runtime_configuration_preserves_custom_worker_startup_hooks( + process_module: ModuleType, monkeypatch: pytest.MonkeyPatch, explicit: bool, remove: bool +) -> None: + monkeypatch.setenv("LITELLM_WORKER_STARTUP_HOOKS", "ambient:configure") + with httpx.Client(base_url="http://127.0.0.1:1", trust_env=False) as client: + environment: Final = process_module._proxy_environment( + Gateway(client, "owned-test-key", "http://127.0.0.1:1"), + {"LITELLM_WORKER_STARTUP_HOOKS": "custom:configure"} if explicit else {}, + ("LITELLM_WORKER_STARTUP_HOOKS",) if remove else (), + ) + expected: Final = "custom:configure" if explicit else ("" if remove else "ambient:configure") + assert environment["LITELLM_WORKER_STARTUP_HOOKS"] == ",".join( + hook for hook in ("integration._support.runtime:configure_executor", expected) if hook + ) diff --git a/tests/unit/integration_support/test_sharding.py b/tests/unit/integration_support/test_sharding.py new file mode 100644 index 00000000000..3d3b53b6f51 --- /dev/null +++ b/tests/unit/integration_support/test_sharding.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +import pytest + +from tests.integration._support.ordering import CaseTiming +from tests.integration._support.sharding import ShardExecution, merged_timings, shard_errors +from tests.integration.conftest import COLLECTED, IntegrationReportPlugin +from tests.integration.conftest import INVENTORY as WORKER_INVENTORY + +INVENTORY: Final = ("test_owned.py::test_first", "test_owned.py::test_skipped", "test_owned.py::test_last") + + +def _execution( + index: int, + collected: tuple[str, ...], + *, + inventory: tuple[str, ...] = INVENTORY, + passed: tuple[str, ...] | None = None, + skipped: tuple[str, ...] = (), + shard_count: int = 2, + complete: bool = True, + exitstatus: int = 0, +) -> ShardExecution: + return ShardExecution( + inventory=inventory, + collected=collected, + passed=tuple(node for node in collected if node not in skipped) if passed is None else passed, + skipped=skipped, + shard_count=shard_count, + shard_index=index, + complete=complete, + exitstatus=exitstatus, + ) + + +def test_shard_qualification_retains_passes_and_skip_identities() -> None: + executions: Final = ( + _execution(0, INVENTORY[:2], skipped=INVENTORY[1:2]), + _execution(1, INVENTORY[2:]), + ) + assert shard_errors(executions, 2) == () + + +@pytest.mark.parametrize( + "executions", + ( + (), + (_execution(0, INVENTORY),), + (_execution(0, INVENTORY[:2]), _execution(0, INVENTORY[2:])), + (_execution(0, INVENTORY[:1]), _execution(1, INVENTORY[2:])), + (_execution(0, INVENTORY), _execution(1, INVENTORY[2:])), + (_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], inventory=INVENTORY[:2])), + (_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], complete=False)), + (_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], exitstatus=1)), + (_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], shard_count=3)), + (_execution(0, INVENTORY[:2], inventory=(*INVENTORY, INVENTORY[0])), _execution(1, INVENTORY[2:])), + (_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], passed=())), + (_execution(0, INVENTORY[:2]), _execution(1, INVENTORY[2:], skipped=INVENTORY[2:] * 2)), + (_execution(0, (*INVENTORY[:2], "test_foreign.py::test_other")), _execution(1, INVENTORY[2:])), + ), +) +def test_shard_qualification_rejects_lost_duplicate_foreign_or_failed_cases( + executions: tuple[ShardExecution, ...], +) -> None: + assert shard_errors(executions, 2) + + +def test_timing_snapshot_includes_every_executed_pass_and_skip_once() -> None: + first: Final = CaseTiming(nodeid=INVENTORY[0], seconds=40.0) + skipped: Final = CaseTiming(nodeid=INVENTORY[1], seconds=0.0) + last: Final = CaseTiming(nodeid=INVENTORY[2], seconds=1.0) + executions: Final = ( + _execution(0, INVENTORY[:2], skipped=INVENTORY[1:2]).model_copy(update={"timings": (first, skipped)}), + _execution(1, INVENTORY[2:]).model_copy(update={"timings": (last,)}), + ) + assert shard_errors(executions, 2) == () + assert merged_timings(executions) == tuple(sorted((first, skipped, last), key=lambda timing: timing.nodeid)) + + +@pytest.mark.parametrize( + "timings", + ( + (), + (CaseTiming(nodeid=INVENTORY[0], seconds=1.0),), + (CaseTiming(nodeid=INVENTORY[0], seconds=1.0),) * 2, + (CaseTiming(nodeid="foreign", seconds=1.0), CaseTiming(nodeid=INVENTORY[1], seconds=0.0)), + ), +) +def test_incomplete_duplicate_or_foreign_measurements_cannot_seed_the_timing_cache( + timings: tuple[CaseTiming, ...], +) -> None: + execution: Final = _execution(0, INVENTORY[:2]).model_copy(update={"timings": timings}) + assert merged_timings((execution,)) == () + + +@dataclass(frozen=True, slots=True) +class WorkerResult: + workeroutput: Mapping[str, object] + + +def test_worker_results_retain_the_full_inventory_and_reject_disagreement() -> None: + config: Final = pytest.Config(pytest.PytestPluginManager()) + config.stash[COLLECTED] = INVENTORY[:1] + plugin: Final = IntegrationReportPlugin(config) + complete: Final = WorkerResult(MappingProxyType({"integration_inventory": INVENTORY})) + plugin.pytest_testnodedown(complete, None) + plugin.pytest_testnodedown(complete, None) + assert config.stash[WORKER_INVENTORY] == INVENTORY + assert config.stash[COLLECTED] == INVENTORY[:1] + incomplete: Final = WorkerResult(MappingProxyType({"integration_inventory": INVENTORY[:1]})) + with pytest.raises(pytest.UsageError, match="inventories disagree"): + plugin.pytest_testnodedown(incomplete, None) diff --git a/tests/unit/integrations/generic_api/test_generic_api_callback.py b/tests/unit/integrations/generic_api/test_generic_api_callback.py index ac7c0b0d774..e3bb47cad02 100644 --- a/tests/unit/integrations/generic_api/test_generic_api_callback.py +++ b/tests/unit/integrations/generic_api/test_generic_api_callback.py @@ -3,19 +3,22 @@ import importlib import json import logging import os -from collections.abc import AsyncIterator, Iterator -from typing import Final +from collections.abc import AsyncIterator, Iterator, Sequence +from typing import Final, TypedDict from unittest.mock import AsyncMock import httpx import pytest import pytest_asyncio +from pydantic import TypeAdapter +from typing_extensions import ReadOnly import litellm from litellm._logging import verbose_logger from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.integrations.gcs_pubsub.pub_sub import * -from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger +from litellm.integrations.generic_api.generic_api_callback import LOG_FORMAT_TYPES, GenericAPILogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import StandardLoggingPayload from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -628,3 +631,108 @@ def setup_and_teardown(): if hasattr(litellm, "in_memory_llm_clients_cache"): litellm.in_memory_llm_clients_cache.flush_cache() yield + + +class _QueuedLog(TypedDict): + id: ReadOnly[str] + + +_QUEUED_LOG: Final = TypeAdapter(_QueuedLog) +_LOG_BATCH: Final = TypeAdapter(tuple[_QueuedLog, ...]) + + +def _batch_events(body: bytes, log_format: LOG_FORMAT_TYPES) -> tuple[_QueuedLog, ...]: + match log_format: + case "json_array": + return _LOG_BATCH.validate_json(body) + case "ndjson": + return tuple(_QUEUED_LOG.validate_json(line) for line in body.splitlines()) + case "single": + return (_QUEUED_LOG.validate_json(body),) + + +def _received_log_ids(requests: Sequence[httpx.Request], log_format: LOG_FORMAT_TYPES) -> Iterator[str]: + for request in requests: + for event in _batch_events(request.content, log_format): + yield event["id"] + + +class _InProcessLogSink: + def __init__(self, *, hold_first: bool = False) -> None: + self.hold_first: Final = hold_first + self.arrived: Final = asyncio.Event() + self.release: Final = asyncio.Event() + self.requests: tuple[httpx.Request, ...] = () + + async def respond(self, request: httpx.Request) -> httpx.Response: + self.requests = (*self.requests, request) + if self.hold_first and len(self.requests) == 1: + self.arrived.set() + await self.release.wait() + return httpx.Response(200) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_format", ("json_array", "ndjson", "single")) +async def test_generic_api_keeps_failure_arriving_during_success_batch(log_format: LOG_FORMAT_TYPES) -> None: + sink: Final = _InProcessLogSink(hold_first=True) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(sink.respond)) + before_tasks: Final = frozenset(asyncio.all_tasks()) + logger: Final = GenericAPILogger( + endpoint="https://owned-logger.invalid", + async_httpx_client=handler, + log_format=log_format, + batch_size=1000, + flush_interval=3600, + ) + owned_tasks: Final = frozenset(asyncio.all_tasks()) - before_tasks + try: + async with asyncio.TaskGroup() as flushes: + await logger.async_log_success_event({"standard_logging_object": {"id": "before"}}, None, None, None) + first_flush: Final = flushes.create_task(logger.flush_queue()) + try: + await sink.arrived.wait() + await logger.async_log_failure_event({"standard_logging_object": {"id": "during"}}, None, None, None) + finally: + sink.release.set() + await first_flush + await logger.flush_queue() + assert tuple(_received_log_ids(sink.requests, log_format)) == ("before", "during") + finally: + sink.release.set() + for task in owned_tasks: + task.cancel() + await asyncio.gather(*owned_tasks, return_exceptions=True) + await handler.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("log_format", ("json_array", "ndjson", "single")) +async def test_generic_api_concurrent_flushes_deliver_each_event_once(log_format: LOG_FORMAT_TYPES) -> None: + sink: Final = _InProcessLogSink(hold_first=True) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(sink.respond)) + before_tasks: Final = frozenset(asyncio.all_tasks()) + logger: Final = GenericAPILogger( + endpoint="https://owned-logger.invalid", + async_httpx_client=handler, + log_format=log_format, + batch_size=1000, + flush_interval=3600, + ) + owned_tasks: Final = frozenset(asyncio.all_tasks()) - before_tasks + try: + await logger.async_log_success_event({"standard_logging_object": {"id": "once"}}, None, None, None) + async with asyncio.TaskGroup() as flushes: + flushes.create_task(logger.flush_queue()) + flushes.create_task(logger.async_send_batch()) + try: + await sink.arrived.wait() + finally: + sink.release.set() + assert tuple(_received_log_ids(sink.requests, log_format)) == ("once",) + finally: + sink.release.set() + for task in owned_tasks: + task.cancel() + await asyncio.gather(*owned_tasks, return_exceptions=True) + await handler.close() diff --git a/tests/unit/litellm_core_utils/test_aws_partition.py b/tests/unit/litellm_core_utils/test_aws_partition.py index b7f99cf0f18..91fff1996ee 100644 --- a/tests/unit/litellm_core_utils/test_aws_partition.py +++ b/tests/unit/litellm_core_utils/test_aws_partition.py @@ -1,4 +1,5 @@ import ast +from collections.abc import Iterator from pathlib import Path from typing import Final from urllib.parse import urlparse @@ -187,21 +188,47 @@ def test_every_endpoint_builder_keeps_amazonaws_com_outside_cn(builder_name: str assert hostname.endswith(".amazonaws.com"), url -def _fstring_literal_offenders(needle: str) -> list[str]: - litellm_root = Path(litellm.__file__).parent - return [ - f"{path.relative_to(litellm_root)}: {part.value!r}" - for path in sorted(litellm_root.rglob("*.py")) - for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))) - if isinstance(node, ast.JoinedStr) - for part in node.values - if isinstance(part, ast.Constant) and isinstance(part.value, str) and needle in part.value - ] +def _fstring_literals(root: Path) -> Iterator[tuple[str, str]]: + for path in sorted(root.rglob("*.py")): + source: Final = ast.parse(path.read_text(encoding="utf-8")) + relative: Final = str(path.relative_to(root)) + for node in ast.walk(source): + if not isinstance(node, ast.JoinedStr): + continue + for part in node.values: + if isinstance(part, ast.Constant) and isinstance(part.value, str): + yield relative, part.value -def test_no_fstring_hardcodes_the_commercial_dns_suffix() -> None: - assert _fstring_literal_offenders("amazonaws.com") == [] +@pytest.fixture +def fstring_literals() -> tuple[tuple[str, str], ...]: + return tuple(_fstring_literals(Path(litellm.__file__).parent)) -def test_no_fstring_hardcodes_the_commercial_arn_prefix() -> None: - assert _fstring_literal_offenders("arn:aws:") == [] +def _fstring_literal_offenders(needle: str, literals: tuple[tuple[str, str], ...]) -> list[str]: + return [f"{path}: {value!r}" for path, value in literals if needle in value] + + +def test_no_fstring_hardcodes_the_commercial_dns_suffix(fstring_literals: tuple[tuple[str, str], ...]) -> None: + assert _fstring_literal_offenders("amazonaws.com", fstring_literals) == [] + + +def test_no_fstring_hardcodes_the_commercial_arn_prefix(fstring_literals: tuple[tuple[str, str], ...]) -> None: + assert _fstring_literal_offenders("arn:aws:", fstring_literals) == [] + + +def test_fstring_audit_sees_changed_and_new_source(tmp_path: Path) -> None: + source: Final = tmp_path / "source.py" + source.write_text('value = f"{region}.amazonaws.com"\n', encoding="utf-8") + first: Final = tuple(_fstring_literals(tmp_path)) + assert _fstring_literal_offenders("amazonaws.com", first) + assert _fstring_literal_offenders("arn:aws:", first) == [] + source.write_text('value = f"arn:aws:{service}"\n', encoding="utf-8") + changed: Final = tuple(_fstring_literals(tmp_path)) + assert _fstring_literal_offenders("amazonaws.com", changed) == [] + assert _fstring_literal_offenders("arn:aws:", changed) + added_source: Final = tmp_path / "added.py" + added_source.write_text('value = f"{region}.amazonaws.com"\n', encoding="utf-8") + added: Final = tuple(_fstring_literals(tmp_path)) + assert _fstring_literal_offenders("amazonaws.com", added) + assert _fstring_literal_offenders("arn:aws:", added) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 2a1c9d66965..fd72af47e09 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -25,6 +25,10 @@ from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeRespo from litellm.types.utils import GenericGuardrailAPIInputs +def _mock_logging() -> MagicMock: + return MagicMock(dispatch_success_handlers=AsyncMock(), dispatch_failure_handlers=AsyncMock()) + + def _make_transcript_event(text: str, item_id: str = "item_x") -> bytes: return json.dumps( { @@ -39,7 +43,7 @@ def test_realtime_streaming_store_message(): # Setup websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) # Test 1: Session created event (string input) @@ -112,7 +116,7 @@ def test_make_disable_auto_response_message_produces_ga_shape(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) raw = streaming._make_disable_auto_response_message() @@ -135,7 +139,7 @@ def test_make_disable_auto_response_message_produces_beta_shape_for_beta_clients websocket = MagicMock() websocket.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) raw = streaming._make_disable_auto_response_message() @@ -157,7 +161,7 @@ async def test_backend_to_client_send_text_receives_str_not_bytes(): ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() logging_obj.dispatch_success_handlers = AsyncMock() @@ -182,7 +186,7 @@ async def test_backend_to_client_skips_non_utf8_binary_frames(): ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -198,7 +202,7 @@ def _xai_streaming(client_ws=None, backend_ws=None, logging_obj=None): return RealTimeStreaming( client_ws or MagicMock(), backend_ws or MagicMock(), - logging_obj or MagicMock(), + logging_obj or _mock_logging(), event_normalizer=XAIRealtimeNormalizer(), ) @@ -417,7 +421,7 @@ async def test_backend_to_client_drops_ping_events(): ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = _xai_streaming(client_ws, backend_ws, logging_obj) @@ -452,7 +456,7 @@ async def test_backend_to_client_normalizes_empty_response_usage(): ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = _xai_streaming(client_ws, backend_ws, logging_obj) @@ -485,7 +489,7 @@ async def test_backend_to_client_beta_receives_normalized_events(): ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = _xai_streaming(client_ws, backend_ws, logging_obj) @@ -520,7 +524,7 @@ async def test_backend_to_client_stores_normalized_events_for_logging(): ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = _xai_streaming(client_ws, backend_ws, logging_obj) @@ -554,7 +558,7 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_clients(): ) backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.pre_call = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -591,7 +595,7 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): ) backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.pre_call = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=True) @@ -625,7 +629,7 @@ async def test_provider_config_path_translates_ga_events_for_beta_clients(): client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} client_ws.send_text = AsyncMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() provider_config = MagicMock() provider_config.transform_realtime_response = MagicMock( @@ -679,7 +683,7 @@ def test_collect_user_input_from_text_conversation_item(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) msg = json.dumps( @@ -704,7 +708,7 @@ def test_collect_user_input_from_session_update_instructions(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) msg = json.dumps( @@ -727,7 +731,7 @@ def test_collect_user_input_from_transcription_event(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) event_obj = { @@ -748,7 +752,7 @@ def test_collect_user_input_ignores_irrelevant_events(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) # input_audio_buffer.append should not be collected @@ -768,7 +772,7 @@ def test_collect_user_input_empty_transcript_not_collected(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) event_obj = { @@ -787,7 +791,7 @@ async def test_log_messages_sets_input_messages_on_logging_obj(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.model_call_details = {"messages": "default-message-value"} logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -833,7 +837,7 @@ async def test_transcription_captured_in_backend_to_client(): ) backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.model_call_details = {"messages": "default-message-value"} logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -879,7 +883,7 @@ async def test_transcription_session_captures_usage_and_skips_response_create(): backend_ws.recv = AsyncMock(side_effect=[session_created, completed, ConnectionClosed(None, None)]) backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.model_call_details = {} logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -965,7 +969,7 @@ async def test_transcript_triggers_response_create_only_when_a_transcript_guardr streaming: Final = RealTimeStreaming( client_ws, backend_ws, - MagicMock(async_success_handler=AsyncMock()), + _mock_logging(), provider_config=_PassthroughRealtimeConfig() if through_provider_config else None, ) @@ -1156,7 +1160,7 @@ def test_capture_transcription_usage_deduplicates_when_already_stored(): async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup(): websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.pre_call = MagicMock() # Two session.update messages arrive before setupComplete round-trip. @@ -1203,7 +1207,7 @@ async def test_failed_content_send_does_not_block_later_setup(): even though the backend never received any content.""" websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() provider_config = MagicMock() provider_config.transform_realtime_request = MagicMock(side_effect=lambda m, *a, **k: [m]) @@ -1241,7 +1245,7 @@ def test_collect_session_tools_from_session_update(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) msg = json.dumps( @@ -1278,7 +1282,7 @@ def test_collect_tool_calls_from_response_done(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) streaming.logged_real_time_event_types = "*" @@ -1313,7 +1317,7 @@ def test_tool_calls_not_collected_from_non_function_call_output(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) streaming.logged_real_time_event_types = "*" @@ -1344,7 +1348,7 @@ async def test_log_messages_includes_tools_in_model_call_details(): """ websocket = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.model_call_details = {"messages": "default-message-value"} logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -1414,7 +1418,7 @@ async def test_realtime_guardrail_blocks_prompt_injection(monkeypatch: pytest.Mo ) backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -1490,7 +1494,7 @@ async def test_realtime_guardrail_allows_clean_transcript(monkeypatch: pytest.Mo ) backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -1540,7 +1544,7 @@ async def test_realtime_text_input_guardrail_blocks_and_returns_error(monkeypatc backend_ws.send = AsyncMock() backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.pre_call = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -1633,7 +1637,7 @@ async def test_realtime_function_call_output_guardrail_blocks_and_returns_error( backend_ws.send = AsyncMock() backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.pre_call = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -1711,7 +1715,7 @@ async def test_realtime_function_call_output_guardrail_allows_clean_output(monke backend_ws.send = AsyncMock() backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.pre_call = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -1770,7 +1774,7 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pyt client_ws = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) assert streaming._has_realtime_guardrails() is True, ( @@ -1814,7 +1818,7 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra backend_ws.recv = AsyncMock(side_effect=[session_created_event, ConnectionClosed(None, None)]) backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -1870,7 +1874,7 @@ async def test_realtime_session_created_does_not_inject_session_update_for_pre_c backend_ws.recv = AsyncMock(side_effect=[session_created_event, ConnectionClosed(None, None)]) backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -1912,7 +1916,7 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monke client_ws = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming( client_ws, backend_ws, @@ -1967,7 +1971,7 @@ async def test_end_session_after_n_fails_closes_connection(monkeypatch: pytest.M backend_ws.send = AsyncMock() backend_ws.close = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2012,7 +2016,7 @@ async def test_on_violation_end_session_closes_on_first_fail(monkeypatch: pytest backend_ws.send = AsyncMock() backend_ws.close = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2051,7 +2055,7 @@ async def test_provider_path_suppresses_duplicate_session_created_after_syntheti } ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_1" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2103,7 +2107,7 @@ async def test_duplicate_session_created_still_triggers_guardrail_turn_detection } ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_1" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2139,7 +2143,7 @@ async def test_guardrail_update_respects_idempotency_flag(): backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_1" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2188,7 +2192,7 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_1" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2254,7 +2258,7 @@ async def test_guardrail_turn_detection_injection_tolerates_non_dict_value( backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_1" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2317,7 +2321,7 @@ async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_act backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_1" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2371,7 +2375,7 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2420,7 +2424,7 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp client_ws.receive_text = AsyncMock(side_effect=[audio_msg, ConnectionClosed(None, None)]) backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() config = GeminiRealtimeConfig() streaming = RealTimeStreaming( @@ -2454,7 +2458,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)]) backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() config = GeminiRealtimeConfig() streaming = RealTimeStreaming( @@ -2483,7 +2487,7 @@ async def test_deferred_setup_flush_buffers_audio_received_during_flush(): new_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "new-audio"}) client_ws.receive_text = AsyncMock(side_effect=[new_audio_msg, ConnectionClosed(None, None)]) backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() provider_config = MagicMock() provider_config.requires_session_configuration = MagicMock(return_value=False) @@ -2546,7 +2550,7 @@ async def test_deferred_setup_flush_buffers_audio_received_during_flush(): async def test_deferred_setup_flush_retains_unsent_messages_after_send_failure(): client_ws = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) buffered_messages = [ json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="}), @@ -2588,7 +2592,7 @@ async def test_deferred_setup_flushes_audio_on_backend_session_created(monkeypat ConnectionClosed(None, None), ] ) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.litellm_trace_id = "trace_defer" logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() @@ -2627,7 +2631,7 @@ async def test_deferred_setup_caps_non_audio_buffered_messages(monkeypatch): ) backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming( client_ws, @@ -2658,7 +2662,7 @@ async def test_deferred_setup_caps_non_audio_buffered_bytes(monkeypatch): client_ws.receive_text = AsyncMock(side_effect=[audio_msg, big_non_audio, ConnectionClosed(None, None)]) backend_ws = MagicMock() backend_ws.send = AsyncMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming( client_ws, @@ -2690,7 +2694,7 @@ def _ga_client_ws(): def _streaming_with(client_ws): backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() return RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2752,7 +2756,7 @@ async def test_beta_client_receives_translated_audio_delta(): frame = json.dumps({"type": "response.output_audio.delta", "delta": "QUJD", "event_id": "e1"}) backend_ws = MagicMock() backend_ws.recv = AsyncMock(side_effect=[frame.encode(), ConnectionClosed(None, None)]) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2771,7 +2775,7 @@ async def test_ga_client_receives_raw_passthrough(): frame = json.dumps({"type": "response.output_audio.delta", "delta": "QUJD", "event_id": "e1"}) backend_ws = MagicMock() backend_ws.recv = AsyncMock(side_effect=[frame.encode(), ConnectionClosed(None, None)]) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2791,7 +2795,7 @@ async def test_beta_client_non_translated_event_forwarded_raw(): frame = json.dumps({"type": "error", "error": {"message": "boom"}}) backend_ws = MagicMock() backend_ws.recv = AsyncMock(side_effect=[frame.encode(), ConnectionClosed(None, None)]) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2808,7 +2812,7 @@ async def test_beta_client_drops_conversation_item_done(): frame = json.dumps({"type": "conversation.item.done", "item": {"id": "i1"}}) backend_ws = MagicMock() backend_ws.recv = AsyncMock(side_effect=[frame.encode(), ConnectionClosed(None, None)]) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2834,7 +2838,7 @@ async def test_audio_delta_frame_parsed_at_most_once(): frame = json.dumps({"type": "response.output_audio.delta", "delta": "QUJD", "event_id": "e1"}) backend_ws = MagicMock() backend_ws.recv = AsyncMock(side_effect=[frame.encode(), ConnectionClosed(None, None)]) - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) @@ -2870,7 +2874,7 @@ def test_collapse_buffered_audio_messages_applies_clear_semantics(): async def test_deferred_setup_clear_drops_buffered_appends_on_flush(): client_ws = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) old_audio = json.dumps({"type": "input_audio_buffer.append", "audio": "old"}) @@ -2892,7 +2896,7 @@ async def test_deferred_setup_clear_drops_buffered_appends_on_flush(): async def test_deferred_setup_clear_drops_appends_when_buffered(): client_ws = MagicMock() backend_ws = MagicMock() - logging_obj = MagicMock() + logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) old_audio = json.dumps({"type": "input_audio_buffer.append", "audio": "old"}) @@ -3003,7 +3007,7 @@ async def test_provider_config_path_captures_transcription_usage(): client_ws.send_text = AsyncMock() backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() - logging_obj: Final = MagicMock() + logging_obj: Final = _mock_logging() usage: Final[RealtimeInputAudioTranscriptionUsage] = { "type": "tokens", @@ -3066,7 +3070,7 @@ async def test_session_close_flushes_unbilled_transcription_usage(): client_ws.send_text = AsyncMock() backend_ws: Final = MagicMock() backend_ws.recv = AsyncMock(side_effect=[b'{"serverContent": {}}', ConnectionClosed(None, None)]) - logging_obj: Final = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() logging_obj.dispatch_success_handlers = AsyncMock() @@ -3142,7 +3146,7 @@ async def test_session_close_flush_noop_without_unbilled_usage(): client_ws.send_text = AsyncMock() backend_ws: Final = MagicMock() backend_ws.recv = AsyncMock(side_effect=ConnectionClosed(None, None)) - logging_obj: Final = MagicMock() + logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() logging_obj.dispatch_success_handlers = AsyncMock() @@ -3686,7 +3690,7 @@ async def test_transcription_guardrail_sends_no_session_update_after_transcripti streaming: Final = RealTimeStreaming( client_ws, backend_ws, - MagicMock(), + _mock_logging(), provider_config=provider_config, model="gpt-4o-transcribe", force_transcription_model="gpt-4o-transcribe", @@ -3721,7 +3725,9 @@ async def test_provider_transcription_session_created_flags_session_without_inte backend_ws: Final = MagicMock() backend_ws.recv = AsyncMock(side_effect=[json.dumps(session_created).encode(), ConnectionClosed(None, None)]) backend_ws.send = AsyncMock() - streaming: Final = RealTimeStreaming(client_ws, backend_ws, MagicMock(), provider_config=provider_config, model="m") + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, _mock_logging(), provider_config=provider_config, model="m" + ) await streaming.backend_to_client_send_messages() @@ -3840,7 +3846,7 @@ async def test_transcription_session_guardrail_block_only_reports_violation( streaming: Final = RealTimeStreaming( client_ws, backend_ws, - MagicMock(), + _mock_logging(), provider_config=_passthrough_transcription_config() if uses_provider_config else None, model="gpt-4o-transcribe", force_transcription_model="gpt-4o-transcribe", diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index d6986a2697a..a705c169782 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -265,6 +265,7 @@ async def test_vertex_realtime_text_in_text_out(): VertexAIRealtimeConfig for message translation. All I/O is mocked. """ from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming + from litellm.litellm_core_utils.logging_worker import LoggingWorker cfg = VertexAIRealtimeConfig( access_token="fake-token", @@ -314,7 +315,8 @@ async def test_vertex_realtime_text_in_text_out(): backend_ws.send = AsyncMock(side_effect=_backend_send) - logging_obj = MagicMock() + logging_obj: Final = MagicMock(dispatch_success_handlers=AsyncMock(), dispatch_failure_handlers=AsyncMock()) + logging_worker: Final = LoggingWorker() logging_obj.litellm_trace_id = "test-trace-id" logging_obj.pre_call = MagicMock() logging_obj.async_success_handler = AsyncMock() @@ -324,6 +326,7 @@ async def test_vertex_realtime_text_in_text_out(): websocket=client_ws, backend_ws=backend_ws, logging_obj=logging_obj, + logging_worker=logging_worker, provider_config=cfg, model="gemini-2.0-flash-live-001", ) @@ -331,6 +334,9 @@ async def test_vertex_realtime_text_in_text_out(): # Run backend→client forwarding for the three queued messages, then stop. # We don't run client_ack_messages here to avoid the blocking receive loop. await streaming.backend_to_client_send_messages() + await logging_worker.flush() + await logging_worker.stop() + logging_obj.dispatch_success_handlers.assert_awaited_once_with(streaming.messages, prefer_async_handlers=True) # --- Assertions --- diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index d86bdd84a62..58c9a5e4d76 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -140,6 +140,7 @@ def _flush_caches(targets): _LITELLM_STATE = _snapshot_mutable_state(litellm) _PROXY_SERVER_STATE = _snapshot_mutable_state(litellm.proxy.proxy_server) _FLUSHABLE_CACHES = _collect_flushable_caches() +_PROXY_APP_LIFESPAN: Final = litellm.proxy.proxy_server.app.router.lifespan_context @pytest.fixture(scope="function", autouse=True) @@ -383,6 +384,7 @@ def create_proxy_test_client( ) cleanup_router_config_variables() + monkeypatch.setattr(app.router, "lifespan_context", _PROXY_APP_LIFESPAN) filepath = os.path.dirname(os.path.abspath(__file__)) default_config_fp = os.path.join( diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index bf5deb4d362..14e5e76c692 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -5,6 +5,7 @@ import time from collections.abc import Awaitable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime, timedelta +from importlib import import_module from types import MappingProxyType, SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -22,7 +23,6 @@ from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64 from litellm.models.credentials import CredentialItem from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.router import Router from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, _show_no_redis_warning, @@ -33,12 +33,14 @@ from litellm.proxy.health_endpoints._health_endpoints import ( from litellm.proxy.health_endpoints._health_endpoints import ( test_model_connection as health_test_model_connection, ) +from litellm.router import Router from litellm.types.utils import oauth_token_exchange_litellm_params from litellm.types.workload_identity import ( ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS, WIF_SECRET_BEARING_KEYS, ) +from tests._master_key import MASTER_KEY # Import shared proxy test helpers from conftest from tests.unit.proxy.conftest import create_proxy_test_client @@ -1633,8 +1635,23 @@ def test_health_liveness_endpoint(proxy_client): # Verify response is fast (should be < 100ms for a simple endpoint) assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint" - # Log the duration for visibility (useful for CI/CD monitoring) - print(f"\n/health/liveness response time: {duration_ms:.2f}ms") + + +def test_proxy_client_serves_full_proxy_routes_after_gateway_import(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.proxy_server import app + + monkeypatch.setattr(app.router, "lifespan_context", app.router.lifespan_context) + import_module("gateway.main") + + with create_proxy_test_client(monkeypatch) as client: + response: Final = client.post( + "/utils/token_counter", + headers={"Authorization": f"Bearer {MASTER_KEY}"}, + json={"model": "gpt-4o", "prompt": "hello"}, + ) + + assert response.status_code == 200, response.text + assert response.json()["total_tokens"] > 0 def test_health_backlog_includes_admission_control_stats(proxy_client): @@ -1678,7 +1695,6 @@ def test_health_readiness(proxy_client): assert set(response_data.keys()) == {"status", "db"} assert response_data["status"] == "healthy" assert response_data["db"] in {"connected", "disconnected", "Not connected"} - print(f"Response time: {duration_ms:.2f}ms") def test_health_readiness_details_returns_diagnostic_fields(monkeypatch): @@ -2597,7 +2613,6 @@ async def test_health_endpoint_keeps_federation_identity_admin_only(): def test_no_server_owned_identity_field_reaches_a_non_admin_health_entry(server_owned_field: str): """Every server-owned identity field is hidden from non-admin health entries.""" from litellm.proxy.health_check import clean_endpoint_data - from litellm.proxy.health_endpoints._health_endpoints import ( _strip_admin_only_fields_from_health_result, ) diff --git a/tests/unit/proxy/middleware/test_gzip_middleware.py b/tests/unit/proxy/middleware/test_gzip_middleware.py index 271ae46bb89..7313d57de81 100644 --- a/tests/unit/proxy/middleware/test_gzip_middleware.py +++ b/tests/unit/proxy/middleware/test_gzip_middleware.py @@ -208,6 +208,14 @@ def test_proxy_app_gzips_large_responses_for_clients_that_accept_it(): compressed = client.get("/openapi.json", headers={"accept-encoding": "gzip"}) identity = client.get("/openapi.json", headers={"accept-encoding": "identity"}) + assert compressed.status_code == identity.status_code == 200, ( + f"gzip={compressed.status_code} {compressed.text[:MINIMUM_SIZE_BYTES]}, " + f"identity={identity.status_code} {identity.text[:MINIMUM_SIZE_BYTES]}" + ) + assert "content-encoding" in compressed.headers, ( + f"status={compressed.status_code}, headers={dict(compressed.headers)}, " + f"body_bytes={len(compressed.content)}, body={compressed.text[:MINIMUM_SIZE_BYTES]}" + ) assert compressed.headers["content-encoding"] == "gzip" assert int(compressed.headers["content-length"]) < int(identity.headers["content-length"]) assert compressed.json() == identity.json() diff --git a/tests/unit/proxy/test_db_schema_changes.py b/tests/unit/proxy/test_db_schema_changes.py index c7685763a24..3a5d0fb1a0a 100644 --- a/tests/unit/proxy/test_db_schema_changes.py +++ b/tests/unit/proxy/test_db_schema_changes.py @@ -1,115 +1,42 @@ +from typing import Final + import pytest -import subprocess -import re -from typing import Dict, List, Set + +from scripts.check_schema_compatibility import check_breaking_changes, parse_model_fields + +ORIGINAL: Final = "model Item {\n id String @id\n label String?\n}\n" -def get_schema_from_branch(branch: str = "main") -> str: - """Get schema from specified git branch""" - result = subprocess.run( - ["git", "show", f"{branch}:schema.prisma"], capture_output=True, text=True - ) - return result.stdout +def test_aaaaaschema_compatibility() -> None: + added: Final = "model Item {\n id String @id\n label String?\n count Int?\n}\nmodel Extra {\n id String @id\n}\n" + assert check_breaking_changes(parse_model_fields(ORIGINAL), parse_model_fields(added)) == () -def parse_model_fields(schema: str) -> Dict[str, Dict[str, str]]: - """Parse Prisma schema into dict of models and their fields""" - models = {} - current_model = None - - for line in schema.split("\n"): - line = line.strip() - - # Find model definition - if line.startswith("model "): - current_model = line.split(" ")[1] - models[current_model] = {} - continue - - # Inside model definition - if current_model and line and not line.startswith("}"): - # Split field definition into name and type - parts = line.split() - if len(parts) >= 2: - field_name = parts[0] - field_type = " ".join(parts[1:]) - models[current_model][field_name] = field_type - - # End of model definition - if line.startswith("}"): - current_model = None - - return models +def test_parse_model_fields_retains_types_and_attributes() -> None: + assert parse_model_fields(ORIGINAL) == {"Item": {"id": "String @id", "label": "String?"}} -def check_breaking_changes( - old_schema: Dict[str, Dict[str, str]], new_schema: Dict[str, Dict[str, str]] -) -> List[str]: - """Check for breaking changes between schemas""" - breaking_changes = [] - - # Check each model in old schema - for model_name, old_fields in old_schema.items(): - if model_name not in new_schema: - breaking_changes.append(f"Breaking: Model {model_name} was removed") - continue - - new_fields = new_schema[model_name] - - # Check each field in old model - for field_name, old_type in old_fields.items(): - if field_name not in new_fields: - breaking_changes.append( - f"Breaking: Field {model_name}.{field_name} was removed" - ) - continue - - new_type = new_fields[field_name] - - # Check for type changes - if old_type != new_type: - # Check specific type changes that are breaking - if "?" in old_type and "?" not in new_type: - breaking_changes.append( - f"Breaking: Field {model_name}.{field_name} changed from optional to required" - ) - if not old_type.startswith(new_type.split("?")[0]): - breaking_changes.append( - f"Breaking: Field {model_name}.{field_name} changed type from {old_type} to {new_type}" - ) - - return breaking_changes +def test_parse_model_fields_keeps_json_defaults_and_empty_models() -> None: + schema: Final = 'model Item {\n data Json @default("{}")\n}\nmodel Empty {\n}\n' + assert parse_model_fields(schema) == {"Item": {"data": 'Json @default("{}")'}, "Empty": {}} -def test_aaaaaschema_compatibility(): - """Test if current schema has breaking changes compared to main""" - import os - - print("Current directory:", os.getcwd()) - - # Get schemas - old_schema = get_schema_from_branch("main") - with open("./schema.prisma", "r") as f: - new_schema = f.read() - - # Parse schemas - old_models = parse_model_fields(old_schema) - new_models = parse_model_fields(new_schema) - - # Check for breaking changes - breaking_changes = check_breaking_changes(old_models, new_models) - - # Fail if breaking changes found - if breaking_changes: - pytest.fail("\n".join(breaking_changes)) - - # Print informational diff - print("\nNon-breaking changes detected:") - for model_name, new_fields in new_models.items(): - if model_name not in old_models: - print(f"Added new model: {model_name}") - continue - - for field_name, new_type in new_fields.items(): - if field_name not in old_models[model_name]: - print(f"Added new field: {model_name}.{field_name}") +@pytest.mark.parametrize( + ("schema", "expected"), + ( + ("", ("Breaking: Model Item was removed",)), + ("model Item {\n id String @id\n}\n", ("Breaking: Field Item.label was removed",)), + ( + "model Item {\n id String @id\n label String\n}\n", + ("Breaking: Field Item.label changed from optional to required",), + ), + ( + "model Item {\n id String @id\n label Int?\n}\n", + ("Breaking: Field Item.label changed type from String? to Int?",), + ), + ), +) +def test_schema_compatibility_rejects_removed_models_fields_and_narrowed_types( + schema: str, expected: tuple[str, ...] +) -> None: + assert check_breaking_changes(parse_model_fields(ORIGINAL), parse_model_fields(schema)) == expected diff --git a/tests/unit/proxy/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py index d5c81440383..20a13bdc566 100644 --- a/tests/unit/proxy/test_proxy_utils.py +++ b/tests/unit/proxy/test_proxy_utils.py @@ -1,6 +1,5 @@ import asyncio import json -import os from datetime import datetime from typing import Any, Dict, Final, List, Optional, Union from unittest.mock import Mock @@ -765,17 +764,19 @@ async def test_prepare_key_update_data(): ({"NO_REDOC": "True"}, None), # Redoc disabled ], ) -def test_get_redoc_url(env_vars, expected_url): - # Clear relevant environment variables - for key in ["REDOC_URL", "NO_REDOC"]: - os.environ.pop(key, None) +def test_get_redoc_url(env_vars: dict[str, str], expected_url: str | None) -> None: + original_url: Final = get_redoc_url() + with pytest.MonkeyPatch.context() as environment: + for key in ("REDOC_URL", "NO_REDOC"): + environment.delenv(key, raising=False) - # Set test environment variables - for key, value in env_vars.items(): - os.environ[key] = value + for key, value in env_vars.items(): + environment.setenv(key, value) - result = get_redoc_url() - assert result == expected_url + result: Final = get_redoc_url() + assert result == expected_url + + assert get_redoc_url() == original_url @pytest.mark.parametrize( @@ -790,17 +791,19 @@ def test_get_redoc_url(env_vars, expected_url): ({"NO_DOCS": "True"}, None), # docs disabled ], ) -def test_get_docs_url(env_vars, expected_url): - # Clear relevant environment variables - for key in ["DOCS_URL", "NO_DOCS"]: - os.environ.pop(key, None) +def test_get_docs_url(env_vars: dict[str, str], expected_url: str | None) -> None: + original_url: Final = get_docs_url() + with pytest.MonkeyPatch.context() as environment: + for key in ("DOCS_URL", "NO_DOCS"): + environment.delenv(key, raising=False) - # Set test environment variables - for key, value in env_vars.items(): - os.environ[key] = value + for key, value in env_vars.items(): + environment.setenv(key, value) - result = get_docs_url() - assert result == expected_url + result: Final = get_docs_url() + assert result == expected_url + + assert get_docs_url() == original_url @pytest.mark.parametrize( @@ -815,17 +818,19 @@ def test_get_docs_url(env_vars, expected_url): ({"NO_OPENAPI": "True"}, None), # openapi disabled ], ) -def test_get_openapi_url(env_vars, expected_url): - # Clear relevant environment variables - for key in ["OPENAPI_URL", "NO_OPENAPI"]: - os.environ.pop(key, None) +def test_get_openapi_url(env_vars: dict[str, str], expected_url: str | None) -> None: + original_url: Final = get_openapi_url() + with pytest.MonkeyPatch.context() as environment: + for key in ("OPENAPI_URL", "NO_OPENAPI"): + environment.delenv(key, raising=False) - # Set test environment variables - for key, value in env_vars.items(): - os.environ[key] = value + for key, value in env_vars.items(): + environment.setenv(key, value) - result = get_openapi_url() - assert result == expected_url + result: Final = get_openapi_url() + assert result == expected_url + + assert get_openapi_url() == original_url @pytest.mark.parametrize( @@ -1881,11 +1886,11 @@ async def test_health_check_not_called_when_disabled(monkeypatch): } }, ) -def test_custom_openapi(mock_get_openapi_schema, monkeypatch): +def test_custom_openapi(mock_get_openapi_schema: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.proxy_server import app, custom_openapi monkeypatch.setattr(app, "openapi_schema", None) - openapi_schema = custom_openapi() + openapi_schema: Final = custom_openapi() assert openapi_schema is not None diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index a692f38ec04..9d11d2191fa 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -14,7 +14,6 @@ from litellm.models.credentials import CredentialItem from litellm.realtime_api import main as realtime_main from litellm.realtime_api.main import _with_resolved_session_model from litellm.types.guardrails import GuardrailEventHooks -from typing import List @pytest.fixture @@ -728,11 +727,14 @@ async def _build_streaming(client_events, backend_ws): client_ws.send_text = send_text client_ws.receive_text = input_queue.get - logging_obj = MagicMock() - logging_obj.pre_call = MagicMock() - logging_obj.async_success_handler = AsyncMock() - logging_obj.success_handler = MagicMock() - logging_obj.model_call_details = {} + logging_obj: Final = MagicMock( + pre_call=MagicMock(), + async_success_handler=AsyncMock(), + success_handler=MagicMock(), + dispatch_success_handlers=AsyncMock(), + dispatch_failure_handlers=AsyncMock(), + model_call_details={}, + ) streaming = RealTimeStreaming( websocket=client_ws, @@ -754,7 +756,7 @@ async def test_voice_transcript_blocked_by_guardrail(): guardrail = _make_guardrail(GuardrailEventHooks.realtime_input_transcription) litellm.callbacks = [guardrail] - client_events: List[dict] = [] + client_events: list[dict] = [] # Build the transcript event that would come from the OpenAI backend transcript_event = json.dumps( diff --git a/tests/unit/responses/mcp/conftest.py b/tests/unit/responses/mcp/conftest.py new file mode 100644 index 00000000000..c41ba624ac8 --- /dev/null +++ b/tests/unit/responses/mcp/conftest.py @@ -0,0 +1,8 @@ +import importlib + +import pytest + + +@pytest.fixture(scope="module", autouse=True) +def import_mcp_server_before_manager_doubles() -> None: + importlib.import_module("litellm.proxy._experimental.mcp_server.server") diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 4498dbe1498..4c9fe1739e6 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -461,3 +461,50 @@ def test_a_workflow_does_not_credit_a_file_it_never_names( assert not any( coverage._token_covers(token, "tests/local_testing/test_unrun.py") for token in named ) + + +def test_slice_audit_rechecks_changed_and_new_files_between_calls( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workflows: Final = tmp_path / ".github/workflows" + workflows.mkdir(parents=True) + tests: Final = tmp_path / "tests/local_testing" + tests.mkdir(parents=True) + circle: Final = tmp_path / ".circleci/config.yml" + circle.parent.mkdir() + circle.write_text( + yaml.safe_dump( + { + "jobs": { + selector: { + "steps": [ + { + "run": f'circleci tests glob "tests/local_testing/test_*.py" | xargs pytest -k "{selector}"' + } + ] + } + for selector in ("selected", "neighbor") + } + } + ) + ) + monkeypatch.setattr(coverage, "REPO_ROOT", tmp_path) + monkeypatch.setattr(coverage, "TESTS_ROOT", tmp_path / "tests") + monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) + monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", circle) + allowlist: Final = coverage.Allowlist((), ()) + existing: Final = tests / "test_existing.py" + existing.write_text("def test_selected(): pass\n") + assert coverage._deselected_everywhere(allowlist) == () + + existing.write_text("def test_uncovered(): pass\n") + assert tuple(finding.subject for finding in coverage._deselected_everywhere(allowlist)) == ( + "tests/local_testing/test_existing.py", + ) + + existing.write_text("def test_selected(): pass\n") + added: Final = tests / "test_added.py" + added.write_text("def test_uncovered(): pass\n") + assert tuple(finding.subject for finding in coverage._deselected_everywhere(allowlist)) == ( + "tests/local_testing/test_added.py", + ) diff --git a/tests/unit/test_conftest_isolation.py b/tests/unit/test_conftest_isolation.py index 15183e68f66..63db0eb5bba 100644 --- a/tests/unit/test_conftest_isolation.py +++ b/tests/unit/test_conftest_isolation.py @@ -1,11 +1,80 @@ +import asyncio +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from threading import Thread +from typing import Final, Literal + +import pytest + import litellm from litellm import Router from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module +from litellm.caching.caching import DualCache +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from tests.unit import conftest as unit_harness CANARY_MODEL = "conftest-isolation-canary-model" +@dataclass(slots=True) +class _LoggingOwner: + current: str = "original-test" + delivered: tuple[str, ...] = () + + async def deliver(self) -> None: + self.delivered = (*self.delivered, self.current) + + +@pytest.mark.parametrize("loop_state", ("stopped", "running", "closed")) +def test_queued_logging_finishes_under_its_original_owner(loop_state: Literal["stopped", "running", "closed"]) -> None: + owner: Final = _LoggingOwner() + loop: Final = asyncio.new_event_loop() + + async def enqueue() -> None: + GLOBAL_LOGGING_WORKER._ensure_queue() + GLOBAL_LOGGING_WORKER.enqueue(owner.deliver()) + + thread: Final = Thread(target=loop.run_forever) if loop_state == "running" else None + if thread is not None: + thread.start() + asyncio.run_coroutine_threadsafe(enqueue(), loop).result() + else: + loop.run_until_complete(enqueue()) + if loop_state == "closed": + loop.close() + try: + unit_harness._flush_completed_test_logging() + owner.current = "next-test" + assert owner.delivered == ("original-test",) + finally: + if thread is not None: + asyncio.run_coroutine_threadsafe(GLOBAL_LOGGING_WORKER.stop(), loop).result() + loop.call_soon_threadsafe(loop.stop) + thread.join() + elif not loop.is_closed(): + loop.run_until_complete(GLOBAL_LOGGING_WORKER.stop()) + if not loop.is_closed(): + loop.close() + + +def test_dequeued_logging_finishes_under_its_original_owner() -> None: + owner: Final = _LoggingOwner() + + async def enqueue() -> None: + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(owner.deliver()) + + asyncio.run(enqueue()) + assert GLOBAL_LOGGING_WORKER._queue is not None + assert GLOBAL_LOGGING_WORKER._queue.empty() + assert owner.delivered == () + unit_harness._flush_completed_test_logging() + owner.current = "next-test" + assert owner.delivered == ("original-test",) + + class _CanaryRouterHolder: router: Router | None = None @@ -34,3 +103,67 @@ def test_live_router_membership_is_scoped_to_this_test(): def test_live_router_membership_was_rolled_back(): assert _CanaryRouterHolder.router is not None assert _CanaryRouterHolder.router not in litellm_router_module._live_routers + + +def test_aws_cache_reset_sees_new_handlers_and_replaced_caches(monkeypatch: pytest.MonkeyPatch) -> None: + module: Final = unit_harness.importlib.import_module("litellm.files.main") + handler: Final = BaseAWSLLM() + first_cache: Final = DualCache() + handler.iam_cache = first_cache + monkeypatch.setattr(module, "unit_reset_canary", handler, raising=False) + first_cache.set_cache("unit-reset-key", "first-secret") + unit_harness._reset_aws_auth_caches() + assert first_cache.get_cache("unit-reset-key") is None + second_cache: Final = DualCache() + handler.iam_cache = second_cache + second_cache.set_cache("unit-reset-key", "second-secret") + unit_harness._reset_aws_auth_caches() + assert second_cache.get_cache("unit-reset-key") is None + + +def test_aws_cache_reset_sees_a_replaced_module(monkeypatch: pytest.MonkeyPatch) -> None: + name: Final = "litellm.files.main" + original_module: Final = unit_harness.importlib.import_module(name) + package: Final = unit_harness.importlib.import_module("litellm.files") + monkeypatch.delitem(sys.modules, name) + monkeypatch.setattr(package, "main", original_module) + module: Final = unit_harness.importlib.import_module(name) + handler: Final = BaseAWSLLM() + cache: Final = DualCache() + handler.iam_cache = cache + monkeypatch.setattr(module, "unit_reset_canary", handler, raising=False) + cache.set_cache("unit-reset-key", "module-secret") + unit_harness._reset_aws_auth_caches() + assert cache.get_cache("unit-reset-key") is None + + +@pytest.fixture(scope="module", autouse=True) +def registered_collection_models() -> Iterator[None]: + original_registrations: Final = dict(litellm_utils_module._runtime_registered_model_cost) + original_routers: Final = frozenset(litellm_router_module._live_routers) + litellm.register_model( + {"collection-isolation-ledger": {"litellm_provider": "openai", "input_cost_per_token": 0.001}} + ) + collection_router: Final = Router( + model_list=[ + { + "model_name": "collection-isolation-router", + "litellm_params": {"model": "openai/collection-isolation-backend", "api_key": "sk-canary"}, + } + ] + ) + yield + litellm_utils_module._runtime_registered_model_cost.clear() + litellm_utils_module._runtime_registered_model_cost.update(original_registrations) + litellm_router_module._live_routers.discard(collection_router) + litellm_router_module._live_routers.update(original_routers) + + +def test_collection_model_metadata_is_not_replayed_into_a_test_catalog() -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + catalog: Final = {"unit-test-catalog-model": {"litellm_provider": "openai", "input_cost_per_token": 0.001}} + adopt_model_cost_map(catalog) + assert "collection-isolation-ledger" not in litellm.model_cost + assert "collection-isolation-router" not in litellm.model_cost + assert "unit-test-catalog-model" in litellm.model_cost From 65831b8de9e64b93200c108974a7c666de0a336e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 10 Oct 2026 11:18:32 -0700 Subject: [PATCH 03/96] fix(ui): load embedded Lens styles in the gateway (#45829) Load the Lens package stylesheet so embedded pages retain their toolbar layout, header alignment, and content margins. --- ui/litellm-dashboard/src/app/globals.css | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index 90ce92399a0..57b8d0108ac 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -1,6 +1,7 @@ @layer theme, base, components, utilities; @import "tailwindcss"; +@import "@litellm/lens-ui/styles.css"; @source "../../node_modules/@litellm/lens-ui/src"; @import "tw-animate-css"; From 33b9d846b039e26bc9de4af04ac56f9584e20e61 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:23:30 -0700 Subject: [PATCH 04/96] fix(ui): expose both decisions playground endpoints (#45817) * fix(ui): simplify playground decision endpoint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): expose both decision API shapes in playground Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): omit undiscovered decision models from examples Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): align decisions form and response views Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../systemOneUI/DecisionFormControls.tsx | 43 ++ .../components/systemOneUI/JsonEditor.tsx | 22 +- .../systemOneUI/OpenAIDecisionsForm.tsx | 267 +++++++++++ ....tsx => ResponseView.integration.test.tsx} | 65 ++- .../components/systemOneUI/ResponseView.tsx | 132 ++++-- .../components/systemOneUI/SystemOneForm.tsx | 43 +- .../SystemOneUI.integration.test.tsx | 432 +++++++++++++++--- .../components/systemOneUI/SystemOneUI.tsx | 140 +++--- .../systemOneUI/lib/decisions.test.ts | 53 ++- .../components/systemOneUI/lib/example.ts | 33 +- .../systemOneUI/lib/openAIDecisions.ts | 98 ++++ .../lib/openAIDecisionsForm.test.ts | 79 ++++ .../systemOneUI/lib/openAIDecisionsForm.ts | 54 +++ .../systemOneUI/lib/validatePayload.ts | 33 +- .../playground/llm_calls/system_one.test.ts | 94 +++- .../playground/llm_calls/system_one.ts | 58 ++- 16 files changed, 1423 insertions(+), 223 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/DecisionFormControls.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/OpenAIDecisionsForm.tsx rename ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/{ResponseView.test.tsx => ResponseView.integration.test.tsx} (57%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisionsForm.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisionsForm.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/DecisionFormControls.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/DecisionFormControls.tsx new file mode 100644 index 00000000000..af821e77916 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/DecisionFormControls.tsx @@ -0,0 +1,43 @@ +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/cva.config"; +import { Plus, Trash2 } from "lucide-react"; +import { useId, type ReactNode } from "react"; +import { IssueList, ValidationStatus } from "./JsonEditor"; +import type { PayloadValidation } from "./lib/validatePayload"; + +export function RemoveButton({ label, onClick }: { label: string; onClick: () => void }) { + return ( + + ); +} + +export function AddButton({ children, onClick }: { children: ReactNode; onClick: () => void }) { + return ( + + ); +} + +export function FormFrame({ validation, children }: { validation: PayloadValidation; children: ReactNode }) { + const issuesId = useId(); + + return ( +
+
+ Request form + +
+
{children}
+ +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/JsonEditor.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/JsonEditor.tsx index cc7bebb0bb8..7311c3a055a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/JsonEditor.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/JsonEditor.tsx @@ -7,7 +7,7 @@ import { createElement, PrismLight as SyntaxHighlighter } from "react-syntax-hig import type { SyntaxHighlighterProps } from "react-syntax-highlighter"; import json from "react-syntax-highlighter/dist/esm/languages/prism/json"; import { vs, vscDarkPlus } from "react-syntax-highlighter/dist/esm/styles/prism"; -import type { SystemOnePayloadValidation } from "./lib/validatePayload"; +import type { PayloadValidation } from "./lib/validatePayload"; SyntaxHighlighter.registerLanguage("json", json); @@ -57,10 +57,12 @@ const TRANSPARENT_PRE = { interface JsonEditorProps { value: string; onChange: (value: string) => void; - validation: SystemOnePayloadValidation; + validation: PayloadValidation; + label?: string; + placeholder?: string; } -export function ValidationStatus({ validation }: { validation: SystemOnePayloadValidation }) { +export function ValidationStatus({ validation }: { validation: PayloadValidation }) { const errorCount = validation.issues.filter((issue) => issue.severity === "error").length; if (errorCount > 0) { return ( @@ -72,7 +74,7 @@ export function ValidationStatus({ validation }: { validation: SystemOnePayloadV return Valid payload; } -export function IssueList({ id, validation }: { id: string; validation: SystemOnePayloadValidation }) { +export function IssueList({ id, validation }: { id: string; validation: PayloadValidation }) { if (validation.issues.length === 0) { return (

@@ -112,7 +114,13 @@ function LineRows({ rows, stylesheet, useInlineStyles }: LineRendererProps) { }); } -export default function JsonEditor({ value, onChange, validation }: JsonEditorProps) { +export default function JsonEditor({ + value, + onChange, + validation, + label = "System One JSON payload", + placeholder = "Paste or write a System One request", +}: JsonEditorProps) { const issuesId = useId(); const highlightRef = useRef(null); const syntaxTheme = useTheme().resolvedTheme === "dark" ? VS_CODE_DARK_PLUS : VS_CODE_LIGHT_PLUS; @@ -156,7 +164,7 @@ export default function JsonEditor({ value, onChange, validation }: JsonEditorPr