mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-09 08:18:12 +00:00
Merge remote-tracking branch 'origin/main' into litellm_mixpanel_otel_preset
; Conflicts: ; tests/unit/integrations/otel/test_otel_v2_vendor_mappers.py ; tests/unit/litellm_core_utils/test_litellm_logging.py
This commit is contained in:
Nausherwan Korai
commit
357b2a7fe4
556 changed files with 18545 additions and 8093 deletions
49
.github/workflows/spend-logs-guard.yml
vendored
Normal file
49
.github/workflows/spend-logs-guard.yml
vendored
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
name: Spend logs guard
|
||||
|
||||
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
|
||||
pull_request_target:
|
||||
branches:
|
||||
- main
|
||||
pull_request_review:
|
||||
types:
|
||||
- submitted
|
||||
- dismissed
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
spend-logs-guard:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Run the guard
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
uv run --frozen python ci_cd/spend_logs_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --approver kerry-berri --reviews "$RUNNER_TEMP/reviews.json"
|
||||
6
.github/workflows/test-linting.yml
vendored
6
.github/workflows/test-linting.yml
vendored
|
|
@ -524,6 +524,12 @@ jobs:
|
|||
- name: check_provider_folders_documented
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
|
||||
|
||||
- name: check_provider_endpoints_support_generated
|
||||
run: uv run --no-sync python scripts/generate_provider_endpoints_support.py --check
|
||||
|
||||
- name: test_generate_provider_endpoints_support
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_generate_provider_endpoints_support.py
|
||||
|
||||
- name: check_prisma_binary_cache
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
|
||||
|
||||
|
|
|
|||
56
.github/workflows/test-unit.yml
vendored
56
.github/workflows/test-unit.yml
vendored
|
|
@ -579,13 +579,13 @@ jobs:
|
|||
|
||||
- shard: mcp-elicitation
|
||||
test-path: |-
|
||||
tests/unit/experimental_mcp_client/test_mcp_client.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_interactions.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_operations.py
|
||||
tests/unit/mcp_client/test_mcp_client.py
|
||||
tests/unit/proxy/mcp_server/test_capabilities.py
|
||||
tests/unit/proxy/mcp_server/test_interactions.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_server_manager.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py
|
||||
tests/unit/proxy/mcp_server/test_operations.py
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
reruns: 0
|
||||
|
|
@ -603,7 +603,7 @@ jobs:
|
|||
tests/unit/proxy/spend_tracking
|
||||
--ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py
|
||||
tests/unit/proxy/pass_through_endpoints
|
||||
tests/unit/proxy/experimental
|
||||
tests/unit/proxy/mcp_server/test_tool_registry.py
|
||||
tests/unit/proxy/common_utils
|
||||
--ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py
|
||||
--ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py
|
||||
|
|
@ -811,14 +811,14 @@ jobs:
|
|||
|
||||
- shard: mcp-oauth
|
||||
test-path: |-
|
||||
tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py
|
||||
tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py
|
||||
tests/unit/proxy/_experimental/mcp_server/outbound_credentials
|
||||
tests/unit/proxy/mcp_server/test_discoverable_endpoints.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_server_manager.py
|
||||
tests/unit/proxy/mcp_server/test_db_credentials.py
|
||||
tests/unit/proxy/mcp_server/test_gateway_dcr_flow.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_oauth_passthrough_tools.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_max_concurrent_requests.py
|
||||
tests/unit/proxy/mcp_server/faults/test_list_outcomes.py
|
||||
tests/unit/proxy/mcp_server/outbound_credentials
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
reruns: 2
|
||||
|
|
@ -828,19 +828,21 @@ jobs:
|
|||
|
||||
- shard: mcp-server
|
||||
test-path: |-
|
||||
tests/unit/mcp_client
|
||||
tests/unit/experimental_mcp_client
|
||||
tests/unit/proxy/mcp_server
|
||||
tests/unit/proxy/_experimental/mcp_server
|
||||
tests/unit/responses/mcp
|
||||
--ignore=tests/unit/experimental_mcp_client/test_mcp_client.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials
|
||||
--ignore=tests/unit/mcp_client/test_mcp_client.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_capabilities.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_interactions.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_mcp_elicitation_handler.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_mcp_server_manager.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_operations.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_discoverable_endpoints.py
|
||||
--ignore=tests/unit/proxy/mcp_server/test_db_credentials.py
|
||||
--ignore=tests/unit/proxy/mcp_server/outbound_credentials
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
reruns: 0
|
||||
|
|
@ -950,7 +952,7 @@ jobs:
|
|||
dist: loadscope
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
timeout-minutes: 3
|
||||
timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 10 || 3 }}
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }}
|
||||
|
|
|
|||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -105,7 +105,7 @@ litellm_config.yaml
|
|||
.cursor
|
||||
litellm/proxy/to_delete_loadtest_work/*
|
||||
update_model_cost_map.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_server_manager.py
|
||||
scripts/test_vertex_ai_search.py
|
||||
LAZY_LOADING_IMPROVEMENTS.md
|
||||
STABILIZATION_TODO.md
|
||||
|
|
|
|||
|
|
@ -336,9 +336,9 @@ Each translation is isolated in its own file, making it easy to test and modify
|
|||
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
|
||||
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
|
||||
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/messages/native/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py` |
|
||||
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
|
||||
|
||||
### Example: Debugging prompt caching
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -313,7 +313,7 @@ test-unit-proxy-core: install-test-deps
|
|||
$(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20
|
||||
|
||||
test-unit-proxy-misc: install-test-deps
|
||||
$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20
|
||||
$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20
|
||||
|
||||
test-unit-proxy-root: install-test-deps
|
||||
$(UV_RUN) pytest tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_aproxy_startup.py --ignore=tests/unit/proxy/test_credential_slot_registry.py --ignore=tests/unit/proxy/test_custom_callback_input.py --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py --ignore=tests/unit/proxy/test_db_schema_changes.py --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py --ignore=tests/unit/proxy/test_get_favicon.py --ignore=tests/unit/proxy/test_get_image.py --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py --ignore=tests/unit/proxy/test_prompt_test_endpoint.py --ignore=tests/unit/proxy/test_proxy_config_unit_test.py --ignore=tests/unit/proxy/test_proxy_custom_auth.py --ignore=tests/unit/proxy/test_proxy_reject_logging.py --ignore=tests/unit/proxy/test_proxy_server.py --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py --ignore=tests/unit/proxy/test_proxy_token_counter.py --ignore=tests/unit/proxy/test_proxy_utils.py --ignore=tests/unit/proxy/test_reducto_ocr_route.py --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py --ignore=tests/unit/proxy/test_server_root_path.py --ignore=tests/unit/proxy/test_ui_path_detection.py --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py --ignore=tests/unit/proxy/test_update_spend.py --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py --tb=short -vv -n 4 --durations=20
|
||||
|
|
@ -325,7 +325,7 @@ test-unit-core-utils: install-test-deps
|
|||
$(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20
|
||||
|
||||
test-unit-other: install-test-deps
|
||||
$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
|
||||
$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20
|
||||
|
||||
test-unit-root: install-test-deps
|
||||
$(UV_RUN) pytest tests/unit/test_*.py --tb=short -vv -n 4 --durations=20
|
||||
|
|
|
|||
|
|
@ -223,7 +223,7 @@ async with httpx.AsyncClient(headers=headers, timeout=60.0) as http_client:
|
|||
```python
|
||||
from mcp import ClientSession, StdioServerParameters
|
||||
from mcp.client.stdio import stdio_client
|
||||
from litellm import experimental_mcp_client
|
||||
from litellm import mcp_client
|
||||
import litellm
|
||||
|
||||
server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
|
||||
|
|
@ -233,7 +233,7 @@ async with stdio_client(server_params) as (read, write):
|
|||
await session.initialize()
|
||||
|
||||
# Load MCP tools in OpenAI format
|
||||
tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
|
||||
tools = await mcp_client.load_mcp_tools(session=session, format="openai")
|
||||
|
||||
# Use with any LiteLLM model
|
||||
response = await litellm.acompletion(
|
||||
|
|
|
|||
119
ci_cd/spend_logs_guard.py
Normal file
119
ci_cd/spend_logs_guard.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""Required check: a pull request that touches LiteLLM_SpendLogs in schema.prisma or a migration
|
||||
needs an approving review from the spend-log owner before it can merge.
|
||||
|
||||
CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
|
||||
so a path rule would put every migration on the owner. This gate reads the diff instead: when no
|
||||
added line in the watched files mentions the table it passes at once, otherwise it passes only once
|
||||
the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
WATCHED: Final = (
|
||||
"schema.prisma",
|
||||
"litellm/proxy/schema.prisma",
|
||||
"litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
|
||||
)
|
||||
MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
|
||||
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
|
||||
|
||||
|
||||
class Reviewer(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
login: str
|
||||
|
||||
|
||||
class Review(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
state: str
|
||||
user: Reviewer
|
||||
|
||||
|
||||
_REVIEWS: Final = TypeAdapter(list[Review])
|
||||
|
||||
|
||||
def watched(path: str) -> bool:
|
||||
return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
|
||||
|
||||
|
||||
def _added_lines(section: str) -> tuple[str, ...]:
|
||||
body: Final = section.split("\n@@", 1)
|
||||
return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
|
||||
|
||||
|
||||
def spend_log_hits(diff: str) -> tuple[str, ...]:
|
||||
"""`path: added line` for every added line in a watched file that names the table."""
|
||||
sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
|
||||
return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
|
||||
|
||||
|
||||
def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
|
||||
pairs: Final = tuple((_path_of(section), section) for section in sections)
|
||||
return tuple(
|
||||
(path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
|
||||
) # comprehension-ok: path then its lines
|
||||
|
||||
|
||||
def _path_of(section: str) -> str:
|
||||
match: Final = re.search(r"^\+\+\+ b/(.+)$", section, flags=re.MULTILINE)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def approved_by(reviews: Sequence[Review], login: str) -> bool:
|
||||
"""True when `login`'s most recent review on the pull request is an approval."""
|
||||
states: Final = tuple(review.state for review in reviews if review.user.login == login)
|
||||
return bool(states) and states[-1] == "APPROVED"
|
||||
|
||||
|
||||
def added_diff(base: str, head: str) -> str:
|
||||
result: Final = subprocess.run(
|
||||
("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
class Arguments(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
base: str
|
||||
head: str
|
||||
approver: str
|
||||
reviews: Path
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
parser: Final = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base", required=True, help="merge base of the pull request")
|
||||
parser.add_argument("--head", required=True, help="head sha of the pull request")
|
||||
parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
|
||||
parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
|
||||
args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
|
||||
hits: Final = spend_log_hits(added_diff(args.base, args.head))
|
||||
if not hits:
|
||||
sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
|
||||
return 0
|
||||
if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
|
||||
sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
|
||||
return 0
|
||||
listed: Final = "".join(f" {hit}\n" for hit in hits)
|
||||
sys.stdout.write(
|
||||
f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
|
||||
f"{listed}The check re-runs when a review is submitted.\n"
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
|
|
@ -18,8 +18,8 @@ from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
|||
from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.proxy._experimental.mcp_server.oauth_utils import get_request_base_url
|
||||
from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.mcp_server.oauth_utils import get_request_base_url
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.proxy.auth.auth_checks import UserNotFoundError
|
||||
|
||||
|
|
@ -122,7 +122,7 @@ def _cookie_name(token: str) -> str:
|
|||
|
||||
|
||||
async def _session_user(request: Request) -> str | None:
|
||||
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import (
|
||||
from litellm.proxy.mcp_server.byok_oauth_endpoints import (
|
||||
get_authenticated_browser_user_id,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,639 @@
|
|||
"""Read-only checks that the rows already in the database can take the pending schema
|
||||
changes, run before `prisma migrate deploy` and before each request-log index build.
|
||||
|
||||
Postgres has a short list of schema statements that fail because of existing rows: a
|
||||
btree index whose entry for some row would not fit in a third of a page (2704 bytes on
|
||||
the default 8 KB page, counting every indexed column), UNIQUE (duplicate values), SET NOT NULL and ADD COLUMN NOT NULL with no default (nulls, any row at all),
|
||||
FOREIGN KEY (orphans) and CHECK (rows failing the expression). Each pending statement
|
||||
of one of those kinds becomes one count query derived from the statement text and the
|
||||
column types in the catalog. A count above zero names the table, the column, the
|
||||
statement it blocks and the SQL that clears it, and the migration is refused before
|
||||
anything is applied. Every other statement a migration can contain cannot fail on
|
||||
existing data and gets no query. A statement the derivation cannot read gets none
|
||||
either, so the migration keeps today's behavior for it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
from litellm_proxy_extras._logging import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
DEFAULT_BLOCK_SIZE: Final = 8192
|
||||
INDEX_TUPLE_HEADER_BYTES: Final = 8
|
||||
NULL_BITMAP_BYTES: Final = 8
|
||||
VARLENA_HEADER_BYTES: Final = 4
|
||||
SHORT_VARLENA_MAX_BYTES: Final = 126
|
||||
_PAGE_HEADER_WITH_THREE_LINE_POINTERS_BYTES: Final = 40
|
||||
_BTREE_PAGE_OPAQUE_BYTES: Final = 16
|
||||
_HEAP_TID_BYTES: Final = 8
|
||||
_ALIGNMENT: Final = MappingProxyType({"c": 1, "s": 2, "i": 4, "d": 8})
|
||||
_OCTET_LENGTH_TYPES: Final = frozenset({"text", "varchar", "bpchar", "bytea", "name"})
|
||||
MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIMEOUT"
|
||||
DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0
|
||||
REFUSAL: Final = "Existing rows would make a pending schema change fail:"
|
||||
LEGACY_SPEND_LOG_KEY_REMEDY: Final = (
|
||||
"Rows written before v1.99.0 hold the raw key. Hash them in place (one-way, idempotent):\n"
|
||||
' UPDATE "LiteLLM_SpendLogs" SET api_key = encode(sha256(convert_to('
|
||||
"regexp_replace(api_key, '^[Bb][Ee][Aa][Rr][Ee][Rr] ', ''), 'UTF8')), 'hex')\n"
|
||||
" WHERE api_key <> '' AND api_key !~ '^[0-9a-fA-F]{64}$' AND api_key !~ '^hashed-jwt-[0-9A-Fa-f]{64}$';"
|
||||
)
|
||||
|
||||
Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Check:
|
||||
kind: Kind
|
||||
table: str
|
||||
columns: tuple[str, ...]
|
||||
statement: str
|
||||
source: str
|
||||
index: str | None = None
|
||||
referenced_table: str | None = None
|
||||
referenced_columns: tuple[str, ...] = ()
|
||||
expression: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BlockedChange:
|
||||
check: Check
|
||||
rows: int
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CouldNotVerify:
|
||||
check: Check
|
||||
timeout_seconds: float
|
||||
|
||||
|
||||
Finding = BlockedChange | CouldNotVerify
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ColumnType:
|
||||
"""One column as `pg_attribute` and `pg_type` describe it: `length` is `attlen` (-1 for a
|
||||
variable-length value), `align` is `attalign` (c, s, i or d), `typmod` is `atttypmod`."""
|
||||
|
||||
type_name: str
|
||||
length: int
|
||||
align: str
|
||||
typmod: int
|
||||
|
||||
@property
|
||||
def is_variable_length(self) -> bool:
|
||||
return self.length == -1
|
||||
|
||||
@property
|
||||
def can_overflow_btree(self) -> bool:
|
||||
"""Whether one value of this column alone can push a btree entry past the limit: a
|
||||
variable-length type with no declared cap, or a cap of more bytes than the limit."""
|
||||
if not self.is_variable_length:
|
||||
return False
|
||||
return self.typmod == -1 or (self.typmod - VARLENA_HEADER_BYTES) * 4 > btree_max_entry_bytes(DEFAULT_BLOCK_SIZE)
|
||||
|
||||
|
||||
ColumnTypeLookup = Callable[[str, str], ColumnType | None]
|
||||
|
||||
_IDENT: Final = r'"?(\w+)"?'
|
||||
_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?'
|
||||
_CREATE_INDEX: Final = re.compile(
|
||||
rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?"
|
||||
rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ALTER_TABLE: Final = re.compile(
|
||||
rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE)
|
||||
_ADD_COLUMN: Final = re.compile(
|
||||
rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?"
|
||||
_ADD_FOREIGN_KEY: Final = re.compile(
|
||||
rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL)
|
||||
_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL)
|
||||
_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE)
|
||||
_COMMENT: Final = re.compile(r"--[^\n]*")
|
||||
_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
|
||||
_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def migration_data_check_timeout_seconds() -> float:
|
||||
"""Seconds one count query may run before the preflight gives up on verifying it."""
|
||||
raw: Final = os.getenv(MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR)
|
||||
if raw is None:
|
||||
return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
|
||||
try:
|
||||
return float(raw)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"%s=%r is not a number, using %s",
|
||||
MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR,
|
||||
raw,
|
||||
DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def statements(migration_sql: str) -> tuple[str, ...]:
|
||||
"""The statements of one migration file, comments stripped, empty ones dropped."""
|
||||
parts: Final = _COMMENT.sub("", migration_sql).split(";")
|
||||
return tuple(part.strip() for part in parts if part.strip())
|
||||
|
||||
|
||||
def _split_top_level(text: str) -> tuple[str, ...]:
|
||||
"""Split on the commas outside parentheses."""
|
||||
depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text)
|
||||
cuts: Final = tuple(
|
||||
position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0
|
||||
)
|
||||
starts: Final = (0, *(cut + 1 for cut in cuts))
|
||||
stops: Final = (*cuts, len(text))
|
||||
pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True))
|
||||
return tuple(piece for piece in pieces if piece)
|
||||
|
||||
|
||||
def _plain_columns(column_list: str) -> tuple[str, ...] | None:
|
||||
"""The column names of an index or constraint column list, None when one entry is an
|
||||
expression the derivation cannot turn into a count."""
|
||||
matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list))
|
||||
if any(match is None for match in matches):
|
||||
return None
|
||||
return tuple(match.group(1) for match in matches if match is not None)
|
||||
|
||||
|
||||
def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]:
|
||||
unique, name, table, method, column_list = match.groups()
|
||||
columns: Final = _plain_columns(column_list)
|
||||
if columns is None:
|
||||
return
|
||||
label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"'
|
||||
if method is None or method.lower() == "btree":
|
||||
yield Check("too_long", table, columns, label, source, index=name)
|
||||
if unique:
|
||||
yield Check("duplicates", table, columns, label, source, index=name)
|
||||
|
||||
|
||||
def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]:
|
||||
if (set_not_null := _SET_NOT_NULL.match(action)) is not None:
|
||||
yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source)
|
||||
return
|
||||
if (add_column := _ADD_COLUMN.match(action)) is not None:
|
||||
column, body = add_column.groups()
|
||||
if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None:
|
||||
yield Check(
|
||||
"any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source
|
||||
)
|
||||
return
|
||||
if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None:
|
||||
constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups()
|
||||
own: Final = _plain_columns(columns)
|
||||
theirs: Final = _plain_columns(referenced_columns)
|
||||
if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper():
|
||||
return
|
||||
yield Check(
|
||||
"orphans",
|
||||
table,
|
||||
own,
|
||||
_constraint_label(table, constraint, "FOREIGN KEY"),
|
||||
source,
|
||||
referenced_table=referenced_table,
|
||||
referenced_columns=theirs,
|
||||
)
|
||||
return
|
||||
if (unique := _ADD_UNIQUE.match(action)) is not None:
|
||||
constraint, columns = unique.groups()
|
||||
unique_columns: Final = _plain_columns(columns)
|
||||
if unique_columns is not None:
|
||||
yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source)
|
||||
return
|
||||
if (check := _ADD_CHECK.match(action)) is not None:
|
||||
constraint, expression, not_valid = check.groups()
|
||||
if not_valid is None:
|
||||
yield Check(
|
||||
"check_violations",
|
||||
table,
|
||||
(),
|
||||
_constraint_label(table, constraint, "CHECK"),
|
||||
source,
|
||||
expression=expression,
|
||||
)
|
||||
|
||||
|
||||
def _constraint_label(table: str, constraint: str | None, kind: str) -> str:
|
||||
named: Final = f'CONSTRAINT "{constraint}" ' if constraint else ""
|
||||
return f'ALTER TABLE "{table}" ADD {named}{kind}'
|
||||
|
||||
|
||||
def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
|
||||
"""The count queries one SQL statement needs before it can be applied, empty for a
|
||||
statement that cannot fail on existing rows or that the derivation cannot read."""
|
||||
if (index := _CREATE_INDEX.match(statement_text)) is not None:
|
||||
return tuple(_index_checks(index, statement_text, source))
|
||||
if (alter := _ALTER_TABLE.match(statement_text)) is not None:
|
||||
table, actions = alter.groups()
|
||||
return tuple(
|
||||
check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source)
|
||||
) # comprehension-ok: flattening the per-action checks of one statement
|
||||
return ()
|
||||
|
||||
|
||||
def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
|
||||
return tuple(
|
||||
check
|
||||
for statement_text in statements(migration_sql)
|
||||
for check in checks_for_statement(statement_text, f"migration {name}")
|
||||
) # comprehension-ok: flattening the per-statement checks of one migration
|
||||
|
||||
|
||||
def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]:
|
||||
"""The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`."""
|
||||
return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),)
|
||||
|
||||
|
||||
def _applied_migrations(connection: "psycopg.Connection[tuple[object, ...]]", schema: str) -> frozenset[str]:
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
try:
|
||||
rows: Final = connection.execute(
|
||||
sql.SQL("SELECT migration_name FROM {} WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL").format(
|
||||
sql.Identifier(schema, "_prisma_migrations")
|
||||
)
|
||||
).fetchall()
|
||||
except psycopg.errors.UndefinedTable:
|
||||
return frozenset()
|
||||
return frozenset(str(row[0]) for row in rows)
|
||||
|
||||
|
||||
def pending_migrations(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]", schema: str, migrations_dir: Path
|
||||
) -> tuple[tuple[str, str], ...]:
|
||||
"""(name, sql) of every shipped migration the ledger has not finished, oldest first.
|
||||
Without a ledger every migration is pending, and the checks skip tables that do not exist yet."""
|
||||
applied: Final = _applied_migrations(connection, schema)
|
||||
folders: Final = tuple(folder for folder in sorted(migrations_dir.iterdir()) if folder.is_dir())
|
||||
return tuple(
|
||||
(folder.name, (folder / "migration.sql").read_text())
|
||||
for folder in folders
|
||||
if folder.name not in applied and (folder / "migration.sql").is_file()
|
||||
)
|
||||
|
||||
|
||||
def _int(value: object, default: int) -> int:
|
||||
return value if isinstance(value, int) else default
|
||||
|
||||
|
||||
def column_type_lookup(connection: "psycopg.Connection[tuple[object, ...]]", schema: str) -> ColumnTypeLookup:
|
||||
def lookup(table: str, column: str) -> ColumnType | None:
|
||||
row: Final = connection.execute(
|
||||
"SELECT t.typname, a.attlen, a.attalign, a.atttypmod FROM pg_attribute a "
|
||||
"JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace "
|
||||
"JOIN pg_type t ON t.oid = a.atttypid "
|
||||
"WHERE n.nspname = %s AND c.relname = %s AND a.attname = %s AND a.attnum > 0 AND NOT a.attisdropped",
|
||||
(schema, table, column),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return ColumnType(str(row[0]), _int(row[1], -1), str(row[2]), _int(row[3], -1))
|
||||
|
||||
return lookup
|
||||
|
||||
|
||||
def _column_types(check: Check, column_type: ColumnTypeLookup) -> tuple[ColumnType, ...] | None:
|
||||
"""The catalog types of the check's columns, None when one of them is not there yet."""
|
||||
types: Final = tuple(column_type(check.table, column) for column in check.columns)
|
||||
if any(column is None for column in types):
|
||||
return None
|
||||
return tuple(column for column in types if column is not None)
|
||||
|
||||
|
||||
def _table_exists(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, table: str) -> bool:
|
||||
row: Final = connection.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name = %s", (schema, table)
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _index_is_valid(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: str) -> bool:
|
||||
row: Final = connection.execute(
|
||||
"SELECT x.indisvalid FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid "
|
||||
"JOIN pg_namespace n ON n.oid = i.relnamespace WHERE n.nspname = %s AND i.relname = %s",
|
||||
(schema, index),
|
||||
).fetchone()
|
||||
return row is not None and bool(row[0])
|
||||
|
||||
|
||||
def btree_max_entry_bytes(block_size: int) -> int:
|
||||
"""`BTMaxItemSize` from nbtree.h for btree version 4: a third of the page left after the page
|
||||
header, three line pointers and the btree opaque area, rounded down to 8, minus the heap TID a
|
||||
pivot tuple carries. 2704 on the default 8 KB page, the number in Postgres's error."""
|
||||
usable: Final = block_size - _PAGE_HEADER_WITH_THREE_LINE_POINTERS_BYTES - _BTREE_PAGE_OPAQUE_BYTES
|
||||
return (usable // 3) // 8 * 8 - _HEAP_TID_BYTES
|
||||
|
||||
|
||||
def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed":
|
||||
from psycopg import sql
|
||||
|
||||
if column.type_name in _OCTET_LENGTH_TYPES:
|
||||
return sql.SQL("octet_length({})").format(identifier)
|
||||
return sql.SQL("octet_length({}::text)").format(identifier)
|
||||
|
||||
|
||||
def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed":
|
||||
from psycopg import sql
|
||||
|
||||
unit: Final = _ALIGNMENT.get(align, 8)
|
||||
return sql.SQL("(({off} + {pad}) / {unit}) * {unit}").format(
|
||||
off=offset, pad=sql.Literal(unit - 1), unit=sql.Literal(unit)
|
||||
)
|
||||
|
||||
|
||||
def _value_end(column: ColumnType, identifier: "sql.Identifier", offset: "sql.Composable") -> "sql.Composed":
|
||||
"""Where this column's value ends when laid down from `offset` the way `index_form_tuple`
|
||||
does it: NULL takes no room, a fixed-length value is aligned to its type, a variable-length
|
||||
value of up to 126 bytes gets a 1-byte header and no alignment, a longer one a 4-byte header."""
|
||||
from psycopg import sql
|
||||
|
||||
length: Final = _value_bytes(column, identifier)
|
||||
placed: Final = (
|
||||
sql.SQL("{} + {}").format(_aligned(offset, column.align), sql.Literal(column.length))
|
||||
if not column.is_variable_length
|
||||
else sql.SQL("CASE WHEN {len} <= {short} THEN {off} + 1 + {len} ELSE {aligned} + {hdr} + {len} END").format(
|
||||
len=length,
|
||||
short=sql.Literal(SHORT_VARLENA_MAX_BYTES),
|
||||
off=offset,
|
||||
aligned=_aligned(offset, column.align),
|
||||
hdr=sql.Literal(VARLENA_HEADER_BYTES),
|
||||
)
|
||||
)
|
||||
return sql.SQL("CASE WHEN {col} IS NULL THEN {off} ELSE {placed} END").format(
|
||||
col=identifier, off=offset, placed=placed
|
||||
)
|
||||
|
||||
|
||||
def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
|
||||
"""count(*) and the largest entry over the rows whose btree entry would not fit, plus the
|
||||
longest value of each variable-length column: an 8-byte tuple header, 8 more for the null
|
||||
bitmap when a column is NULL, each value laid down in index order, the total rounded up to 8."""
|
||||
from psycopg import sql
|
||||
|
||||
identifiers: Final = tuple(sql.Identifier(column) for column in check.columns)
|
||||
any_null: Final = sql.SQL(" OR ").join(sql.SQL("{} IS NULL").format(identifier) for identifier in identifiers)
|
||||
header: Final = sql.SQL("{} + CASE WHEN {} THEN {} ELSE 0 END").format(
|
||||
sql.Literal(INDEX_TUPLE_HEADER_BYTES), any_null, sql.Literal(NULL_BITMAP_BYTES)
|
||||
)
|
||||
innermost: Final = sql.SQL("SELECT {cols}, {hdr} AS off0 FROM {t}").format(
|
||||
cols=sql.SQL(", ").join(identifiers), hdr=header, t=sql.Identifier(schema, check.table)
|
||||
)
|
||||
|
||||
def lay_down(inner: "sql.Composed", step: tuple[int, tuple[ColumnType, "sql.Identifier"]]) -> "sql.Composed":
|
||||
position, (column, identifier) = step
|
||||
return sql.SQL("SELECT *, {end} AS {next} FROM ({inner}) AS {alias}").format(
|
||||
end=_value_end(column, identifier, sql.Identifier(f"off{position}")),
|
||||
next=sql.Identifier(f"off{position + 1}"),
|
||||
inner=inner,
|
||||
alias=sql.Identifier(f"s{position}"),
|
||||
)
|
||||
|
||||
laid_out: Final = reduce(lay_down, enumerate(zip(types, identifiers, strict=True)), innermost)
|
||||
entry: Final = _aligned(sql.Identifier(f"off{len(types)}"), "d")
|
||||
longest: Final = sql.SQL("").join(
|
||||
sql.SQL(", max({})").format(_value_bytes(column, identifier))
|
||||
for column, identifier in zip(types, identifiers, strict=True)
|
||||
if column.is_variable_length
|
||||
)
|
||||
return sql.SQL("SELECT count(*), max({entry}){longest} FROM ({rows}) AS entries WHERE {entry} > {limit}").format(
|
||||
entry=entry, longest=longest, rows=laid_out, limit=sql.Literal(limit_bytes)
|
||||
)
|
||||
|
||||
|
||||
def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
|
||||
from psycopg import sql
|
||||
|
||||
table: Final = sql.Identifier(schema, check.table)
|
||||
columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns)
|
||||
match check.kind:
|
||||
case "too_long":
|
||||
return _entry_size_query(check, schema, types, limit_bytes)
|
||||
case "duplicates":
|
||||
not_null: Final = sql.SQL(" AND ").join(
|
||||
sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
|
||||
)
|
||||
return sql.SQL(
|
||||
"SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d"
|
||||
).format(t=table, nn=not_null, cols=columns)
|
||||
case "nulls":
|
||||
return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format(
|
||||
t=table, col=sql.Identifier(check.columns[0])
|
||||
)
|
||||
case "any_row":
|
||||
return sql.SQL("SELECT count(*) FROM {t}").format(t=table)
|
||||
case "orphans":
|
||||
pairs: Final = sql.SQL(" AND ").join(
|
||||
sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own))
|
||||
for own, theirs in zip(check.columns, check.referenced_columns, strict=True)
|
||||
)
|
||||
present: Final = sql.SQL(" AND ").join(
|
||||
sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
|
||||
)
|
||||
return sql.SQL(
|
||||
"SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})"
|
||||
).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs)
|
||||
case "check_violations":
|
||||
return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format(
|
||||
t=table,
|
||||
expr=sql.SQL(check.expression or "true"), # pyright: ignore[reportArgumentType] # copied from our own migration file
|
||||
)
|
||||
|
||||
|
||||
def _applies(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]",
|
||||
schema: str,
|
||||
check: Check,
|
||||
types: tuple[ColumnType, ...] | None,
|
||||
) -> bool:
|
||||
"""False when the check has nothing to count yet: the table or a column is not there
|
||||
(a pending migration adds it), the index is already valid, or no column can overflow."""
|
||||
if not _table_exists(connection, schema, check.table):
|
||||
return False
|
||||
if check.index is not None and _index_is_valid(connection, schema, check.index):
|
||||
return False
|
||||
if check.kind == "any_row":
|
||||
return types is None
|
||||
if types is None:
|
||||
return False
|
||||
if check.kind == "too_long":
|
||||
return any(column.can_overflow_btree for column in types)
|
||||
if check.kind == "orphans":
|
||||
return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table)
|
||||
return True
|
||||
|
||||
|
||||
def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str:
|
||||
variable: Final = tuple(
|
||||
column for column, column_type in zip(check.columns, types, strict=True) if column_type.is_variable_length
|
||||
)
|
||||
longest: Final = ", ".join(
|
||||
f"{column} up to {length} bytes"
|
||||
for column, length in zip(variable, values[2:], strict=True)
|
||||
if length is not None
|
||||
)
|
||||
return f"with an index entry over {limit_bytes} bytes, largest {values[1]} bytes ({longest})"
|
||||
|
||||
|
||||
def _run_check(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]",
|
||||
schema: str,
|
||||
check: Check,
|
||||
types: tuple[ColumnType, ...],
|
||||
limit_bytes: int,
|
||||
timeout_seconds: float,
|
||||
) -> Finding | None:
|
||||
import psycopg
|
||||
|
||||
try:
|
||||
row: Final = connection.execute(_count_query(check, schema, types, limit_bytes)).fetchone()
|
||||
except psycopg.errors.QueryCanceled:
|
||||
return CouldNotVerify(check, timeout_seconds)
|
||||
values: Final[tuple[object, ...]] = row if row is not None else ()
|
||||
rows: Final = _int(values[0], 0) if values else 0
|
||||
if rows == 0:
|
||||
return None
|
||||
match check.kind:
|
||||
case "too_long":
|
||||
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
|
||||
case "duplicates":
|
||||
return BlockedChange(check, rows, "duplicated value groups")
|
||||
case "nulls":
|
||||
return BlockedChange(check, rows, "NULL")
|
||||
case "any_row":
|
||||
return BlockedChange(
|
||||
check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows"
|
||||
)
|
||||
case "orphans":
|
||||
return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"')
|
||||
case "check_violations":
|
||||
return BlockedChange(check, rows, f"failing CHECK ({check.expression})")
|
||||
|
||||
|
||||
def find_blockers(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]",
|
||||
schema: str,
|
||||
checks: tuple[Check, ...],
|
||||
timeout_seconds: float = DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS,
|
||||
) -> tuple[Finding, ...]:
|
||||
"""Run every applicable check read-only, each under `timeout_seconds`. Never writes
|
||||
and never takes a table lock; a check that times out is a finding, not a pass."""
|
||||
from psycopg import sql
|
||||
|
||||
previous_timeout: Final = connection.execute("SHOW statement_timeout").fetchone()
|
||||
previous_read_only: Final = connection.execute("SHOW default_transaction_read_only").fetchone()
|
||||
block_size: Final = connection.execute("SHOW block_size").fetchone()
|
||||
limit_bytes: Final = btree_max_entry_bytes(int(str(block_size[0])) if block_size else DEFAULT_BLOCK_SIZE)
|
||||
connection.execute("SET default_transaction_read_only = on")
|
||||
connection.execute(sql.SQL("SET statement_timeout = {}").format(sql.Literal(int(timeout_seconds * 1000))))
|
||||
column_type: Final = column_type_lookup(connection, schema)
|
||||
typed: Final = tuple((check, _column_types(check, column_type)) for check in checks)
|
||||
applicable: Final = tuple(
|
||||
(check, types if types is not None else ())
|
||||
for check, types in typed
|
||||
if _applies(connection, schema, check, types)
|
||||
)
|
||||
results: Final = tuple(
|
||||
_run_check(connection, schema, check, types, limit_bytes, timeout_seconds) for check, types in applicable
|
||||
)
|
||||
connection.execute(
|
||||
sql.SQL("SET statement_timeout = {}").format(sql.Literal(str(previous_timeout[0]) if previous_timeout else "0"))
|
||||
)
|
||||
connection.execute(
|
||||
sql.SQL("SET default_transaction_read_only = {}").format(
|
||||
sql.Literal(str(previous_read_only[0]) if previous_read_only else "off")
|
||||
)
|
||||
)
|
||||
return tuple(result for result in results if result is not None)
|
||||
|
||||
|
||||
def _remedy(check: Check) -> str:
|
||||
if check.kind == "too_long" and check.table == "LiteLLM_SpendLogs" and "api_key" in check.columns:
|
||||
return LEGACY_SPEND_LOG_KEY_REMEDY
|
||||
joined: Final = ", ".join(check.columns)
|
||||
match check.kind:
|
||||
case "too_long":
|
||||
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
|
||||
case "duplicates":
|
||||
return f'Remove or merge the duplicate "{check.table}" ({joined}) values.'
|
||||
case "nulls":
|
||||
return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.'
|
||||
case "any_row":
|
||||
return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards."
|
||||
case "orphans":
|
||||
return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.'
|
||||
case "check_violations":
|
||||
return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.'
|
||||
|
||||
|
||||
def _describe(finding: Finding) -> str:
|
||||
check: Final = finding.check
|
||||
where: Final = f'"{check.table}"' + (f".{', '.join(check.columns)}" if check.columns else "")
|
||||
match finding:
|
||||
case CouldNotVerify(timeout_seconds=seconds):
|
||||
return (
|
||||
f" {where}: could not be checked within {seconds:g} s, refusing rather than guessing\n"
|
||||
f" blocks: {check.statement} ({check.source})\n"
|
||||
f" raise {MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR} to give the scan more time"
|
||||
)
|
||||
case BlockedChange(rows=rows, detail=detail):
|
||||
return (
|
||||
f" {where}: {rows} row(s) {detail}\n"
|
||||
f" blocks: {check.statement} ({check.source})\n"
|
||||
f" fix: {_remedy(check).replace(chr(10), chr(10) + ' ' * 8)}"
|
||||
)
|
||||
|
||||
|
||||
def report(findings: tuple[Finding, ...]) -> str:
|
||||
"""The operator-facing refusal: one block per finding, then what to do next."""
|
||||
lines: Final = "\n".join(_describe(finding) for finding in findings)
|
||||
return (
|
||||
f"{REFUSAL}\n{lines}\n"
|
||||
"No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again."
|
||||
)
|
||||
|
||||
|
||||
def pending_change_checks(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]",
|
||||
schema: str,
|
||||
migrations_dir: Path | None,
|
||||
indexes: tuple[tuple[str, str, tuple[str, ...]], ...],
|
||||
) -> tuple[Check, ...]:
|
||||
"""Every check the pending migrations and the request-log indexes need. `migrations_dir`
|
||||
is None when the schema is pushed rather than migrated, so only the indexes are checked."""
|
||||
migration_checks: Final = (
|
||||
tuple(
|
||||
check
|
||||
for name, migration_sql in pending_migrations(connection, schema, migrations_dir)
|
||||
for check in checks_for_migration(name, migration_sql)
|
||||
) # comprehension-ok: flattening the per-migration checks
|
||||
if migrations_dir is not None
|
||||
else ()
|
||||
)
|
||||
index_checks: Final = tuple(
|
||||
check for table, name, columns in indexes for check in checks_for_index(table, name, columns)
|
||||
) # comprehension-ok: flattening the per-index checks
|
||||
return migration_checks + index_checks
|
||||
|
|
@ -18,6 +18,12 @@ from dataclasses import dataclass
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm_proxy_extras._logging import logger
|
||||
from litellm_proxy_extras.migration_data_checks import (
|
||||
checks_for_index,
|
||||
find_blockers,
|
||||
migration_data_check_timeout_seconds,
|
||||
report,
|
||||
)
|
||||
from litellm_proxy_extras.migration_lock import held_migration_lock
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -355,6 +361,12 @@ def _build_leaf_index(
|
|||
connection.execute(sql.SQL("DROP INDEX CONCURRENTLY {}").format(sql.Identifier(schema, name)))
|
||||
elif _adopt_equivalent_index(connection, schema, table, name, index):
|
||||
return True
|
||||
blockers: Final = find_blockers(
|
||||
connection, schema, checks_for_index(table, name, index.columns), migration_data_check_timeout_seconds()
|
||||
)
|
||||
if blockers:
|
||||
logger.warning("Not building index %s on %s. %s", name, table, report(blockers))
|
||||
return False
|
||||
logger.info("Building index %s on %s concurrently", name, table)
|
||||
prefix: Final = sql.SQL("CREATE INDEX CONCURRENTLY IF NOT EXISTS {} ON {} ").format(
|
||||
sql.Identifier(name), sql.Identifier(schema, table)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,13 @@ from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
|
|||
from litellm_proxy_extras import prisma_toolchain
|
||||
from litellm_proxy_extras._logging import logger
|
||||
from litellm_proxy_extras._redaction import _redact_credentials
|
||||
from litellm_proxy_extras.migration_data_checks import (
|
||||
Finding,
|
||||
find_blockers,
|
||||
migration_data_check_timeout_seconds,
|
||||
pending_change_checks,
|
||||
report,
|
||||
)
|
||||
from litellm_proxy_extras.migration_lock import held_migration_lock
|
||||
from litellm_proxy_extras.prisma_toolchain import (
|
||||
PRISMA_COMMAND_TIMEOUT_ENV_VAR,
|
||||
|
|
@ -28,7 +35,11 @@ from litellm_proxy_extras.replica_identity import (
|
|||
REPLICA_IDENTITY_FULL_ENV_VAR,
|
||||
apply_replica_identity_full,
|
||||
)
|
||||
from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes, filter_request_log_index_diff
|
||||
from litellm_proxy_extras.request_log_indexes import (
|
||||
REQUEST_LOG_INDEXES,
|
||||
ensure_request_log_indexes,
|
||||
filter_request_log_index_diff,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import psycopg
|
||||
|
|
@ -53,9 +64,7 @@ def _get_prisma_env() -> dict:
|
|||
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
|
||||
# These env vars prevent Prisma from attempting downloads
|
||||
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
|
||||
prisma_env["NPM_CONFIG_CACHE"] = os.getenv(
|
||||
"NPM_CONFIG_CACHE", "/app/.cache/npm"
|
||||
)
|
||||
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
|
||||
return prisma_env
|
||||
|
||||
|
||||
|
|
@ -86,6 +95,7 @@ class _InvalidIndex:
|
|||
name: str
|
||||
table_size: str
|
||||
|
||||
|
||||
MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
|
||||
LIBPQ_URL_PARAMS: Final = frozenset(
|
||||
{
|
||||
|
|
@ -135,9 +145,7 @@ class _MigrateAttemptBudget:
|
|||
|
||||
|
||||
_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
|
||||
_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
|
||||
r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
|
||||
)
|
||||
_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE)
|
||||
_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
|
||||
r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
|
||||
r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
|
||||
|
|
@ -146,9 +154,9 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile(
|
|||
|
||||
PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
|
||||
"LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
|
||||
"so its primary key must include the partition key (\"startTime\"). `prisma db push` "
|
||||
'so its primary key must include the partition key ("startTime"). `prisma db push` '
|
||||
"reconciles the database against schema.prisma, which declares the unpartitioned "
|
||||
"primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
|
||||
'primary key ("request_id"), and Postgres rejects that rewrite with: unique '
|
||||
"constraint on partitioned table must include all partitioning columns. Start the "
|
||||
"proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
|
||||
"applies shipped migrations and leaves the partitioned primary key alone."
|
||||
|
|
@ -157,9 +165,7 @@ PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
|
|||
|
||||
def _without_sql_comments(statement: str) -> str:
|
||||
return "\n".join(
|
||||
line
|
||||
for line in statement.splitlines()
|
||||
if line.strip() and not line.strip().startswith("--")
|
||||
line for line in statement.splitlines() if line.strip() and not line.strip().startswith("--")
|
||||
).strip()
|
||||
|
||||
|
||||
|
|
@ -169,7 +175,7 @@ def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
|
|||
return statement
|
||||
kept = tuple(
|
||||
clause.strip()
|
||||
for clause in statement[prefix_match.end():].split(",\n")
|
||||
for clause in statement[prefix_match.end() :].split(",\n")
|
||||
if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
|
||||
)
|
||||
if not kept:
|
||||
|
|
@ -243,8 +249,7 @@ def _get_prisma_command() -> str:
|
|||
|
||||
# If not found, log warning and fall back
|
||||
logger.warning(
|
||||
f"Prisma CLI not found at {default_cli_path}. "
|
||||
"Falling back to Python wrapper (may attempt downloads)"
|
||||
f"Prisma CLI not found at {default_cli_path}. Falling back to Python wrapper (may attempt downloads)"
|
||||
)
|
||||
|
||||
# Fall back to the Python wrapper (will work in online mode)
|
||||
|
|
@ -331,9 +336,7 @@ class ProxyExtrasDBManager:
|
|||
|
||||
return True
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(
|
||||
"Migration timed out - the database might be under heavy load."
|
||||
)
|
||||
logger.warning("Migration timed out - the database might be under heavy load.")
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(
|
||||
|
|
@ -386,10 +389,12 @@ class ProxyExtrasDBManager:
|
|||
except ImportError:
|
||||
return None
|
||||
|
||||
from psycopg import sql
|
||||
|
||||
cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
|
||||
ledger_table: Final = psycopg.sql.SQL("{}.{}").format(
|
||||
psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"),
|
||||
psycopg.sql.Identifier("_prisma_migrations"),
|
||||
ledger_table: Final = sql.SQL("{}.{}").format(
|
||||
sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"),
|
||||
sql.Identifier("_prisma_migrations"),
|
||||
)
|
||||
try:
|
||||
with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
|
||||
|
|
@ -500,9 +505,7 @@ class ProxyExtrasDBManager:
|
|||
return filter_partitioned_spend_logs_diff(without_indexes)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_all_migrations(
|
||||
migrations_dir: str, schema_path: str, mark_all_applied: bool = True
|
||||
):
|
||||
def _resolve_all_migrations(migrations_dir: str, schema_path: str, mark_all_applied: bool = True):
|
||||
"""
|
||||
1. Compare the current database state to schema.prisma and generate a migration for the diff.
|
||||
2. Run prisma migrate deploy to apply any pending migrations.
|
||||
|
|
@ -571,9 +574,7 @@ class ProxyExtrasDBManager:
|
|||
)
|
||||
logger.info(f"Applied migration: {mig_file.parent.name}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(
|
||||
f"Failed to apply migration {mig_file.parent.name}: {e.stderr}"
|
||||
)
|
||||
logger.warning(f"Failed to apply migration {mig_file.parent.name}: {e.stderr}")
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(f"Migration {mig_file.parent.name} timed out.")
|
||||
return
|
||||
|
|
@ -645,9 +646,7 @@ class ProxyExtrasDBManager:
|
|||
logger.debug(f"Resolved migration: {migration_name}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
if "is already recorded as applied in the database." not in e.stderr:
|
||||
logger.warning(
|
||||
f"Failed to resolve migration {migration_name}: {e.stderr}"
|
||||
)
|
||||
logger.warning(f"Failed to resolve migration {migration_name}: {e.stderr}")
|
||||
|
||||
@staticmethod
|
||||
def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None:
|
||||
|
|
@ -707,9 +706,7 @@ class ProxyExtrasDBManager:
|
|||
|
||||
cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
|
||||
try:
|
||||
with psycopg.connect(
|
||||
cleaned_url, connect_timeout=10, autocommit=True
|
||||
) as conn:
|
||||
with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT 1 "
|
||||
"FROM pg_partitioned_table pt "
|
||||
|
|
@ -717,10 +714,7 @@ class ProxyExtrasDBManager:
|
|||
"JOIN pg_namespace n ON n.oid = c.relnamespace "
|
||||
"WHERE c.relname = 'LiteLLM_SpendLogs' "
|
||||
" AND n.nspname = %s",
|
||||
(
|
||||
ProxyExtrasDBManager._prisma_schema_param(database_url)
|
||||
or "public",
|
||||
),
|
||||
(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public",),
|
||||
).fetchone()
|
||||
except (psycopg.OperationalError, psycopg.DatabaseError):
|
||||
return False
|
||||
|
|
@ -815,9 +809,7 @@ class ProxyExtrasDBManager:
|
|||
# it, psycopg3's `with conn` calls COMMIT on clean exit — which
|
||||
# fails after `UndefinedTable` (fresh DB) leaves the transaction
|
||||
# in an aborted state.
|
||||
with psycopg.connect(
|
||||
cleaned_url, connect_timeout=10, autocommit=True
|
||||
) as conn:
|
||||
with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT migration_name FROM _prisma_migrations "
|
||||
|
|
@ -838,9 +830,7 @@ class ProxyExtrasDBManager:
|
|||
return
|
||||
|
||||
head_newest_ts = _max_migration_timestamp(known)
|
||||
hostile = {
|
||||
name for name in unknown if _migration_timestamp(name) > head_newest_ts
|
||||
}
|
||||
hostile = {name for name in unknown if _migration_timestamp(name) > head_newest_ts}
|
||||
if not hostile:
|
||||
return
|
||||
|
||||
|
|
@ -931,8 +921,7 @@ class ProxyExtrasDBManager:
|
|||
if not found:
|
||||
return True
|
||||
logger.warning(
|
||||
"Found %d invalid index(es) left by an interrupted CREATE INDEX "
|
||||
"CONCURRENTLY, rebuilding: %s",
|
||||
"Found %d invalid index(es) left by an interrupted CREATE INDEX CONCURRENTLY, rebuilding: %s",
|
||||
len(found),
|
||||
", ".join(f"{index.name} (table size {index.table_size})" for index in found),
|
||||
)
|
||||
|
|
@ -1403,8 +1392,7 @@ class ProxyExtrasDBManager:
|
|||
prisma_env = _get_prisma_env()
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Could not resolve the migrations directory for the REPLICA "
|
||||
"IDENTITY FULL step, skipping it. Error: %s",
|
||||
"Could not resolve the migrations directory for the REPLICA IDENTITY FULL step, skipping it. Error: %s",
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
|
@ -1435,18 +1423,57 @@ class ProxyExtrasDBManager:
|
|||
Returns:
|
||||
bool: True if setup was successful, False otherwise
|
||||
"""
|
||||
ensure_prisma_toolchain(
|
||||
prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env()
|
||||
)
|
||||
migrated = ProxyExtrasDBManager._run_migrations(
|
||||
use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
|
||||
)
|
||||
ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env())
|
||||
ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate)
|
||||
migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver)
|
||||
if not migrated:
|
||||
return False
|
||||
ProxyExtrasDBManager.repair_invalid_indexes()
|
||||
ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]:
|
||||
"""What in the existing rows would make a pending migration or request-log index
|
||||
fail, read with count queries only: no write, no table lock. Empty when nothing
|
||||
blocks, when no database is configured, or when it cannot be reached, so the
|
||||
migration step that follows reports the connection problem the way it does today."""
|
||||
import psycopg
|
||||
|
||||
database_url: Final = os.environ.get("DATABASE_URL")
|
||||
if not database_url:
|
||||
return ()
|
||||
direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
|
||||
os.environ.get("DIRECT_URL") or database_url
|
||||
)
|
||||
schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
|
||||
migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None
|
||||
indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES)
|
||||
try:
|
||||
with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection:
|
||||
checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes)
|
||||
return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds())
|
||||
except psycopg.OperationalError as exc:
|
||||
logger.warning(
|
||||
"Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc))
|
||||
)
|
||||
return ()
|
||||
|
||||
@staticmethod
|
||||
def migration_data_check_report(use_migrate: bool = True) -> Optional[str]:
|
||||
"""The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks."""
|
||||
findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate)
|
||||
return report(findings) if findings else None
|
||||
|
||||
@staticmethod
|
||||
def raise_if_rows_block_pending_changes(use_migrate: bool) -> None:
|
||||
"""Refuse to run migrations that the existing rows would make fail, before any is
|
||||
applied. The RuntimeError carries the report and reaches the operator through the
|
||||
proxy's "Database migration cannot proceed" exit."""
|
||||
refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate)
|
||||
if refusal is not None:
|
||||
raise RuntimeError(refusal)
|
||||
|
||||
@staticmethod
|
||||
def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
|
||||
"""Build the indexes in `REQUEST_LOG_INDEXES` on the writer, in the schema the
|
||||
|
|
@ -1513,9 +1540,7 @@ class ProxyExtrasDBManager:
|
|||
# Skip sanity check when deploy reports no pending migrations —
|
||||
# DB already matches schema, no drift to correct.
|
||||
if "No pending migrations to apply" in result.stdout:
|
||||
logger.info(
|
||||
"No pending migrations — skipping post-migration sanity check"
|
||||
)
|
||||
logger.info("No pending migrations — skipping post-migration sanity check")
|
||||
return True
|
||||
|
||||
# Run sanity check to ensure DB matches schema
|
||||
|
|
@ -1530,9 +1555,7 @@ class ProxyExtrasDBManager:
|
|||
logger.info(f"prisma db error: {stderr}, e: {e.stdout}")
|
||||
if "P3009" in stderr:
|
||||
# Extract the failed migration name from the error message
|
||||
migration_match = re.search(
|
||||
r"`(\d+_.*)` migration", stderr
|
||||
)
|
||||
migration_match = re.search(r"`(\d+_.*)` migration", stderr)
|
||||
if migration_match:
|
||||
failed_migration = migration_match.group(1)
|
||||
if ProxyExtrasDBManager._is_idempotent_error(stderr):
|
||||
|
|
@ -1540,9 +1563,7 @@ class ProxyExtrasDBManager:
|
|||
f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied"
|
||||
)
|
||||
try:
|
||||
ProxyExtrasDBManager._roll_back_migration(
|
||||
failed_migration
|
||||
)
|
||||
ProxyExtrasDBManager._roll_back_migration(failed_migration)
|
||||
except (
|
||||
subprocess.CalledProcessError,
|
||||
subprocess.TimeoutExpired,
|
||||
|
|
@ -1552,9 +1573,7 @@ class ProxyExtrasDBManager:
|
|||
f"It may already be in a rolled-back state."
|
||||
)
|
||||
try:
|
||||
ProxyExtrasDBManager._resolve_specific_migration(
|
||||
failed_migration
|
||||
)
|
||||
ProxyExtrasDBManager._resolve_specific_migration(failed_migration)
|
||||
logger.info(
|
||||
f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations"
|
||||
)
|
||||
|
|
@ -1562,9 +1581,7 @@ class ProxyExtrasDBManager:
|
|||
subprocess.CalledProcessError,
|
||||
subprocess.TimeoutExpired,
|
||||
) as resolve_err:
|
||||
logger.warning(
|
||||
f"Failed to resolve migration {failed_migration}: {resolve_err}"
|
||||
)
|
||||
logger.warning(f"Failed to resolve migration {failed_migration}: {resolve_err}")
|
||||
# Apply any schema drift not covered by the marked-as-applied migration
|
||||
ProxyExtrasDBManager._resolve_all_migrations(
|
||||
migrations_dir,
|
||||
|
|
@ -1572,9 +1589,7 @@ class ProxyExtrasDBManager:
|
|||
mark_all_applied=False,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Found failed migration: {failed_migration}, marking as rolled back"
|
||||
)
|
||||
logger.info(f"Found failed migration: {failed_migration}, marking as rolled back")
|
||||
# Mark the failed migration as rolled back
|
||||
prisma_toolchain.run_prisma(
|
||||
[
|
||||
|
|
@ -1587,23 +1602,14 @@ class ProxyExtrasDBManager:
|
|||
timeout=prisma_command_timeout(),
|
||||
env=_get_prisma_env(),
|
||||
)
|
||||
logger.info(
|
||||
f"✅ Migration {failed_migration} marked as rolled back... retrying"
|
||||
)
|
||||
elif (
|
||||
"P3005" in stderr
|
||||
and "database schema is not empty" in stderr
|
||||
):
|
||||
logger.info(f"✅ Migration {failed_migration} marked as rolled back... retrying")
|
||||
elif "P3005" in stderr and "database schema is not empty" in stderr:
|
||||
logger.info(
|
||||
"Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system"
|
||||
)
|
||||
ProxyExtrasDBManager._create_baseline_migration(schema_path)
|
||||
logger.info(
|
||||
"Baseline migration created, resolving all migrations"
|
||||
)
|
||||
ProxyExtrasDBManager._resolve_all_migrations(
|
||||
migrations_dir, schema_path
|
||||
)
|
||||
logger.info("Baseline migration created, resolving all migrations")
|
||||
ProxyExtrasDBManager._resolve_all_migrations(migrations_dir, schema_path)
|
||||
logger.info("✅ All migrations resolved.")
|
||||
return True
|
||||
elif "P3018" in stderr:
|
||||
|
|
@ -1611,14 +1617,8 @@ class ProxyExtrasDBManager:
|
|||
if ProxyExtrasDBManager._is_permission_error(stderr):
|
||||
# Permission errors should NOT be marked as applied
|
||||
# Extract migration name for logging
|
||||
migration_match = re.search(
|
||||
r"Migration name: (\d+_.*)", stderr
|
||||
)
|
||||
migration_name = (
|
||||
migration_match.group(1)
|
||||
if migration_match
|
||||
else "unknown"
|
||||
)
|
||||
migration_match = re.search(r"Migration name: (\d+_.*)", stderr)
|
||||
migration_name = migration_match.group(1) if migration_match else "unknown"
|
||||
|
||||
logger.error(
|
||||
f"❌ Migration {migration_name} failed due to insufficient permissions. "
|
||||
|
|
@ -1628,16 +1628,10 @@ class ProxyExtrasDBManager:
|
|||
# Mark as rolled back and exit with error
|
||||
if migration_match:
|
||||
try:
|
||||
ProxyExtrasDBManager._roll_back_migration(
|
||||
migration_name
|
||||
)
|
||||
logger.info(
|
||||
f"Migration {migration_name} marked as rolled back"
|
||||
)
|
||||
ProxyExtrasDBManager._roll_back_migration(migration_name)
|
||||
logger.info(f"Migration {migration_name} marked as rolled back")
|
||||
except Exception as rollback_error:
|
||||
logger.warning(
|
||||
f"Failed to mark migration as rolled back: {rollback_error}"
|
||||
)
|
||||
logger.warning(f"Failed to mark migration as rolled back: {rollback_error}")
|
||||
|
||||
# Re-raise the error to prevent silent failures
|
||||
raise RuntimeError(
|
||||
|
|
@ -1652,18 +1646,12 @@ class ProxyExtrasDBManager:
|
|||
"resolving as applied"
|
||||
)
|
||||
# Extract the migration name from the error message
|
||||
migration_match = re.search(
|
||||
r"Migration name: (\d+_.*)", stderr
|
||||
)
|
||||
migration_match = re.search(r"Migration name: (\d+_.*)", stderr)
|
||||
if migration_match:
|
||||
migration_name = migration_match.group(1)
|
||||
try:
|
||||
logger.info(
|
||||
f"Rolling back migration {migration_name}"
|
||||
)
|
||||
ProxyExtrasDBManager._roll_back_migration(
|
||||
migration_name
|
||||
)
|
||||
logger.info(f"Rolling back migration {migration_name}")
|
||||
ProxyExtrasDBManager._roll_back_migration(migration_name)
|
||||
except (
|
||||
subprocess.CalledProcessError,
|
||||
subprocess.TimeoutExpired,
|
||||
|
|
@ -1677,9 +1665,7 @@ class ProxyExtrasDBManager:
|
|||
f"Resolving migration {migration_name} that failed "
|
||||
f"due to existing schema objects"
|
||||
)
|
||||
ProxyExtrasDBManager._resolve_specific_migration(
|
||||
migration_name
|
||||
)
|
||||
ProxyExtrasDBManager._resolve_specific_migration(migration_name)
|
||||
logger.info(
|
||||
f"✅ Migration {migration_name} resolved, "
|
||||
f"retrying to apply remaining migrations"
|
||||
|
|
@ -1688,9 +1674,7 @@ class ProxyExtrasDBManager:
|
|||
subprocess.CalledProcessError,
|
||||
subprocess.TimeoutExpired,
|
||||
) as resolve_err:
|
||||
logger.warning(
|
||||
f"Failed to resolve migration {migration_name}: {resolve_err}"
|
||||
)
|
||||
logger.warning(f"Failed to resolve migration {migration_name}: {resolve_err}")
|
||||
# Apply any schema drift not covered by the marked-as-applied migration
|
||||
ProxyExtrasDBManager._resolve_all_migrations(
|
||||
migrations_dir,
|
||||
|
|
@ -1732,14 +1716,8 @@ class ProxyExtrasDBManager:
|
|||
time.sleep(random.randrange(5, 15))
|
||||
except subprocess.CalledProcessError as e:
|
||||
attempts_left = 3 - attempt
|
||||
retry_msg = (
|
||||
f" Retrying... ({attempts_left} attempts left)"
|
||||
if attempts_left > 0
|
||||
else ""
|
||||
)
|
||||
stderr_detail: Final = (
|
||||
f" stderr: {_redact_credentials(str(e.stderr))}" if e.stderr else ""
|
||||
)
|
||||
retry_msg = f" Retrying... ({attempts_left} attempts left)" if attempts_left > 0 else ""
|
||||
stderr_detail: Final = f" stderr: {_redact_credentials(str(e.stderr))}" if e.stderr else ""
|
||||
logger.error(
|
||||
f"The process failed to execute. Details: {_redact_command_error(e)}.{stderr_detail}{retry_msg}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -235,7 +235,7 @@ python_callbacks! {
|
|||
"async_log_stream_event",
|
||||
"async_post_mcp_tool_call_hook",
|
||||
],
|
||||
Dispatch::Python("litellm.llms.anthropic.pass_through.messages.handler") => [
|
||||
Dispatch::Python("litellm.llms.anthropic.messages.native.handler") => [
|
||||
"async_pre_request_hook",
|
||||
],
|
||||
Dispatch::Python("litellm.litellm_core_utils.streaming_handler") => [
|
||||
|
|
|
|||
|
|
@ -235,6 +235,7 @@ bedrock_neutralize_orphaned_tool_blocks: bool = True
|
|||
use_chat_completions_url_for_anthropic_messages: bool = bool(
|
||||
os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
|
||||
) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API
|
||||
anthropic_messages_estimate_initial_input_tokens: bool = False
|
||||
# When True, strip the OpenAI-flavored `usage.total_tokens` field that
|
||||
# LiteLLM injects into non-streaming /v1/messages responses, bringing the
|
||||
# wire response into line with the Anthropic spec (matches the streaming
|
||||
|
|
@ -1789,7 +1790,7 @@ if TYPE_CHECKING:
|
|||
SagemakerNovaConfig as SagemakerNovaConfig,
|
||||
)
|
||||
from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig
|
||||
from .llms.anthropic.pass_through.messages.transformation import (
|
||||
from .llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig as AnthropicMessagesConfig,
|
||||
)
|
||||
from .llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
|
||||
|
|
|
|||
|
|
@ -801,7 +801,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
),
|
||||
"CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"),
|
||||
"AnthropicMessagesConfig": (
|
||||
".llms.anthropic.pass_through.messages.transformation",
|
||||
".llms.anthropic.messages.native.transformation",
|
||||
"AnthropicMessagesConfig",
|
||||
),
|
||||
"BedrockClaudePlatformMessagesConfig": (
|
||||
|
|
|
|||
|
|
@ -57,6 +57,40 @@ async for chunk in response:
|
|||
print(chunk)
|
||||
```
|
||||
|
||||
### Optional opening token estimate
|
||||
|
||||
When `/v1/messages` streams through the Chat Completions adapter, the opening
|
||||
`message_start.usage.input_tokens` defaults to `0`. To show an estimated prompt
|
||||
size while a response streams, enable `anthropic_messages_estimate_initial_input_tokens`.
|
||||
This setting defaults to `false` and applies to all requests using this adapter
|
||||
|
||||
For an existing proxy deployment, add the setting under `litellm_settings` in
|
||||
`config.yaml` and restart the proxy. Use a YAML boolean, without quotes
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
anthropic_messages_estimate_initial_input_tokens: true
|
||||
```
|
||||
|
||||
For the Python SDK, set the option before making streaming Messages requests
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
litellm.anthropic_messages_estimate_initial_input_tokens = True
|
||||
```
|
||||
|
||||
Send a streaming Messages request to a Chat Completions-backed model, such as a
|
||||
`hosted_vllm/` model. The opening `message_start` event includes the estimated
|
||||
input count. If estimation fails, the count remains `0`. Set the option to
|
||||
`false` in YAML or `False` in Python to disable estimation and skip token counting
|
||||
|
||||
The opening count is approximate and is not suitable for billing or enforcing
|
||||
exact context limits. Opening output and cache counts remain `0`. Final
|
||||
`message_delta` usage still comes from the provider and takes precedence over the
|
||||
opening estimate. Native Anthropic streams, the Responses adapter, and
|
||||
non-streaming responses are unaffected. This setting does not select the adapter
|
||||
|
||||
### LiteLLM Proxy Server
|
||||
|
||||
|
||||
|
|
@ -113,4 +147,4 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
|
|||
],
|
||||
"max_tokens": 100
|
||||
}'
|
||||
```
|
||||
```
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ from litellm.types.utils import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.pass_through.messages.response_cache import (
|
||||
from litellm.llms.anthropic.messages.native.response_cache import (
|
||||
AnthropicMessagesStreamCacheWriter,
|
||||
)
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper
|
||||
|
|
@ -182,7 +182,7 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo
|
|||
spend and callback records. A plain (non-stream) replay logs here, since nothing
|
||||
else will.
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.messages.response_cache import (
|
||||
from litellm.llms.anthropic.messages.native.response_cache import (
|
||||
CachedAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
|
@ -1040,7 +1040,7 @@ class LLMCachingHandler:
|
|||
elif (
|
||||
call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value
|
||||
) and isinstance(cached_result, dict):
|
||||
from litellm.llms.anthropic.pass_through.messages.response_cache import (
|
||||
from litellm.llms.anthropic.messages.native.response_cache import (
|
||||
convert_cached_anthropic_messages_result,
|
||||
)
|
||||
|
||||
|
|
@ -1270,7 +1270,7 @@ class LLMCachingHandler:
|
|||
return result
|
||||
if not isinstance(result, AsyncIterator):
|
||||
return result
|
||||
from litellm.llms.anthropic.pass_through.messages.response_cache import (
|
||||
from litellm.llms.anthropic.messages.native.response_cache import (
|
||||
AnthropicMessagesStreamCacheWriter,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1980,7 +1980,7 @@ def completion_cost(
|
|||
base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
|
||||
)
|
||||
elif call_type == _MCP_CALL_TYPE:
|
||||
from litellm.proxy._experimental.mcp_server.cost_calculator import (
|
||||
from litellm.proxy.mcp_server.cost_calculator import (
|
||||
MCPCostCalculator,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
try:
|
||||
from .tools import call_openai_tool, load_mcp_tools
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name not in ("mcp", "httpx2"):
|
||||
raise
|
||||
raise ImportError("MCP client dependencies are missing. Install them with: pip install 'litellm[mcp]'") from exc
|
||||
import litellm.mcp_client as _mcp_client
|
||||
from litellm.litellm_core_utils.deprecated_package import alias_deprecated_package
|
||||
from litellm.mcp_client import call_openai_tool, load_mcp_tools
|
||||
|
||||
alias_deprecated_package(__name__, _mcp_client)
|
||||
|
||||
__all__ = ["call_openai_tool", "load_mcp_tools"]
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.types.integrations.gcs_bucket import *
|
||||
from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
|
|
@ -140,49 +140,16 @@ class GCSBucketBase(CustomBatchLogger):
|
|||
|
||||
async def get_gcs_logging_config(self, kwargs: dict[str, Any] | None = {}) -> GCSLoggingConfig:
|
||||
"""
|
||||
This function is used to get the GCS logging config for the GCS Bucket Logger.
|
||||
It checks if the dynamic parameters are provided in the kwargs and uses them to get the GCS logging config.
|
||||
If no dynamic parameters are provided, it uses the default values.
|
||||
Returns the bucket and service account this logger was configured with.
|
||||
"""
|
||||
if kwargs is None:
|
||||
kwargs = {}
|
||||
|
||||
standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
|
||||
"standard_callback_dynamic_params", None
|
||||
)
|
||||
|
||||
bucket_name: str
|
||||
path_service_account: str | None
|
||||
if standard_callback_dynamic_params is not None:
|
||||
verbose_logger.debug("Using dynamic GCS logging")
|
||||
verbose_logger.debug("standard_callback_dynamic_params: %s", standard_callback_dynamic_params)
|
||||
|
||||
_bucket_name: str | None = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME
|
||||
_path_service_account: Final[str | None] = (
|
||||
standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json
|
||||
if self.BUCKET_NAME is None:
|
||||
raise ValueError(
|
||||
"GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
|
||||
)
|
||||
|
||||
if _bucket_name is None:
|
||||
raise ValueError(
|
||||
"GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
|
||||
)
|
||||
bucket_name = _bucket_name
|
||||
path_service_account = _path_service_account
|
||||
vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account)
|
||||
else:
|
||||
# If no dynamic parameters, use the default instance
|
||||
if self.BUCKET_NAME is None:
|
||||
raise ValueError(
|
||||
"GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment."
|
||||
)
|
||||
bucket_name = self.BUCKET_NAME
|
||||
path_service_account = self.path_service_account_json
|
||||
vertex_instance = await self.get_or_create_vertex_instance(credentials=path_service_account)
|
||||
|
||||
return GCSLoggingConfig(
|
||||
bucket_name=bucket_name,
|
||||
vertex_instance=vertex_instance,
|
||||
path_service_account=path_service_account,
|
||||
bucket_name=self.BUCKET_NAME,
|
||||
vertex_instance=await self.get_or_create_vertex_instance(credentials=self.path_service_account_json),
|
||||
path_service_account=self.path_service_account_json,
|
||||
)
|
||||
|
||||
async def get_or_create_vertex_instance(self, credentials: str | None) -> VertexBase:
|
||||
|
|
|
|||
|
|
@ -1,19 +1,32 @@
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
|
||||
|
||||
from opentelemetry import trace
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations._types.open_inference import ErrorAttributes
|
||||
from litellm.integrations.arize import _utils
|
||||
from litellm.integrations.langfuse.langfuse_otel_attributes import (
|
||||
LangfuseLLMObsOTELAttributes,
|
||||
serialize_langfuse_datetime,
|
||||
)
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
|
||||
from litellm.integrations.opentelemetry import (
|
||||
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
|
||||
LITELLM_PROXY_REQUEST_SPAN_NAME,
|
||||
OpenTelemetry,
|
||||
OpenTelemetryConfig,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.integrations.langfuse_otel import (
|
||||
LangfuseSpanAttributes,
|
||||
)
|
||||
|
|
@ -31,6 +44,86 @@ LANGFUSE_CLOUD_EU_ENDPOINT: Final = "https://cloud.langfuse.com/api/public/otel"
|
|||
LANGFUSE_CLOUD_US_ENDPOINT: Final = "https://us.cloud.langfuse.com/api/public/otel"
|
||||
LANGFUSE_INGESTION_VERSION_HEADER: Final = "x-langfuse-ingestion-version"
|
||||
LANGFUSE_INGESTION_VERSION: Final = "4"
|
||||
_ERROR_ATTRIBUTES: Final = frozenset(
|
||||
{
|
||||
ErrorAttributes.ERROR_CODE,
|
||||
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
|
||||
ErrorAttributes.ERROR_TYPE,
|
||||
ErrorAttributes.ERROR_MESSAGE,
|
||||
ErrorAttributes.ERROR_LLM_PROVIDER,
|
||||
ErrorAttributes.ERROR_STACK_TRACE,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _Gettable(Protocol):
|
||||
def get(self, key: str, default: object = None) -> object: ...
|
||||
|
||||
|
||||
def _as_gettable(value: object) -> _Gettable | None:
|
||||
return value if isinstance(value, _Gettable) else None
|
||||
|
||||
|
||||
def _attr(item: object, name: str, default: object = None) -> object:
|
||||
return getattr(item, name, default)
|
||||
|
||||
|
||||
def _preset_cache_key(kwargs: Mapping[str, object]) -> object:
|
||||
if litellm.cache is None:
|
||||
return None
|
||||
return litellm.cache.get_preset_cache_key_from_kwargs(**kwargs)
|
||||
|
||||
|
||||
def _mask_flag(value: object) -> bool:
|
||||
parsed: Final = str_to_bool(value) if isinstance(value, str) else None
|
||||
return bool(value) if parsed is None else parsed
|
||||
|
||||
|
||||
def _caller_tag_list(value: object) -> tuple[str, ...]:
|
||||
parsed: Final = (
|
||||
safe_json_loads(value, default=value) if isinstance(value, str) and value.strip().startswith("[") else value
|
||||
)
|
||||
if isinstance(parsed, list):
|
||||
return tuple(tag if isinstance(tag, str) else json.dumps(tag, default=str) for tag in parsed)
|
||||
if isinstance(parsed, str) and parsed:
|
||||
return tuple(tag.strip() for tag in parsed.split(",")) if "," in parsed else (parsed,)
|
||||
return ()
|
||||
|
||||
|
||||
def _parent_of(span: object) -> trace.SpanContext | None:
|
||||
parent: Final = getattr(span, "parent", None)
|
||||
return parent if isinstance(parent, trace.SpanContext) else None
|
||||
|
||||
|
||||
def _litellm_proxy_span(kwargs: Mapping[str, object]) -> trace.Span | None:
|
||||
litellm_params: Final = _as_gettable(kwargs.get("litellm_params")) or {}
|
||||
carried: Final = (
|
||||
(_as_gettable(litellm_params.get(key)) or {}).get("litellm_parent_otel_span")
|
||||
for key in ("metadata", "litellm_metadata")
|
||||
)
|
||||
return next(
|
||||
(
|
||||
span
|
||||
for span in carried
|
||||
if isinstance(span, trace.Span) and getattr(span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
|
||||
if span is trace.get_current_span():
|
||||
return False
|
||||
parent: Final = _parent_of(span)
|
||||
if parent is None:
|
||||
return True
|
||||
proxy_span: Final = _litellm_proxy_span(kwargs)
|
||||
return (
|
||||
proxy_span is not None
|
||||
and _parent_of(proxy_span) is None
|
||||
and proxy_span.get_span_context().span_id == parent.span_id
|
||||
)
|
||||
|
||||
|
||||
class LangfuseOtelLogger(OpenTelemetry):
|
||||
|
|
@ -129,98 +222,42 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
safe_set_attribute(span, enum_attr.value, value)
|
||||
|
||||
@staticmethod
|
||||
def _set_observation_output(span: Span, response_obj):
|
||||
"""Helper to set observation output attributes."""
|
||||
from litellm.integrations.arize._utils import safe_set_attribute
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
def _observation_output(response_obj: _Gettable | None) -> str | None:
|
||||
if not response_obj or not hasattr(response_obj, "get"):
|
||||
return
|
||||
return None
|
||||
return _extract_output_items(response_obj) or _extract_choices_output(response_obj)
|
||||
|
||||
choices: Final = response_obj.get("choices", [])
|
||||
if choices:
|
||||
first_choice: Final = choices[0]
|
||||
message: Final = first_choice.get("message", {})
|
||||
tool_calls: Final = message.get("tool_calls")
|
||||
if tool_calls:
|
||||
transformed_tool_calls: Final = []
|
||||
for tool_call in tool_calls:
|
||||
function = tool_call.get("function", {})
|
||||
arguments_str = function.get("arguments", "{}")
|
||||
try:
|
||||
arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
|
||||
except json.JSONDecodeError:
|
||||
arguments_obj = {}
|
||||
langfuse_tool_call = {
|
||||
"id": response_obj.get("id", ""),
|
||||
"name": function.get("name", ""),
|
||||
"call_id": tool_call.get("id", ""),
|
||||
"type": "function_call",
|
||||
"arguments": arguments_obj,
|
||||
}
|
||||
transformed_tool_calls.append(langfuse_tool_call)
|
||||
safe_set_attribute(
|
||||
span,
|
||||
LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
|
||||
safe_dumps(transformed_tool_calls),
|
||||
)
|
||||
else:
|
||||
output_data: Final = {}
|
||||
if message.get("role"):
|
||||
output_data["role"] = message.get("role")
|
||||
if message.get("content") is not None:
|
||||
output_data["content"] = message.get("content")
|
||||
if output_data:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
|
||||
safe_dumps(output_data),
|
||||
)
|
||||
@staticmethod
|
||||
def _trace_tags(kwargs: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[str, ...]:
|
||||
caller_tags: Final = metadata.get("tags")
|
||||
request_tags: Final = (_as_gettable(kwargs.get("standard_logging_object")) or {}).get("request_tags")
|
||||
default_tags: Final = litellm.langfuse_default_tags
|
||||
|
||||
output: Final = response_obj.get("output", [])
|
||||
if output:
|
||||
output_items_data: Final[list[dict]] = []
|
||||
for item in output:
|
||||
if hasattr(item, "type"):
|
||||
item_type = item.type
|
||||
if item_type == "reasoning" and hasattr(item, "summary"):
|
||||
for summary in item.summary:
|
||||
if hasattr(summary, "text"):
|
||||
output_items_data.append(
|
||||
{
|
||||
"role": "reasoning_summary",
|
||||
"content": summary.text,
|
||||
}
|
||||
)
|
||||
elif item_type == "message":
|
||||
item_content: Sequence[object] = getattr(item, "content", [{}])
|
||||
output_items_data.append(
|
||||
{
|
||||
"role": getattr(item, "role", "assistant"),
|
||||
"content": getattr(item_content[0], "text", ""),
|
||||
}
|
||||
)
|
||||
elif item_type == "function_call":
|
||||
item_arguments: object = getattr(item, "arguments", "{}")
|
||||
arguments_obj = (
|
||||
safe_json_loads(item_arguments, default={})
|
||||
if isinstance(item_arguments, str)
|
||||
else item_arguments
|
||||
)
|
||||
langfuse_tool_call = {
|
||||
"id": getattr(item, "id", ""),
|
||||
"name": getattr(item, "name", ""),
|
||||
"call_id": getattr(item, "call_id", ""),
|
||||
"type": "function_call",
|
||||
"arguments": arguments_obj,
|
||||
}
|
||||
output_items_data.append(langfuse_tool_call)
|
||||
if output_items_data:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
|
||||
safe_dumps(output_items_data),
|
||||
def _default_tag(key: str) -> str | None:
|
||||
if key == "cache_hit":
|
||||
return f"cache_hit:{kwargs.get('cache_hit', False)}"
|
||||
if key == "cache_key":
|
||||
hidden_params: Final = _as_gettable(metadata.get("hidden_params", {})) or {}
|
||||
cache_key: Final = (
|
||||
hidden_params.get("cache_key")
|
||||
if hidden_params.get("cache_key") is not None
|
||||
else _preset_cache_key(kwargs)
|
||||
)
|
||||
return f"cache_key:{cache_key}"
|
||||
if key == "proxy_base_url":
|
||||
proxy_base_url: Final = os.environ.get("PROXY_BASE_URL")
|
||||
return f"proxy_base_url:{proxy_base_url}" if proxy_base_url is not None else None
|
||||
if key in metadata and metadata[key] is not None:
|
||||
return f"{key}:{metadata[key]}"
|
||||
return None
|
||||
|
||||
expanded: Final = tuple(_default_tag(key) for key in default_tags) if isinstance(default_tags, list) else ()
|
||||
candidates: Final = (
|
||||
_caller_tag_list(caller_tags)
|
||||
+ (tuple(tag for tag in request_tags if isinstance(tag, str)) if isinstance(request_tags, list) else ())
|
||||
+ tuple(tag for tag in expanded if tag is not None)
|
||||
)
|
||||
return tuple(dict.fromkeys(candidates))
|
||||
|
||||
@staticmethod
|
||||
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj):
|
||||
|
|
@ -249,13 +286,12 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
metadata: Final = LangfuseOtelLogger._extract_langfuse_metadata(kwargs)
|
||||
LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata)
|
||||
|
||||
messages: Final = kwargs.get("messages")
|
||||
if messages:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
LangfuseSpanAttributes.OBSERVATION_INPUT.value,
|
||||
safe_dumps(messages),
|
||||
)
|
||||
joins_existing_trace: Final = metadata.get("existing_trace_id") is not None or not _starts_its_own_trace(
|
||||
span, kwargs
|
||||
)
|
||||
input_json: Final = safe_dumps(kwargs.get("messages")) if kwargs.get("messages") else None
|
||||
if input_json is not None:
|
||||
safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_INPUT.value, input_json)
|
||||
|
||||
completion_start_time: Final = kwargs.get("completion_start_time")
|
||||
if isinstance(completion_start_time, datetime):
|
||||
|
|
@ -265,7 +301,64 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
serialize_langfuse_datetime(completion_start_time),
|
||||
)
|
||||
|
||||
LangfuseOtelLogger._set_observation_output(span=span, response_obj=response_obj)
|
||||
output_json: Final = LangfuseOtelLogger._observation_output(response_obj)
|
||||
if output_json is not None:
|
||||
safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, output_json)
|
||||
|
||||
if not joins_existing_trace:
|
||||
LangfuseOtelLogger._set_trace_fields(
|
||||
span=span, kwargs=kwargs, metadata=metadata, input_json=input_json, output_json=output_json
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _set_trace_fields(
|
||||
span: Span,
|
||||
kwargs: Mapping[str, object],
|
||||
metadata: Mapping[str, object],
|
||||
input_json: str | None,
|
||||
output_json: str | None,
|
||||
) -> None:
|
||||
from litellm.integrations.otel.emitter import carried_keys, span_attribute_limit
|
||||
|
||||
tags: Final = LangfuseOtelLogger._trace_tags(kwargs, metadata)
|
||||
fields: Final = (
|
||||
*(
|
||||
((LangfuseSpanAttributes.TRACE_NAME.value, f"litellm-{kwargs.get('call_type') or 'completion'}"),)
|
||||
if metadata.get("trace_name") is None
|
||||
else ()
|
||||
),
|
||||
*(((LangfuseSpanAttributes.TAGS.value, json.dumps(list(tags))),) if tags else ()),
|
||||
*(
|
||||
(
|
||||
(
|
||||
LangfuseSpanAttributes.TRACE_INPUT.value,
|
||||
REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_input")) else input_json,
|
||||
),
|
||||
)
|
||||
if input_json is not None
|
||||
else ()
|
||||
),
|
||||
*(
|
||||
(
|
||||
(
|
||||
LangfuseSpanAttributes.TRACE_OUTPUT.value,
|
||||
REDACTED_BY_LITELLM if _mask_flag(metadata.get("mask_output")) else output_json,
|
||||
),
|
||||
)
|
||||
if output_json is not None
|
||||
else ()
|
||||
),
|
||||
)
|
||||
limit: Final = span_attribute_limit(span)
|
||||
carried: Final = carried_keys(span)
|
||||
new_keys: Final = tuple(key for key, _value in fields if key not in carried)
|
||||
room: Final = (
|
||||
len(new_keys) if limit is None else max(limit - len(carried) - len(_ERROR_ATTRIBUTES - carried), 0)
|
||||
)
|
||||
written: Final = carried | frozenset(new_keys[:room])
|
||||
for key, value in fields:
|
||||
if key in written:
|
||||
_utils.safe_set_attribute(span, key, value)
|
||||
|
||||
@staticmethod
|
||||
def _get_langfuse_otel_host() -> str | None:
|
||||
|
|
@ -457,3 +550,101 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
"""
|
||||
Langfuse should not receive service failure logs.
|
||||
"""
|
||||
|
||||
|
||||
def _extract_choices_output(response_obj: _Gettable) -> str | None:
|
||||
choices: Final = response_obj.get("choices", [])
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return None
|
||||
first_choice: Final = _as_gettable(choices[0])
|
||||
if first_choice is None:
|
||||
return None
|
||||
message: Final = _as_gettable(first_choice.get("message", {}))
|
||||
if message is None:
|
||||
return None
|
||||
tool_calls: Final = message.get("tool_calls")
|
||||
if isinstance(tool_calls, list) and tool_calls:
|
||||
transformed_tool_calls: Final = [
|
||||
entry for tool_call in tool_calls if (entry := _transformed_tool_call(response_obj, tool_call)) is not None
|
||||
]
|
||||
return safe_dumps(transformed_tool_calls)
|
||||
output_data: Final = {
|
||||
key: value
|
||||
for key, value in (
|
||||
("role", message.get("role") or None),
|
||||
("content", message.get("content")),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
return safe_dumps(output_data) if output_data else None
|
||||
|
||||
|
||||
def _transformed_tool_call(response_obj: _Gettable, tool_call: object) -> dict[str, object] | None:
|
||||
call: Final = _as_gettable(tool_call)
|
||||
if call is None:
|
||||
return None
|
||||
function: Final = _as_gettable(call.get("function", {}))
|
||||
if function is None:
|
||||
return None
|
||||
return {
|
||||
"id": response_obj.get("id", ""),
|
||||
"name": function.get("name", ""),
|
||||
"call_id": call.get("id", ""),
|
||||
"type": "function_call",
|
||||
"arguments": _tool_call_arguments(function.get("arguments", "{}")),
|
||||
}
|
||||
|
||||
|
||||
def _tool_call_arguments(arguments: object) -> object:
|
||||
if not isinstance(arguments, str):
|
||||
return arguments
|
||||
try:
|
||||
return json.loads(arguments)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
def _extract_output_items(response_obj: _Gettable) -> str | None:
|
||||
output: Final = response_obj.get("output", [])
|
||||
if not isinstance(output, list) or not output:
|
||||
return None
|
||||
rendered: Final = tuple(chain.from_iterable(map(_output_items, output)))
|
||||
return safe_dumps(list(rendered)) if rendered else None
|
||||
|
||||
|
||||
def _output_items(item: object) -> tuple[dict[str, object], ...]:
|
||||
item_type: Final = _attr(item, "type")
|
||||
if item_type == "reasoning":
|
||||
summaries: Final = _attr(item, "summary")
|
||||
if not isinstance(summaries, Iterable):
|
||||
return ()
|
||||
return tuple(
|
||||
{"role": "reasoning_summary", "content": _attr(summary, "text")}
|
||||
for summary in summaries
|
||||
if hasattr(summary, "text")
|
||||
)
|
||||
if item_type == "message":
|
||||
content_items: Final = _attr(item, "content", [{}])
|
||||
first_content: Final = (
|
||||
content_items[0]
|
||||
if isinstance(content_items, Sequence) and not isinstance(content_items, (str, bytes)) and content_items
|
||||
else {}
|
||||
)
|
||||
return (
|
||||
{
|
||||
"role": _attr(item, "role", "assistant"),
|
||||
"content": _attr(first_content, "text", ""),
|
||||
},
|
||||
)
|
||||
if item_type == "function_call":
|
||||
arguments: Final = _attr(item, "arguments", "{}")
|
||||
return (
|
||||
{
|
||||
"id": _attr(item, "id", ""),
|
||||
"name": _attr(item, "name", ""),
|
||||
"call_id": _attr(item, "call_id", ""),
|
||||
"type": "function_call",
|
||||
"arguments": safe_json_loads(arguments, default={}) if isinstance(arguments, str) else arguments,
|
||||
},
|
||||
)
|
||||
return ()
|
||||
|
|
|
|||
|
|
@ -1,18 +1,25 @@
|
|||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.mappers.langfuse import (
|
||||
LANGFUSE_OBSERVATION_INPUT,
|
||||
LANGFUSE_OBSERVATION_OUTPUT,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LangfuseMapper,
|
||||
)
|
||||
from litellm.integrations.otel.model.metadata import internal_call_origin
|
||||
from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output
|
||||
from litellm.integrations.otel.model.trace_controls import caller_trace_controls
|
||||
from litellm.integrations.otel.model.utils import as_str
|
||||
from litellm.integrations.otel.plumbing.context import request_root_span
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
|
|
@ -24,9 +31,23 @@ class LangfuseOpenTelemetryV2(OpenTelemetryV2):
|
|||
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
|
||||
root: Final = request_root_span()
|
||||
if root is not None and root.is_recording():
|
||||
root.set_attributes(LangfuseMapper.trace_attributes(caller_trace_controls(kwargs)))
|
||||
root.set_attributes(
|
||||
LangfuseMapper.trace_attributes(
|
||||
caller_trace_controls(kwargs),
|
||||
as_str(kwargs.get("call_type")) if self._derives_trace_name(root, kwargs) else None,
|
||||
)
|
||||
)
|
||||
super().log_pre_api_call(model, messages, kwargs)
|
||||
|
||||
def _derives_trace_name(self, root: "Span", kwargs: Mapping[str, object]) -> bool:
|
||||
return (
|
||||
internal_call_origin(None, kwargs) is None
|
||||
and isinstance(root, ReadableSpan)
|
||||
and root.parent is None
|
||||
and root.resource is self.tracer_provider.resource
|
||||
and LANGFUSE_TRACE_NAME not in (root.attributes or {})
|
||||
)
|
||||
|
||||
|
||||
class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2):
|
||||
"""Stamps the request's input and output on the root observation while it is still recording.
|
||||
|
|
|
|||
|
|
@ -96,10 +96,10 @@ class LangfuseMapper:
|
|||
return {}
|
||||
|
||||
@staticmethod
|
||||
def trace_attributes(trace: TraceControls) -> AttributeMap:
|
||||
def trace_attributes(trace: TraceControls, call_type: str | None = None) -> AttributeMap:
|
||||
return drop_none_pairs(
|
||||
(
|
||||
(LANGFUSE_TRACE_NAME, trace.name or None),
|
||||
(LANGFUSE_TRACE_NAME, trace.name or (f"litellm-{call_type}" if call_type else None)),
|
||||
(LANGFUSE_TRACE_USER_ID, trace.user_id or None),
|
||||
(LANGFUSE_TRACE_SESSION_ID, trace.session_id or None),
|
||||
(LANGFUSE_TRACE_TAGS, trace.tags or None),
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.integrations.otel.mappers.utils import json_or_none
|
||||
from litellm.llms.anthropic.pass_through.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream
|
||||
from litellm.llms.anthropic.messages.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ def _chat_request_from_anthropic_messages(
|
|||
because the logged optional_params switch dialect per provider path (the bridge's
|
||||
inner completion rewrites them to chat shape mid-flight); the adapter translates
|
||||
them alongside the messages, and sampling params copy through untranslated."""
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ Claude Code (Anthropic's official CLI) sends web search requests using Anthropic
|
|||
|
||||
Native tools are converted to LiteLLM standard format **before** sending to the provider:
|
||||
|
||||
1. **Conversion Point** (`litellm/llms/anthropic/pass_through/messages/handler.py`):
|
||||
1. **Conversion Point** (`litellm/llms/anthropic/messages/native/handler.py`):
|
||||
- In `anthropic_messages()` function (lines 60-127)
|
||||
- Runs BEFORE the API request is made
|
||||
- Detects native web search tools using `is_web_search_tool()`
|
||||
|
|
|
|||
|
|
@ -313,19 +313,35 @@ def _spend_metadata(logging_obj: "LiteLLMLoggingObj") -> Mapping[str, object]:
|
|||
}
|
||||
|
||||
|
||||
def _sent_tags_are_a_list(litellm_params: Mapping[str, object]) -> bool:
|
||||
"""Whether the first non-empty ``tags`` in ``metadata`` then ``litellm_metadata`` is absent or a list."""
|
||||
buckets: Final = (litellm_params.get("metadata"), litellm_params.get("litellm_metadata"))
|
||||
sent: Final = next(
|
||||
(
|
||||
tags
|
||||
for bucket in buckets
|
||||
if isinstance(bucket, Mapping) and (tags := _OBJECT_MAPPING.validate_python(bucket).get("tags"))
|
||||
),
|
||||
None,
|
||||
)
|
||||
return sent is None or isinstance(sent, list)
|
||||
|
||||
|
||||
def _request_tags(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
|
||||
try:
|
||||
return {
|
||||
"tags": StandardLoggingPayloadSetup.get_request_tags(
|
||||
litellm_params=_OBJECT_DICT.validate_python(litellm_params),
|
||||
proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}),
|
||||
)
|
||||
}
|
||||
if _sent_tags_are_a_list(litellm_params):
|
||||
return {
|
||||
"tags": StandardLoggingPayloadSetup.get_request_tags(
|
||||
litellm_params=_OBJECT_DICT.validate_python(litellm_params),
|
||||
proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}),
|
||||
)
|
||||
}
|
||||
except AttributeError:
|
||||
verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list")
|
||||
return {}
|
||||
pass
|
||||
verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list")
|
||||
return {}
|
||||
|
||||
|
||||
def _create_context(logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str) -> BackgroundInteractionCreateContext:
|
||||
|
|
|
|||
73
litellm/litellm_core_utils/deprecated_package.py
Normal file
73
litellm/litellm_core_utils/deprecated_package.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import importlib
|
||||
import importlib.abc
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import sys
|
||||
import warnings
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import ModuleType
|
||||
from typing import Final
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AliasLoader(importlib.abc.Loader):
|
||||
new_dotted: str
|
||||
canonical_spec: importlib.machinery.ModuleSpec
|
||||
old_dotted: str
|
||||
|
||||
def create_module(self, spec: importlib.machinery.ModuleSpec) -> ModuleType:
|
||||
return importlib.import_module(self.new_dotted)
|
||||
|
||||
def exec_module(self, module: ModuleType) -> None:
|
||||
module.__spec__ = self.canonical_spec # rebind-ok: [LIT011] restore import metadata overwritten by importlib
|
||||
if self.canonical_spec.submodule_search_locations is not None:
|
||||
_alias_attributes(self.old_dotted, self.new_dotted)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AliasFinder(importlib.abc.MetaPathFinder):
|
||||
old_name: str
|
||||
new_name: str
|
||||
|
||||
def find_spec(
|
||||
self,
|
||||
fullname: str,
|
||||
path: Sequence[str] | None = None,
|
||||
target: ModuleType | None = None,
|
||||
) -> importlib.machinery.ModuleSpec | None:
|
||||
if not fullname.startswith(f"{self.old_name}."):
|
||||
return None
|
||||
new_dotted: Final = f"{self.new_name}{fullname.removeprefix(self.old_name)}"
|
||||
new_spec: Final = importlib.util.find_spec(new_dotted)
|
||||
if new_spec is None:
|
||||
return None
|
||||
return importlib.util.spec_from_loader(
|
||||
fullname,
|
||||
_AliasLoader(new_dotted, new_spec, fullname),
|
||||
is_package=new_spec.submodule_search_locations is not None,
|
||||
)
|
||||
|
||||
|
||||
def _deprecated_attribute(old_name: str, new_name: str, name: str) -> ModuleType:
|
||||
if importlib.util.find_spec(f"{new_name}.{name}") is None:
|
||||
raise AttributeError(f"module {old_name!r} has no attribute {name!r}")
|
||||
return importlib.import_module(f"{old_name}.{name}")
|
||||
|
||||
|
||||
def _alias_attributes(old_name: str, new_name: str) -> None:
|
||||
if "__getattr__" not in vars(sys.modules[old_name]):
|
||||
sys.modules[old_name].__getattr__ = partial(_deprecated_attribute, old_name, new_name)
|
||||
|
||||
|
||||
def alias_deprecated_package(old_name: str, new_package: ModuleType) -> None:
|
||||
warnings.warn(
|
||||
f"{old_name} is deprecated, import {new_package.__name__} instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
finder: Final = _AliasFinder(old_name, new_package.__name__)
|
||||
if finder not in sys.meta_path:
|
||||
sys.meta_path.insert(0, finder)
|
||||
_alias_attributes(old_name, new_package.__name__)
|
||||
|
|
@ -97,8 +97,13 @@ from litellm.litellm_core_utils.internal_call_metadata import (
|
|||
is_unbilled_non_inference_call,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
|
||||
bedrock_guardrail_trace,
|
||||
cost_breakdown_with_guardrail,
|
||||
guardrail_information_cost,
|
||||
is_str_keyed_dict,
|
||||
native_bedrock_guardrail_information,
|
||||
object_mapping,
|
||||
with_guardrail_entry,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
|
||||
StandardBuiltInToolCostTracking,
|
||||
|
|
@ -2725,6 +2730,31 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return
|
||||
self.model_call_details["response_headers"] = headers
|
||||
|
||||
def record_native_bedrock_guardrail(self, result: object) -> None:
|
||||
"""Bill the guardrail trace a Bedrock response carries when the deployment's ``guardrailConfig`` ran it, by
|
||||
recording it as guardrail information on this call, the same entry the proxy-side Bedrock guardrail writes.
|
||||
A cache hit carries the original response's trace but made no Bedrock call, so it bills nothing."""
|
||||
trace: Final = bedrock_guardrail_trace(result)
|
||||
if trace is None or served_from_cache(result) or self.model_call_details.get("cache_hit") is True:
|
||||
return
|
||||
optional_params: Final = object_mapping(getattr(self, "optional_params", None)) or {}
|
||||
guardrail_config: Final = object_mapping(optional_params.get("guardrailConfig")) or {}
|
||||
identifier: Final = guardrail_config.get("guardrailIdentifier")
|
||||
litellm_params_in: Final = object_mapping(getattr(self, "litellm_params", None)) or {}
|
||||
region: Final = optional_params.get("aws_region_name") or litellm_params_in.get("aws_region_name")
|
||||
entry: Final = native_bedrock_guardrail_information(
|
||||
trace,
|
||||
aws_region_name=region if isinstance(region, str) else None,
|
||||
guardrail_identifier=identifier if isinstance(identifier, str) else None,
|
||||
)
|
||||
litellm_params: Final[object] = self.model_call_details.get("litellm_params")
|
||||
if entry is None or not is_str_keyed_dict(litellm_params):
|
||||
return
|
||||
metadata: Final[object] = litellm_params.get("metadata")
|
||||
bucket: Final = metadata if is_str_keyed_dict(metadata) else {}
|
||||
litellm_params["metadata"] = bucket
|
||||
with_guardrail_entry(bucket, entry)
|
||||
|
||||
def _merge_hidden_params_from_response_into_metadata(self, logging_result: object) -> None:
|
||||
"""
|
||||
Copy response._hidden_params into litellm_params.metadata['hidden_params'].
|
||||
|
|
@ -7130,12 +7160,12 @@ class StandardLoggingPayloadSetup:
|
|||
request_headers: Final = cast( # cast-ok: request headers are untyped framework data
|
||||
dict[str, str], headers
|
||||
)
|
||||
|
||||
header_tags: Final = []
|
||||
for header_name in extra_headers:
|
||||
header_value = request_headers.get(header_name)
|
||||
if header_value:
|
||||
header_tags.append(f"{header_name}: {header_value}")
|
||||
lowercase_request_headers: Final = {str(name).lower(): value for name, value in request_headers.items()}
|
||||
header_tags: Final = [
|
||||
f"{header_name}: {header_value}"
|
||||
for header_name in extra_headers
|
||||
if (header_value := lowercase_request_headers.get(header_name.lower()))
|
||||
]
|
||||
|
||||
return header_tags if header_tags else None
|
||||
|
||||
|
|
@ -7150,19 +7180,17 @@ class StandardLoggingPayloadSetup:
|
|||
litellm_metadata: Final = cast( # cast-ok: request metadata is caller-provided
|
||||
Mapping[str, object], litellm_params.get("litellm_metadata") or {}
|
||||
)
|
||||
if metadata.get("tags", []):
|
||||
request_tags = cast(list[str], metadata.get("tags", [])).copy() # cast-ok: tags are caller-provided
|
||||
elif litellm_metadata.get("tags", []):
|
||||
request_tags = cast(list[str], litellm_metadata.get("tags", [])).copy() # cast-ok: tags are caller-provided
|
||||
else:
|
||||
request_tags = []
|
||||
request_tags: Final = next(
|
||||
(
|
||||
cast(list[str], tags) # cast-ok: tags are caller-provided
|
||||
for tags in (metadata.get("tags"), litellm_metadata.get("tags"))
|
||||
if isinstance(tags, list) and tags
|
||||
),
|
||||
(),
|
||||
)
|
||||
user_agent_tags: Final = StandardLoggingPayloadSetup._get_user_agent_tags(proxy_server_request)
|
||||
additional_header_tags: Final = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request)
|
||||
if user_agent_tags is not None:
|
||||
request_tags.extend(user_agent_tags)
|
||||
if additional_header_tags is not None:
|
||||
request_tags.extend(additional_header_tags)
|
||||
return request_tags
|
||||
return [*request_tags, *(user_agent_tags or ()), *(additional_header_tags or ())]
|
||||
|
||||
_get_request_tags = get_request_tags
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,24 @@
|
|||
import math
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from typing import Annotated, Final
|
||||
|
||||
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
redact_nested_match_and_regex_keys, # pyright: ignore[reportUnknownVariableType] # signature uses bare dict
|
||||
)
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import CostBreakdown
|
||||
from litellm.types.utils import CostBreakdown, GuardrailStatus, StandardLoggingGuardrailInformation
|
||||
|
||||
BEDROCK_GUARDRAIL_PRICING_KEY: Final = "bedrock/guardrails"
|
||||
BEDROCK_TRACE_FIELDS: Final = ("trace", "amazon-bedrock-trace")
|
||||
_INTERVENING_ACTIONS: Final = frozenset({"BLOCKED", "ANONYMIZED"})
|
||||
|
||||
|
||||
class GuardrailPricing(LiteLLMBaseModel):
|
||||
|
|
@ -96,6 +105,161 @@ def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str
|
|||
return guardrail_cost_total(bedrock_guardrail_cost_by_unit(usage_units, aws_region_name))
|
||||
|
||||
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_OBJECT_ITEMS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def object_mapping(value: object) -> Mapping[str, object] | None:
|
||||
"""``value`` as a str-keyed mapping, None for anything else. Vendor JSON arrives untyped, so this is the one place
|
||||
that turns it into something the rest of the module can read without guessing."""
|
||||
try:
|
||||
return _OBJECT_MAPPING.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def object_items(value: object) -> tuple[object, ...]:
|
||||
if isinstance(value, (str, bytes, Mapping)):
|
||||
return ()
|
||||
try:
|
||||
return _OBJECT_ITEMS.validate_python(value)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def is_str_keyed_dict(
|
||||
value: object,
|
||||
) -> TypeIs[dict[str, object]]: # guard-ok: JSON object keys are str # mutable-ok: the caller edits this dict in place
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _trace_field(response: object, field: str) -> Mapping[str, object] | None:
|
||||
source: Final = object_mapping(response)
|
||||
trace: Final = object_mapping(source.get(field) if source is not None else getattr(response, field, None))
|
||||
if trace is None or object_mapping(trace.get("guardrail")) is None:
|
||||
return None
|
||||
return trace
|
||||
|
||||
|
||||
def bedrock_guardrail_trace(response: object) -> Mapping[str, object] | None:
|
||||
"""The guardrail trace a Bedrock response carries: Converse puts it under ``trace``, Invoke under
|
||||
``amazon-bedrock-trace``. Returned as is, or None when the response has no guardrail section."""
|
||||
traces: Final = map(partial(_trace_field, response), BEDROCK_TRACE_FIELDS)
|
||||
return next((trace for trace in traces if trace is not None), None)
|
||||
|
||||
|
||||
class _Assessment(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
invocation_metrics: Mapping[str, JsonValue] = Field(default_factory=dict, alias="invocationMetrics")
|
||||
|
||||
@property
|
||||
def billable_usage(self) -> Mapping[str, int]:
|
||||
usage: Final = self.invocation_metrics.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
return {}
|
||||
return {
|
||||
counter: units
|
||||
for counter, units in usage.items()
|
||||
if counter.endswith("Units") and isinstance(units, int) and not isinstance(units, bool)
|
||||
}
|
||||
|
||||
|
||||
class _GuardrailTrace(BaseModel):
|
||||
"""Both trace shapes: Converse ``inputAssessment`` {id: a} and ``outputAssessments`` {id: [a, ...]}, Invoke
|
||||
``input`` {id: a} and ``outputs`` [{id: a}, ...]."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
input_assessment: Mapping[str, _Assessment] = Field(default_factory=dict, alias="inputAssessment")
|
||||
input: Mapping[str, _Assessment] = Field(default_factory=dict)
|
||||
output_assessments: Mapping[str, tuple[_Assessment, ...]] = Field(default_factory=dict, alias="outputAssessments")
|
||||
outputs: tuple[Mapping[str, _Assessment], ...] = ()
|
||||
|
||||
@property
|
||||
def assessments(self) -> tuple[tuple[str, _Assessment], ...]:
|
||||
listed: Final = (
|
||||
(gid, a) for gid, items in self.output_assessments.items() for a in items
|
||||
) # comprehension-ok: id then its list
|
||||
return (
|
||||
*self.input_assessment.items(),
|
||||
*self.input.items(),
|
||||
*listed,
|
||||
*chain.from_iterable(by_id.items() for by_id in self.outputs),
|
||||
)
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _summed_usage(usages: Sequence[Mapping[str, int]]) -> Mapping[str, int]:
|
||||
counters: Final = tuple(dict.fromkeys(chain.from_iterable(usages)))
|
||||
return {counter: sum(usage.get(counter, 0) for usage in usages) for counter in counters}
|
||||
|
||||
|
||||
def _actions(node: JsonValue) -> Iterator[str]:
|
||||
if isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_actions, node))
|
||||
return
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
action: Final = node.get("action")
|
||||
if isinstance(action, str):
|
||||
yield action
|
||||
yield from chain.from_iterable(map(_actions, node.values()))
|
||||
|
||||
|
||||
def native_bedrock_guardrail_information(
|
||||
trace: Mapping[str, object], aws_region_name: str | None, guardrail_identifier: str | None
|
||||
) -> StandardLoggingGuardrailInformation | None:
|
||||
"""The guardrail entry for a trace Bedrock returned because the deployment's ``guardrailConfig`` ran with
|
||||
``trace: enabled``: every assessment's ``invocationMetrics.usage`` summed and priced like the proxy-side Bedrock
|
||||
guardrail. None when the trace carries no usage, which is what ``trace: disabled`` leaves."""
|
||||
try:
|
||||
guardrail: Final = _JSON_OBJECT.validate_python(trace.get("guardrail"))
|
||||
pairs: Final = _GuardrailTrace.model_validate(guardrail).assessments
|
||||
except ValidationError:
|
||||
return None
|
||||
usage: Final = _summed_usage(tuple(assessment.billable_usage for _, assessment in pairs))
|
||||
if not usage:
|
||||
return None
|
||||
cost_by_unit: Final = bedrock_guardrail_cost_by_unit(usage, aws_region_name)
|
||||
name: Final = guardrail_identifier or pairs[0][0]
|
||||
status: Final[GuardrailStatus] = (
|
||||
"guardrail_intervened" if any(action in _INTERVENING_ACTIONS for action in _actions(guardrail)) else "success"
|
||||
)
|
||||
priced: Final[StandardLoggingGuardrailInformation] = (
|
||||
{"guardrail_cost_by_unit": cost_by_unit} if cost_by_unit is not None else {}
|
||||
)
|
||||
return {
|
||||
"guardrail_name": name,
|
||||
"guardrail_id": name,
|
||||
"guardrail_provider": "bedrock",
|
||||
"guardrail_mode": GuardrailEventHooks.during_call,
|
||||
"guardrail_response": redact_nested_match_and_regex_keys(guardrail),
|
||||
"guardrail_status": status,
|
||||
"guardrail_usage": usage,
|
||||
"guardrail_cost": guardrail_cost_total(cost_by_unit),
|
||||
**priced,
|
||||
}
|
||||
|
||||
|
||||
def guardrail_entries(raw: object) -> tuple[object, ...]:
|
||||
"""The entries under ``standard_logging_guardrail_information``, which older writers stored as one dict."""
|
||||
single: Final = object_mapping(raw)
|
||||
return (single,) if single is not None else object_items(raw)
|
||||
|
||||
|
||||
def with_guardrail_entry(
|
||||
metadata: dict[str, object], # mutable-ok: the proxy's own request metadata, read later for headers and spend
|
||||
entry: StandardLoggingGuardrailInformation,
|
||||
) -> None:
|
||||
"""Append ``entry`` to the metadata's ``standard_logging_guardrail_information`` once, so the same trace recorded
|
||||
from the response headers path and again from the success handler bills a single time."""
|
||||
existing: Final = guardrail_entries(metadata.get("standard_logging_guardrail_information"))
|
||||
if entry in existing:
|
||||
return
|
||||
metadata["standard_logging_guardrail_information"] = [*existing, entry] # rebind-ok: appending here is the job
|
||||
|
||||
|
||||
AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT: Final = "text_records"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6354,7 +6354,7 @@ def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
|
|||
|
||||
|
||||
def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from collections.abc import Mapping, Sequence
|
|||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
|
|
@ -120,11 +120,55 @@ def _redacted_streaming_response_copy(streaming_response: object) -> object:
|
|||
return _redact_streaming_response(redacted_response)
|
||||
|
||||
|
||||
BEDROCK_GUARDRAIL_TRACE_FIELDS: Final = ("trace", "amazon-bedrock-trace")
|
||||
_MATCHED_TEXT_KEYS: Final = frozenset({"match", "regex"})
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _without_matched_text(trace: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
|
||||
redacted: Final = dict(copy.deepcopy(trace))
|
||||
stack: Final[list[JsonValue]] = [redacted] # mutable-ok: iterative walk, recursive_detector bans recursion here
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if isinstance(node, dict):
|
||||
for key in _MATCHED_TEXT_KEYS & node.keys():
|
||||
node[key] = "[REDACTED]"
|
||||
stack.extend(node.values())
|
||||
elif isinstance(node, list):
|
||||
stack.extend(node)
|
||||
return redacted
|
||||
|
||||
|
||||
def _redacted_bedrock_trace(trace: object) -> object:
|
||||
try:
|
||||
return _without_matched_text(_JSON_OBJECT.validate_python(trace))
|
||||
except ValidationError:
|
||||
return trace
|
||||
|
||||
|
||||
def _redact_bedrock_guardrail_trace(response: object) -> None:
|
||||
"""A Bedrock guardrail trace echoes the matched text of its sensitive-information and word findings back."""
|
||||
mapping: Final = _as_mapping(response)
|
||||
if isinstance(response, dict):
|
||||
traces: Final = {
|
||||
field: _redacted_bedrock_trace(value)
|
||||
for field, value in (mapping or {}).items()
|
||||
if field in BEDROCK_GUARDRAIL_TRACE_FIELDS
|
||||
}
|
||||
response.update(traces)
|
||||
return
|
||||
for field, value in ((name, getattr(response, name, None)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
|
||||
if value is not None:
|
||||
setattr(response, field, _redacted_bedrock_trace(value))
|
||||
|
||||
|
||||
def _redact_streaming_response(streaming_response):
|
||||
if hasattr(streaming_response, "choices"):
|
||||
for choice in streaming_response.choices:
|
||||
_redact_choice_content(choice)
|
||||
redact_vertex_ai_metadata_from_logged_object(streaming_response)
|
||||
if isinstance(streaming_response, litellm.ModelResponse):
|
||||
_redact_bedrock_guardrail_trace(streaming_response)
|
||||
elif isinstance(streaming_response, litellm.ResponsesAPIResponse):
|
||||
return perform_redaction({}, streaming_response)
|
||||
elif hasattr(streaming_response, "output"):
|
||||
|
|
@ -351,6 +395,7 @@ def _redact_standard_logging_object(
|
|||
if isinstance(response.get("choices"), list):
|
||||
_redact_model_response_dict_choices(response["choices"], redacted_str)
|
||||
redact_vertex_ai_metadata_from_logged_object(response)
|
||||
_redact_bedrock_guardrail_trace(response)
|
||||
elif isinstance(response, str):
|
||||
standard_logging_object["response"] = redacted_str
|
||||
else:
|
||||
|
|
@ -523,11 +568,13 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
for choice in _result.choices:
|
||||
_redact_choice_content(choice)
|
||||
redact_vertex_ai_metadata_from_logged_object(_result)
|
||||
_redact_bedrock_guardrail_trace(_result)
|
||||
elif isinstance(_result, dict) and "choices" in _result:
|
||||
# Handle dict representation of ModelResponse (e.g., from model_dump())
|
||||
if _result.get("choices") is not None:
|
||||
_redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
|
||||
redact_vertex_ai_metadata_from_logged_object(_result)
|
||||
_redact_bedrock_guardrail_trace(_result)
|
||||
elif isinstance(_result, dict) and "output" in _result:
|
||||
return _redacted_responses_api_response(_result, redact_instructions=redact_streaming_responses)
|
||||
elif isinstance(_result, litellm.ResponsesAPIResponse):
|
||||
|
|
|
|||
87
litellm/litellm_core_utils/renamed_package_shim.py
Normal file
87
litellm/litellm_core_utils/renamed_package_shim.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import importlib
|
||||
import sys
|
||||
import warnings
|
||||
from collections.abc import Callable, Sequence
|
||||
from importlib.abc import Loader, MetaPathFinder
|
||||
from importlib.machinery import ModuleSpec
|
||||
from importlib.util import spec_from_loader
|
||||
from types import ModuleType
|
||||
from typing import Final
|
||||
|
||||
|
||||
class _AliasLoader(Loader):
|
||||
"""Hand the import system the module already loaded under its new name instead of executing a copy."""
|
||||
|
||||
def __init__(self, target: ModuleType) -> None:
|
||||
self._target: Final = target
|
||||
self._spec: Final = target.__spec__
|
||||
|
||||
def create_module(self, spec: ModuleSpec) -> ModuleType:
|
||||
return self._target
|
||||
|
||||
def exec_module(self, module: ModuleType) -> None:
|
||||
module.__spec__ = self._spec # rebind-ok: the loader restores the spec the import system replaced
|
||||
|
||||
|
||||
class _RenamedPackageFinder(MetaPathFinder):
|
||||
def __init__(self, old_name: str, renames: tuple[tuple[str, str], ...]) -> None:
|
||||
self._prefix: Final = old_name + "."
|
||||
self._renames: Final = renames
|
||||
|
||||
def new_name_for(self, old_module_name: str) -> str | None:
|
||||
if not old_module_name.startswith(self._prefix):
|
||||
return None
|
||||
for old_prefix, new_prefix in self._renames:
|
||||
if old_module_name == old_prefix or old_module_name.startswith(old_prefix + "."):
|
||||
return new_prefix + old_module_name[len(old_prefix) :]
|
||||
return None
|
||||
|
||||
def find_spec(
|
||||
self,
|
||||
fullname: str,
|
||||
path: Sequence[str] | None = None,
|
||||
target: ModuleType | None = None,
|
||||
) -> ModuleSpec | None:
|
||||
new_name: Final = self.new_name_for(fullname)
|
||||
if new_name is None:
|
||||
return None
|
||||
module: Final = importlib.import_module(new_name)
|
||||
spec: Final = spec_from_loader(fullname, _AliasLoader(module), is_package=hasattr(module, "__path__"))
|
||||
if spec is not None and spec.submodule_search_locations is not None:
|
||||
spec.submodule_search_locations = list(module.__path__) # pyright: ignore[reportAny] # module attributes are untyped
|
||||
return spec
|
||||
|
||||
|
||||
def alias_renamed_package(
|
||||
old_name: str,
|
||||
new_name: str,
|
||||
*,
|
||||
nested_renames: tuple[tuple[str, str], ...] = (),
|
||||
) -> Callable[[str], ModuleType]:
|
||||
"""Make every module of the package now at ``new_name`` importable under its old dotted name.
|
||||
|
||||
Nothing is imported up front: a meta path finder resolves ``old_name.x.y`` to the module object at
|
||||
``new_name.x.y`` the moment something imports it, so a deep import of the old path is the very same object
|
||||
as the new path and importing one old module loads no more than importing its new counterpart would.
|
||||
``nested_renames`` lists (old, new) dotted prefixes for sub-packages that changed name inside the move,
|
||||
longest first. The returned function is the shim package's module ``__getattr__``, so attribute access on
|
||||
the old package reaches the renamed sub-packages too.
|
||||
"""
|
||||
warnings.warn(
|
||||
f"{old_name} moved to {new_name}; update imports, the old path will be removed in a future release",
|
||||
DeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
finder: Final = _RenamedPackageFinder(old_name, (*nested_renames, (old_name, new_name)))
|
||||
if not any(
|
||||
isinstance(entry, _RenamedPackageFinder) and entry.new_name_for(old_name + ".x") for entry in sys.meta_path
|
||||
):
|
||||
sys.meta_path.insert(0, finder)
|
||||
|
||||
def getattr_(name: str) -> ModuleType:
|
||||
try:
|
||||
return importlib.import_module(f"{old_name}.{name}")
|
||||
except ModuleNotFoundError as error:
|
||||
raise AttributeError(f"module {old_name!r} has no attribute {name!r}") from error
|
||||
|
||||
return getattr_
|
||||
|
|
@ -110,7 +110,7 @@ def served_output_texts(response: object) -> ServedTexts | None:
|
|||
def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None:
|
||||
if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks):
|
||||
return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)))
|
||||
from litellm.llms.anthropic.pass_through.stream_assembly import (
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
assemble_anthropic_sse_stream,
|
||||
is_anthropic_sse_stream,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
|
||||
)
|
||||
|
|
@ -1484,6 +1485,35 @@ def _simple_text_part(choices: Sequence[object]) -> str | None:
|
|||
return content if isinstance(content, str) else ""
|
||||
|
||||
|
||||
_NO_TRACE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _carry_bedrock_guardrail_trace(chunks: Sequence[object], response: ModelResponse) -> None:
|
||||
"""Bedrock reports a guardrail trace once per stream, on the final metadata chunk, so the built response carries it
|
||||
under the same field as the non-streaming response (``trace`` on Converse, ``amazon-bedrock-trace`` on Invoke)."""
|
||||
traced: Final = map(_bedrock_trace_fields, reversed(chunks))
|
||||
carried: Final = next((fields for fields in traced if fields), _NO_TRACE_FIELDS)
|
||||
for field, trace in carried.items():
|
||||
setattr(response, field, trace)
|
||||
|
||||
|
||||
def _record_bedrock_guardrail_trace(
|
||||
chunks: Sequence[object], response: ModelResponse, logging_obj: Optional["Logging"]
|
||||
) -> None:
|
||||
_carry_bedrock_guardrail_trace(chunks, response)
|
||||
if logging_obj is not None:
|
||||
logging_obj.record_native_bedrock_guardrail(response)
|
||||
|
||||
|
||||
def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]:
|
||||
as_mapping: Final = object_mapping(chunk)
|
||||
raw: Final = (
|
||||
as_mapping.get("provider_specific_fields") if as_mapping else getattr(chunk, "provider_specific_fields", None)
|
||||
)
|
||||
fields: Final[Mapping[str, object]] = object_mapping(raw) or {}
|
||||
return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields}
|
||||
|
||||
|
||||
def stream_chunk_builder(
|
||||
chunks: list,
|
||||
messages: Sequence | None = None,
|
||||
|
|
@ -1565,6 +1595,11 @@ def stream_chunk_builder(
|
|||
)
|
||||
break
|
||||
|
||||
_record_bedrock_guardrail_trace(
|
||||
object_items(chunks), # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param
|
||||
response,
|
||||
logging_obj,
|
||||
)
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(
|
||||
response,
|
||||
|
|
@ -1760,6 +1795,11 @@ def stream_chunk_builder(
|
|||
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
|
||||
)
|
||||
|
||||
_record_bedrock_guardrail_trace(
|
||||
object_items(chunks), # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param
|
||||
response,
|
||||
logging_obj,
|
||||
)
|
||||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
return response
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -1610,6 +1610,8 @@ class CustomStreamWrapper:
|
|||
return _ProviderChunkParsed(response_obj)
|
||||
|
||||
def chunk_creator(self, chunk: Any):
|
||||
if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields:
|
||||
self.logging_obj.record_native_bedrock_guardrail(chunk.provider_specific_fields)
|
||||
if self.custom_llm_provider != "cached_response" and isinstance(
|
||||
chunk, (ChatCompletionChunk, ModelResponseStream)
|
||||
):
|
||||
|
|
@ -1717,6 +1719,8 @@ class CustomStreamWrapper:
|
|||
model_response.usage = Usage(**chunk_usage.model_dump())
|
||||
|
||||
## RETURN ARG
|
||||
if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields:
|
||||
self.copy_model_response_level_provider_specific_fields(chunk, model_response)
|
||||
result: Final = self.return_processed_chunk_logic(
|
||||
completion_obj=completion_obj,
|
||||
model_response=model_response,
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ from litellm.types.llms.anthropic import (
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionDocumentObject,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolParam,
|
||||
|
|
@ -456,7 +455,7 @@ def token_counter(
|
|||
messages: Sequence[AllMessageValues | Message] | None = None,
|
||||
count_response_tokens: bool | None = False,
|
||||
tools: list[ChatCompletionToolParam] | None = None,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None = None,
|
||||
tool_choice: str | Mapping[str, object] | None = None,
|
||||
use_default_image_token_count: bool | None = False,
|
||||
default_token_count: int | None = None,
|
||||
) -> int:
|
||||
|
|
@ -636,7 +635,7 @@ def _count_messages(
|
|||
def _count_extra(
|
||||
count_function: TokenCounterFunction,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None,
|
||||
tool_choice: str | Mapping[str, object] | None,
|
||||
includes_system_message: bool,
|
||||
tool_section: ChatTemplateToolSection | None,
|
||||
) -> int:
|
||||
|
|
@ -673,13 +672,19 @@ def _count_tool_definitions(
|
|||
|
||||
def _count_tool_choice(
|
||||
count_function: TokenCounterFunction,
|
||||
tool_choice: ChatCompletionNamedToolChoiceParam | None,
|
||||
tool_choice: str | Mapping[str, object] | None,
|
||||
) -> int:
|
||||
if tool_choice == "none":
|
||||
return 1
|
||||
if isinstance(tool_choice, dict):
|
||||
return 7 + count_function(str(tool_choice["function"]["name"]))
|
||||
return 0
|
||||
match tool_choice:
|
||||
case "none":
|
||||
return 1
|
||||
case {"function": {"name": str() as name}}:
|
||||
return 7 + count_function(name)
|
||||
case {"type": "none"}:
|
||||
return 1
|
||||
case {"name": str() as name}:
|
||||
return 7 + count_function(name)
|
||||
case _:
|
||||
return 0
|
||||
|
||||
|
||||
def _count_chat_template_tools(
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans
|
|||
continue
|
||||
|
||||
try:
|
||||
from litellm.proxy._experimental.mcp_server.guardrail_translation import (
|
||||
from litellm.proxy.mcp_server.guardrail_translation import (
|
||||
guardrail_translation_mappings as mcp_guardrail_translation_mappings,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,18 +26,18 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
is_provider_native_tool_dict,
|
||||
truncate_tool_name,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import (
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import (
|
||||
keep_non_string_tool_result_texts,
|
||||
scannable_tool_result_request,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
|
||||
from litellm.llms.anthropic.pass_through.stream_assembly import (
|
||||
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
build_complete_streaming_response,
|
||||
build_usage_only_response_from_chunks,
|
||||
)
|
||||
|
|
@ -435,7 +435,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
|
||||
import uuid
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ def _has_non_chat_user_block(message: Mapping[str, object]) -> bool:
|
|||
|
||||
|
||||
def _gets_placeholder_name(tool: Mapping[str, object]) -> bool:
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import is_provider_native_tool_dict
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import is_provider_native_tool_dict
|
||||
|
||||
name: Final = tool.get("name")
|
||||
return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip()))
|
||||
|
|
|
|||
|
|
@ -15,23 +15,23 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.context_management import (
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
PolyfillResult,
|
||||
apply_context_management,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text
|
||||
from litellm.llms.anthropic.pass_through.safeguards import (
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text
|
||||
from litellm.llms.anthropic.messages.safeguards import (
|
||||
SafeguardsEvaluator,
|
||||
build_safeguards_evaluator,
|
||||
read_classifier_model_setting,
|
||||
requested_dangerous_tool_use,
|
||||
with_safeguard_results,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
ProxySubCallLoggingKwargs,
|
||||
is_reasoning_auto_summary_enabled,
|
||||
litellm_logging_obj_from_kwargs,
|
||||
|
|
@ -158,7 +158,7 @@ async def prepare_context_managed_request(
|
|||
drop_invalid_edits: bool = False,
|
||||
) -> PolyfillResult | None:
|
||||
"""Apply client compaction history, then optional context_management polyfill."""
|
||||
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
|
||||
from litellm.llms.anthropic.messages.context_management.editors.compact import (
|
||||
apply_client_compaction_block_history,
|
||||
)
|
||||
|
||||
|
|
@ -239,7 +239,7 @@ def _polyfill_will_run(
|
|||
if edits is None:
|
||||
return False
|
||||
|
||||
from litellm.llms.anthropic.pass_through.context_management.constants import (
|
||||
from litellm.llms.anthropic.messages.context_management.constants import (
|
||||
COMPACT_EDIT_TYPE,
|
||||
)
|
||||
|
||||
|
|
@ -265,7 +265,7 @@ def _spec_has_non_compact_edits(
|
|||
if edits is None:
|
||||
return False
|
||||
|
||||
from litellm.llms.anthropic.pass_through.context_management.constants import (
|
||||
from litellm.llms.anthropic.messages.context_management.constants import (
|
||||
COMPACT_EDIT_TYPE,
|
||||
)
|
||||
|
||||
|
|
@ -298,7 +298,7 @@ def _normalize_spec_edits(
|
|||
if context_management_explicitly_dropped(additional_drop_params):
|
||||
return None
|
||||
|
||||
from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
|
||||
from litellm.llms.anthropic.messages.context_management.dispatcher import (
|
||||
normalize_spec,
|
||||
)
|
||||
|
||||
|
|
@ -543,7 +543,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
Handles both string ("max") and dict ({"effort": "max", "summary": ...})
|
||||
formats. Uses model registry to check supports_xhigh/supports_minimal.
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
normalize_reasoning_effort_value,
|
||||
)
|
||||
|
||||
|
|
@ -806,6 +806,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
tool_name_mapping=tool_name_mapping,
|
||||
polyfill_result=polyfill_result,
|
||||
is_async=True,
|
||||
completion_request=completion_kwargs,
|
||||
litellm_logging_obj=logging_obj,
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
)
|
||||
|
|
@ -952,6 +953,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
tool_name_mapping=tool_name_mapping,
|
||||
polyfill_result=polyfill_result,
|
||||
is_async=False,
|
||||
completion_request=completion_kwargs,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
if transformed_stream is not None:
|
||||
|
|
@ -22,13 +22,14 @@ from typing import (
|
|||
from pydantic import TypeAdapter
|
||||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_params
|
||||
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
|
||||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
AppliedEdit,
|
||||
|
|
@ -41,19 +42,25 @@ from litellm.types.llms.anthropic import (
|
|||
UsageDelta,
|
||||
UsageIteration,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDeltaToolCall, Delta, Function
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
|
||||
_STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType))
|
||||
_SSE_EVENT: Final = TypeAdapter(Mapping[str, object])
|
||||
_THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock)
|
||||
_PROMPT_MESSAGES: Final = TypeAdapter(list[AllMessageValues])
|
||||
_PROMPT_TOOLS: Final = TypeAdapter(list[ChatCompletionToolParam])
|
||||
|
||||
|
||||
class _UsageDeltaWithIterations(UsageDelta, total=False):
|
||||
|
|
@ -234,7 +241,7 @@ class _CombinedChunkSplitter:
|
|||
@staticmethod
|
||||
def _is_combined(chunk: "ModelResponseStream") -> bool:
|
||||
"""True if ``chunk`` carries response content AND a finish_reason."""
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
|
||||
|
|
@ -548,6 +555,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
iterations_usage: list[UsageIteration] | None = None,
|
||||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
completion_request: Mapping[str, object] | None = None,
|
||||
):
|
||||
# Wrap the upstream stream so chunks that carry both content and a
|
||||
# finish_reason (fake-streamed providers) are split into two — see
|
||||
|
|
@ -556,6 +564,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
super().__init__(combined_chunk_splitter)
|
||||
self._combined_chunk_splitter: Final = combined_chunk_splitter
|
||||
self.model = model
|
||||
self.completion_request: Final = completion_request
|
||||
self._message_id: str = f"msg_{uuid.uuid4()}"
|
||||
if litellm_logging_obj is not None:
|
||||
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
|
||||
|
|
@ -752,25 +761,42 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
|
||||
def _create_initial_usage_delta(self) -> UsageDelta:
|
||||
"""
|
||||
Create the initial UsageDelta for the message_start event.
|
||||
Optionally estimate prompt tokens without consuming upstream chunks.
|
||||
|
||||
Initializes cache token fields (cache_creation_input_tokens, cache_read_input_tokens)
|
||||
to 0 to indicate to clients (like Claude Code) that prompt caching is supported.
|
||||
|
||||
The actual cache token values will be provided in the message_delta event at the
|
||||
end of the stream, since Bedrock Converse API only returns usage data in the final
|
||||
response chunk.
|
||||
|
||||
Returns:
|
||||
UsageDelta with all token counts initialized to 0.
|
||||
Output and cache counts start at zero. Provider usage in message_delta
|
||||
remains authoritative. Failed estimates fall back to zero.
|
||||
"""
|
||||
return UsageDelta(
|
||||
input_tokens=0,
|
||||
input_tokens=(
|
||||
self._estimate_initial_input_tokens()
|
||||
if litellm.anthropic_messages_estimate_initial_input_tokens is True
|
||||
else 0
|
||||
),
|
||||
output_tokens=0,
|
||||
cache_creation_input_tokens=0,
|
||||
cache_read_input_tokens=0,
|
||||
)
|
||||
|
||||
def _estimate_initial_input_tokens(self) -> int:
|
||||
try:
|
||||
request: Final = self.completion_request or {}
|
||||
messages: Final = _PROMPT_MESSAGES.validate_python(request.get("messages", self.messages))
|
||||
if not messages:
|
||||
return 0
|
||||
materialized_messages: Final = cast( # cast-ok: JSON-mode serialization preserves validated message types and materializes iterable content
|
||||
list[AllMessageValues], _PROMPT_MESSAGES.dump_python(messages, mode="json")
|
||||
)
|
||||
model: Final = request.get("model", self.model)
|
||||
tools: Final = request.get("tools")
|
||||
return litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is partially untyped
|
||||
model=model if isinstance(model, str) else self.model,
|
||||
messages=materialized_messages,
|
||||
tools=_PROMPT_TOOLS.validate_python(tools) if tools is not None else None,
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # prompt estimation must not interrupt the stream
|
||||
return 0
|
||||
|
||||
def __next__(self):
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
|
|
@ -1308,7 +1334,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
delta: Final = processed_chunk["delta"]
|
||||
if delta.get("stop_reason") == "max_tokens":
|
||||
return processed_chunk
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
refusal_stop_details,
|
||||
)
|
||||
|
||||
|
|
@ -1363,7 +1389,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
@staticmethod
|
||||
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
|
||||
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
|
||||
|
|
@ -1400,7 +1426,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
- Different content types in the response
|
||||
- Specific markers in the content
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
|
||||
|
|
@ -10,8 +10,8 @@ from pydantic import JsonValue, TypeAdapter
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
json_copy_or_original,
|
||||
prompt_cache_key_from_user_id,
|
||||
|
|
@ -139,14 +139,14 @@ from litellm.llms.anthropic.common_utils import (
|
|||
normalize_anthropic_tool_use_id,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.context_management import (
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
PolyfillResult,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import (
|
||||
from litellm.llms.anthropic.messages.native.mid_conversation_system import (
|
||||
convert_mid_conversation_system_turns,
|
||||
is_system_role_message,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
refusal_stop_details,
|
||||
)
|
||||
|
|
@ -210,7 +210,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.llms.anthropic import ContentBlockContentBlockDict
|
||||
|
||||
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]
|
||||
|
|
@ -322,6 +322,7 @@ class AnthropicAdapter:
|
|||
is_async: bool = True,
|
||||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
completion_request: Mapping[str, object] | None = None,
|
||||
) -> AsyncIterator[bytes] | Iterator[bytes] | None:
|
||||
"""
|
||||
Translate OpenAI streaming response to Anthropic format.
|
||||
|
|
@ -350,6 +351,7 @@ class AnthropicAdapter:
|
|||
iterations_usage=iterations_usage,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
completion_request=completion_request,
|
||||
)
|
||||
# Return the SSE-wrapped version for proper event formatting.
|
||||
if is_async:
|
||||
|
|
@ -18,7 +18,7 @@ from .editors import (
|
|||
from .result import PolyfillResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.anthropic.pass_through.utils import ProxySubCallLoggingKwargs
|
||||
from litellm.llms.anthropic.messages.utils import ProxySubCallLoggingKwargs
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
|
|
@ -21,12 +21,12 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.llms.anthropic.pass_through.caller_model_checks import (
|
||||
from litellm.llms.anthropic.messages.caller_model_checks import (
|
||||
caller_can_call_model,
|
||||
caller_within_model_budget,
|
||||
caller_within_model_rate_limit,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
ProxySubCallLoggingKwargs,
|
||||
proxy_general_settings,
|
||||
proxy_sub_call_metadata,
|
||||
|
|
@ -269,7 +269,7 @@ def _count_effective_tokens(
|
|||
threshold check matches the downstream ``input_tokens`` metric.
|
||||
"""
|
||||
# Local import to avoid pulling the adapter at module load time.
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
|
||||
|
|
@ -426,7 +426,7 @@ def _build_summary_messages(
|
|||
system prompt); the conversation history is translated to OpenAI shape;
|
||||
the summarization prompt is appended as a final user turn.
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
|
||||
|
|
@ -12,7 +12,7 @@ from typing_extensions import assert_never
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
|
||||
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
|
||||
from litellm.types.utils import TokenCountMessagesFormat
|
||||
|
||||
_CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType(
|
||||
|
|
@ -336,7 +336,7 @@ class AgenticAnthropicStreamingIterator:
|
|||
await task
|
||||
|
||||
async def aclose(self) -> None:
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
aclose_if_supported,
|
||||
)
|
||||
|
||||
|
|
@ -379,7 +379,7 @@ class AgenticAnthropicStreamingIterator:
|
|||
if hasattr(result, "__aiter__"):
|
||||
self._follow_up_iterator = result.__aiter__()
|
||||
elif isinstance(result, dict):
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
|
|
@ -12,7 +12,7 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta
|
||||
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -365,7 +365,7 @@ async def _try_websearch_short_circuit(
|
|||
),
|
||||
)
|
||||
if stream:
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
|
|
@ -659,7 +659,7 @@ def anthropic_messages_handler(
|
|||
# reference the provider cannot resolve. Popped from kwargs so it never reaches the provider.
|
||||
skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False)
|
||||
if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list
|
||||
from litellm.llms.anthropic.pass_through.messages.mcp_handler import (
|
||||
from litellm.llms.anthropic.messages.native.mcp_handler import (
|
||||
anthropic_messages_with_mcp,
|
||||
)
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
|
|||
custom_llm_provider: str | None,
|
||||
**kwargs,
|
||||
) -> AnthropicMessagesResponse | AsyncIterator:
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
|
|
@ -76,12 +76,12 @@ async def anthropic_messages_with_mcp(
|
|||
upstream provider as a url it cannot reach.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.experimental_mcp_client.tools import (
|
||||
from litellm.llms.anthropic.messages.adapters.handler import request_safeguards_evaluator
|
||||
from litellm.llms.anthropic.messages.native.handler import proxy_classifies_tool_uses
|
||||
from litellm.llms.anthropic.messages.safeguards import with_safeguard_results
|
||||
from litellm.mcp_client.tools import (
|
||||
transform_mcp_tool_to_anthropic_tool,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import request_safeguards_evaluator
|
||||
from litellm.llms.anthropic.pass_through.messages.handler import proxy_classifies_tool_uses
|
||||
from litellm.llms.anthropic.pass_through.safeguards import with_safeguard_results
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import (
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
)
|
||||
|
|
@ -198,7 +198,7 @@ async def anthropic_messages_with_mcp(
|
|||
else response
|
||||
)
|
||||
if stream:
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
|
|
@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
AnthropicMessagesStreamingResponse,
|
||||
BaseAnthropicMessagesStreamingIterator,
|
||||
aclose_if_supported,
|
||||
|
|
@ -18,13 +18,13 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
INCOMPLETE_STREAM_ERROR_MESSAGE,
|
||||
MESSAGE_DELTA_OPTIONAL_KEYS,
|
||||
anthropic_message_delta,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
|
||||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
|
|
@ -33,7 +33,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointT
|
|||
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
|
||||
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()
|
||||
|
||||
|
|
@ -10,8 +10,8 @@ from litellm.exceptions import AuthenticationError
|
|||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts
|
||||
from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
|
||||
from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -736,7 +736,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
) -> AsyncIterator[bytes]:
|
||||
"""Helper function to handle Anthropic streaming responses using the existing logging handlers"""
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
BaseAnthropicMessagesStreamingIterator,
|
||||
)
|
||||
|
||||
|
|
@ -11,7 +11,7 @@ from litellm.llms.anthropic.common_utils import (
|
|||
sanitize_tool_use_ids_in_anthropic_messages,
|
||||
strip_empty_content_blocks_from_anthropic_messages,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled
|
||||
from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicMessagesRequestOptionalParams,
|
||||
AnthropicStopDetails,
|
||||
|
|
@ -201,7 +201,7 @@ def _build_responses_kwargs(
|
|||
if isinstance(reasoning, dict):
|
||||
effort: Final[object] = reasoning.get("effort")
|
||||
if isinstance(effort, str):
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
normalize_reasoning_effort_value,
|
||||
)
|
||||
|
||||
|
|
@ -14,13 +14,13 @@ from litellm.exceptions import MidStreamFallbackError
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
encrypted_reasoning_signature,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
INCOMPLETE_STREAM_ERROR_MESSAGE,
|
||||
refusal_stop_details,
|
||||
responses_output_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
|
||||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.responses.streaming_iterator import (
|
||||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
|
|
@ -35,7 +35,7 @@ from .transformation import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
|
||||
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.llms.openai import ResponseIncompleteEvent
|
||||
|
||||
|
||||
|
|
@ -20,12 +20,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.litellm_core_utils.reasoning_effort_utils import (
|
||||
reasoning_effort_from_thinking,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_parts
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
refusal_stop_details,
|
||||
responses_output_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
json_copy_or_original,
|
||||
prompt_cache_key_from_user_id,
|
||||
|
|
@ -71,7 +71,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
if raw_usage is None:
|
||||
return AnthropicUsage(input_tokens=0, output_tokens=0)
|
||||
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
|
|
@ -11,8 +11,8 @@ from typing_extensions import NotRequired, ReadOnly, Unpack, assert_never
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.pass_through.caller_model_checks import CallerModelVerdict, caller_model_verdict
|
||||
from litellm.llms.anthropic.pass_through.utils import (
|
||||
from litellm.llms.anthropic.messages.caller_model_checks import CallerModelVerdict, caller_model_verdict
|
||||
from litellm.llms.anthropic.messages.utils import (
|
||||
ProxySubCallLoggingKwargs,
|
||||
proxy_general_settings,
|
||||
proxy_sub_call_metadata,
|
||||
|
|
@ -12,7 +12,7 @@ from litellm.llms.anthropic.chat.handler import (
|
|||
ModelResponseIterator as AnthropicModelResponseIterator,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS
|
||||
from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
Message,
|
||||
7
litellm/llms/anthropic/pass_through/__init__.py
Normal file
7
litellm/llms/anthropic/pass_through/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package
|
||||
|
||||
__getattr__ = alias_renamed_package(
|
||||
__name__,
|
||||
"litellm.llms.anthropic.messages",
|
||||
nested_renames=((f"{__name__}.messages", "litellm.llms.anthropic.messages.native"),),
|
||||
)
|
||||
|
|
@ -24,11 +24,11 @@ from litellm.llms.anthropic.common_utils import (
|
|||
)
|
||||
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
|
||||
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils, prepare_native_messages
|
||||
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
|
||||
from litellm.router_utils.baseline_request import (
|
||||
BASELINE_PARAMETERS,
|
||||
capture_baseline_parameters,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
|
|||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
|
|
|
|||
|
|
@ -1955,7 +1955,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
anthropic_beta_list: list,
|
||||
) -> None:
|
||||
"""Keep only compact_20260112 edits for Bedrock; add beta header or drop field."""
|
||||
from litellm.llms.anthropic.pass_through.context_management.constants import (
|
||||
from litellm.llms.anthropic.messages.context_management.constants import (
|
||||
COMPACT_EDIT_TYPE,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
|
@ -2859,6 +2859,8 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# Add "trace" from Bedrock guardrails - if user has opted in to returning it
|
||||
if "trace" in completion_response:
|
||||
setattr(model_response, "trace", completion_response["trace"])
|
||||
if logging_obj is not None:
|
||||
logging_obj.record_native_bedrock_guardrail(model_response)
|
||||
|
||||
# Add service_tier if present in Bedrock response
|
||||
# Map Bedrock serviceTier (object) to OpenAI service_tier (string)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.caching.caching import InMemoryCache
|
|||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
|
||||
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.llms.anthropic.chat.handler import (
|
|||
from litellm.llms.anthropic.chat.transformation import JsonModeRequest
|
||||
from litellm.llms.base_llm import base_model_iterator
|
||||
from litellm.llms.base_llm.base_utils import ResponseFormatSchema
|
||||
from litellm.llms.bedrock.common_utils import BEDROCK_INVOKE_TRACE_FIELD
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -962,8 +964,19 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
|
|||
response_format_schema=response_format_schema,
|
||||
)
|
||||
|
||||
def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
|
||||
return self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data)
|
||||
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
|
||||
parsed: Final = self.anthropic_model_response_iterator.chunk_parser(chunk=chunk_data)
|
||||
trace: Final = object_mapping(chunk_data.get(BEDROCK_INVOKE_TRACE_FIELD))
|
||||
if trace is None:
|
||||
return parsed
|
||||
return parsed.model_copy(
|
||||
update={
|
||||
"provider_specific_fields": {
|
||||
**(parsed.provider_specific_fields or {}),
|
||||
BEDROCK_INVOKE_TRACE_FIELD: trace,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def _stream_end_chunk(self) -> ModelResponseStream | None:
|
||||
return self.anthropic_model_response_iterator.cut_stream_chunk()
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
convert_to_anthropic_image_obj,
|
||||
)
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
|
|||
AmazonInvokeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
BEDROCK_INVOKE_TRACE_FIELD,
|
||||
apply_bedrock_invoke_structured_output,
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
|
|
@ -406,7 +408,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
return AnthropicConfig.transform_response(
|
||||
response: Final = AnthropicConfig.transform_response(
|
||||
self,
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
|
|
@ -420,3 +422,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
body: Final = object_mapping(raw_response.json()) or {}
|
||||
trace: Final = object_mapping(body.get(BEDROCK_INVOKE_TRACE_FIELD))
|
||||
if trace is not None:
|
||||
setattr(response, BEDROCK_INVOKE_TRACE_FIELD, trace)
|
||||
logging_obj.record_native_bedrock_guardrail(response)
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -45,6 +45,8 @@ if TYPE_CHECKING:
|
|||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace"
|
||||
|
||||
|
||||
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
|
||||
_OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.")
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
|
|||
chat_completion_tools_for,
|
||||
is_flat_responses_tool,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
|
||||
KIMI_K3_REPO: Final = "moonshotai/Kimi-K3"
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from litellm.llms.anthropic.common_utils import (
|
|||
carried_thinking_keys,
|
||||
validated_object_mapping,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
|
|
@ -33,6 +33,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
|
|||
from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
bedrock_invoke_guardrail_headers,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
BedrockError,
|
||||
|
|
@ -151,8 +152,14 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[dict, str | None]:
|
||||
raw_guardrail_config: Final = litellm_params.get("guardrailConfig") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # litellm_params is an untyped dict param
|
||||
guardrail_headers: Final = (
|
||||
()
|
||||
if raw_guardrail_config is None
|
||||
else tuple(bedrock_invoke_guardrail_headers(raw_guardrail_config).items()) # pyright: ignore[reportUnknownArgumentType] # read from the untyped dict param
|
||||
)
|
||||
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
|
||||
return merge_bedrock_invoke_headers(headers, (), metadata_headers, owned_names), api_base
|
||||
return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names), api_base # pyright: ignore[reportUnknownArgumentType] # headers is an untyped dict param
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
|
|
@ -878,7 +885,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
merge them from ``message_start`` so logging/cost sees a consistent usage
|
||||
object (fixes negative input costs: LIT-2411).
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
BaseAnthropicMessagesStreamingIterator,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ if TYPE_CHECKING:
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
|
|
@ -2650,7 +2650,7 @@ class BaseLLMHTTPHandler:
|
|||
|
||||
initial_response: AsyncIterator | AnthropicMessagesResponse
|
||||
if stream:
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
AnthropicMessagesStreamingResponse,
|
||||
anthropic_messages_stream_hidden_params,
|
||||
)
|
||||
|
|
@ -2674,7 +2674,7 @@ class BaseLLMHTTPHandler:
|
|||
hidden_params=stream_hidden_params,
|
||||
)
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import (
|
||||
AgenticAnthropicStreamingIterator,
|
||||
)
|
||||
|
||||
|
|
@ -2705,6 +2705,7 @@ class BaseLLMHTTPHandler:
|
|||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
logging_obj.record_native_bedrock_guardrail(initial_response)
|
||||
|
||||
return await self._finalize_anthropic_messages_response(
|
||||
initial_response=initial_response,
|
||||
|
|
@ -6237,7 +6238,7 @@ class BaseLLMHTTPHandler:
|
|||
from typing import cast
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from pydantic import TypeAdapter
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.databricks.common_utils import (
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final, cast # noqa: TID251 # narrows untyped request dicts
|
|||
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ class HostedVLLMDecisionsConfig(BaseDecisionsConfig):
|
|||
api_key_env = ("HOSTED_VLLM_API_KEY",)
|
||||
api_base_env = ("HOSTED_VLLM_API_BASE",)
|
||||
api_key_required = False
|
||||
supports_images = True
|
||||
inlines_remote_images = True
|
||||
health_check_questions = MappingProxyType(
|
||||
{
|
||||
"reachable": MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_text_error
|
||||
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
|
||||
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
|
||||
from litellm.llms.opencode.common_utils import go_api_base, go_headers
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ alongside its standard OpenAI-compatible chat completions endpoint.
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ def vertex_gemini_count_context(
|
|||
def unconvertible_messages_error(
|
||||
messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat
|
||||
) -> VertexGeminiCountContextError | None:
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import unconvertible_blocks_error
|
||||
|
||||
error_message: Final = unconvertible_blocks_error(messages, messages_format)
|
||||
return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
|
||||
|
|
@ -145,7 +145,7 @@ def malformed_tool_result_text_count_error(
|
|||
messages_format: TokenCountMessagesFormat,
|
||||
deployment: Mapping[str, object] | None,
|
||||
) -> VertexGeminiCountContextError | None:
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_count_error
|
||||
|
||||
error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment)
|
||||
return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
|
||||
|
|
@ -235,7 +235,7 @@ def _view_without_edits(
|
|||
system: object | None,
|
||||
typed_system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> InferenceRequestView:
|
||||
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
|
||||
from litellm.llms.anthropic.messages.context_management.editors.compact import (
|
||||
apply_client_compaction_block_history,
|
||||
)
|
||||
|
||||
|
|
@ -253,9 +253,9 @@ def _view_without_edits(
|
|||
async def _anthropic_inference_view(
|
||||
messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
|
||||
) -> InferenceRequestView | VertexGeminiCountContextError:
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import prepare_context_managed_request
|
||||
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
|
||||
from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request
|
||||
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
|
||||
|
||||
typed_system: Final = _typed_system(system)
|
||||
if isinstance(typed_system, VertexGeminiCountContextError):
|
||||
|
|
@ -308,7 +308,7 @@ def _chat_messages(
|
|||
model: str,
|
||||
custom_llm_provider: Literal["vertex_ai", "gemini"],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
match messages_format:
|
||||
case "anthropic":
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package
|
||||
|
||||
__getattr__ = alias_renamed_package(__name__, "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages")
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue