mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge remote-tracking branch 'origin/main' into HEAD
# Conflicts: # litellm/router.py
This commit is contained in:
commit
6680a8dbc4
294 changed files with 7412 additions and 2015 deletions
1
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
1
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -93,6 +93,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
|
|||
Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
|
||||
Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
|
||||
Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
|
||||
A reviewer must be able to rerun every case from the body alone, so the shared setup gives the proxy start command, the config.yaml, env var names (never values) and the client code or curl used, all exactly as run. Wrap any block longer than about 20 lines in a collapsed <details> with a one-line summary
|
||||
|
||||
### Before (<hash>)
|
||||
|
||||
|
|
|
|||
2
.github/workflows/image-scan.yml
vendored
2
.github/workflows/image-scan.yml
vendored
|
|
@ -53,7 +53,7 @@ jobs:
|
|||
if: >-
|
||||
github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
|
|
|
|||
|
|
@ -49,9 +49,9 @@ jobs:
|
|||
id: compliance
|
||||
run: |
|
||||
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
||||
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
--interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \
|
||||
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com
|
||||
|
||||
tracking-issue:
|
||||
name: tracking-issue
|
||||
|
|
|
|||
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
pull-requests: read
|
||||
actions: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
services:
|
||||
redis:
|
||||
image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
|
||||
|
|
|
|||
18
.github/workflows/test-unit.yml
vendored
18
.github/workflows/test-unit.yml
vendored
|
|
@ -45,7 +45,7 @@ jobs:
|
|||
id: rust-bridge-key
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }}
|
||||
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'packaging/litellm-proxy/pyproject.toml') }}
|
||||
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Python
|
||||
|
|
@ -98,6 +98,18 @@ jobs:
|
|||
mkdir -p rust-bridge-dist/litellm/rust_bridge
|
||||
cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
|
||||
|
||||
- name: Build the gateway wheel once for test shards
|
||||
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
|
||||
timeout-minutes: 15
|
||||
working-directory: packaging/litellm-proxy
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
SCCACHE_GHA_VERSION: litellm-ci-v1
|
||||
SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../rust-bridge-dist
|
||||
|
||||
- name: Upload Rust bridge artifact
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 10
|
||||
|
|
@ -1024,7 +1036,7 @@ jobs:
|
|||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
||||
|
|
@ -1486,7 +1498,7 @@ jobs:
|
|||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/de
|
|||
|
||||
Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
|
||||
|
||||
For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
|
||||
For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. The body alone must let a reviewer rerun it, with the proxy command, config, env var names and client code exactly as run, and long blocks collapsed in <details>. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
|
||||
|
||||
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
|
||||
- don't use emojis
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
|||
COPY pyproject.toml uv.lock ./
|
||||
COPY enterprise/pyproject.toml enterprise/
|
||||
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
|
||||
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
|
||||
|
||||
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
|
||||
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
||||
|
|
@ -101,7 +102,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
|
|||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Install project and workspace packages (fast - deps already cached)
|
||||
RUN uv sync --frozen --no-default-groups --no-editable \
|
||||
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
|
||||
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--group admin-mcp \
|
||||
|
|
@ -111,6 +113,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra bedrock-realtime \
|
||||
--python python3.13
|
||||
|
||||
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
|||
COPY pyproject.toml uv.lock ./
|
||||
COPY enterprise/pyproject.toml enterprise/
|
||||
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
|
||||
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
|
||||
|
||||
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
|
||||
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
||||
|
|
@ -99,7 +100,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
|
|||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Install project and workspace packages (fast - deps already cached)
|
||||
RUN uv sync --frozen --no-default-groups --no-editable \
|
||||
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
|
||||
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--group admin-mcp \
|
||||
|
|
@ -109,6 +111,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra bedrock-realtime \
|
||||
--python python3.13
|
||||
|
||||
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
|
|
@ -41,10 +41,14 @@ COPY pyproject.toml README.md rust-toolchain.toml ./
|
|||
COPY .cargo ./.cargo
|
||||
COPY litellm-rust ./litellm-rust
|
||||
COPY model_prices_and_context_window.json ./
|
||||
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
mkdir -p litellm/rust_bridge /wheels && \
|
||||
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
|
||||
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
|
||||
python -c "import maturin; maturin.build_wheel('/wheels')" && \
|
||||
cd packaging/litellm-proxy && \
|
||||
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
|
||||
python -c "import maturin; maturin.build_wheel('/wheels')"
|
||||
|
||||
|
|
@ -85,6 +89,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
|
||||
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
|
||||
--mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \
|
||||
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
|
|
@ -98,7 +103,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
COPY . .
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-install-project --no-default-groups --no-editable \
|
||||
uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--group admin-mcp \
|
||||
|
|
@ -108,7 +113,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--python python3.13
|
||||
|
||||
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
|
||||
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
|
||||
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl
|
||||
|
||||
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
|
||||
# compiled Rust extension must live next to the source or it is never imported.
|
||||
|
|
|
|||
|
|
@ -4,13 +4,7 @@
|
|||
//! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and
|
||||
//! `run_async_fallback` in `router_utils/fallback_event_handlers.py`.
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use std::{borrow::Cow, future::Future, pin::Pin, sync::Arc, time::Duration};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
@ -1231,7 +1225,7 @@ impl Core {
|
|||
.unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at)
|
||||
.await;
|
||||
}
|
||||
let pick_started = Instant::now();
|
||||
let pick_started = self.health.now_time();
|
||||
let (deployment, candidates) = match self
|
||||
.picked(snapshot, op, request, target, inputs, retry, at)
|
||||
.await
|
||||
|
|
@ -1248,7 +1242,11 @@ impl Core {
|
|||
self.host.observe(&RouterEvent::Selected {
|
||||
model_group: candidates.model_group.clone(),
|
||||
deployment_id: deployment.id.clone(),
|
||||
pick: pick_started.elapsed(),
|
||||
pick: self
|
||||
.health
|
||||
.now_time()
|
||||
.duration_since(pick_started)
|
||||
.unwrap_or_default(),
|
||||
});
|
||||
if !self.host.supports(op, deployment.provider()) {
|
||||
return Err(Error::Config(ConfigError::Unsupported(Unsupported::Op {
|
||||
|
|
@ -1274,7 +1272,7 @@ impl Core {
|
|||
},
|
||||
None,
|
||||
snapshot.settings.dynamic.default_ref(),
|
||||
jiff::Timestamp::now(),
|
||||
self.health.timestamp(),
|
||||
),
|
||||
Ok(None)
|
||||
);
|
||||
|
|
|
|||
|
|
@ -247,6 +247,10 @@ impl Health {
|
|||
self.clock.now()
|
||||
}
|
||||
|
||||
pub(crate) fn timestamp(&self) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(self.clock.now()).unwrap_or(jiff::Timestamp::UNIX_EPOCH)
|
||||
}
|
||||
|
||||
fn now(&self) -> f64 {
|
||||
unix_seconds(self.clock.now())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -433,6 +433,10 @@ impl<'a> Lookup<'a> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
pub fn has_team_wildcards(&self, team_id: &str) -> bool {
|
||||
self.host.team_patterns.iter().any(|(t, _)| t == team_id)
|
||||
}
|
||||
|
||||
/// The deployments `team_id`'s own wildcard pattern routes `name` to
|
||||
/// (`_core_team_pattern_routers`, keyed by `team_public_model_name`).
|
||||
pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec<Arc<Deployment>> {
|
||||
|
|
|
|||
|
|
@ -223,14 +223,20 @@ pub(crate) fn wildcard_name<'a>(
|
|||
registry: &'a Registry,
|
||||
name: &'a str,
|
||||
specific_deployment: bool,
|
||||
team_id: Option<&str>,
|
||||
) -> Option<&'a str> {
|
||||
let model = registry.alias_of(name).unwrap_or(name);
|
||||
(registry.has_wildcards()
|
||||
let lookup = crate::lookup::Lookup::new(registry);
|
||||
let team_patterns = team_id.is_some_and(|team| lookup.has_team_wildcards(team));
|
||||
let team_misses = || team_id.is_none_or(|team| lookup.team_wildcard(model, team).is_empty());
|
||||
let global_misses = || registry.has_wildcards() && registry.wildcard_for(model).is_none();
|
||||
((team_patterns || registry.has_wildcards())
|
||||
&& !specific_deployment
|
||||
&& !registry.has_name(model)
|
||||
&& (registry.has_name(name) || registry.deployment(name).is_none())
|
||||
&& registry.routing_group_named(model).is_none()
|
||||
&& registry.wildcard_for(model).is_none())
|
||||
&& team_misses()
|
||||
&& (team_patterns || global_misses()))
|
||||
.then_some(model)
|
||||
}
|
||||
|
||||
|
|
@ -339,15 +345,21 @@ pub fn resolve_request(
|
|||
let kept = reserved(model, deployments, caller, now)?;
|
||||
return resolved(ResolveKind::Group, model, kept);
|
||||
}
|
||||
// `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one.
|
||||
let team_owned = team_id
|
||||
.map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team))
|
||||
.filter(|deployments| !deployments.is_empty());
|
||||
if let Some(deployments) = team_owned {
|
||||
// `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one,
|
||||
// each tried as spelled and then under the name's provider (`get_pattern`).
|
||||
let team_owned = team_id.and_then(|team| {
|
||||
let lookup = crate::lookup::Lookup::new(registry);
|
||||
[Some(model), caller.wildcard_request]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|request| (request, lookup.team_wildcard(request, team)))
|
||||
.find(|(_, deployments)| !deployments.is_empty())
|
||||
});
|
||||
if let Some((request, deployments)) = team_owned {
|
||||
let routed = deployments.iter().flat_map(|deployment| {
|
||||
let public = deployment.model_info.team_public_model_name.as_deref();
|
||||
let pattern = public.unwrap_or(model);
|
||||
routed_wildcard(pattern, model, std::slice::from_ref(deployment))
|
||||
let pattern = public.unwrap_or(request);
|
||||
routed_wildcard(pattern, request, std::slice::from_ref(deployment))
|
||||
});
|
||||
let kept = reserved(model, routed.collect(), caller, now)?;
|
||||
return resolved(ResolveKind::Wildcard, model, kept);
|
||||
|
|
@ -428,6 +440,7 @@ pub fn resolve_request(
|
|||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
|
|
@ -544,8 +557,11 @@ mod tests {
|
|||
#[test]
|
||||
fn a_name_no_pattern_matches_is_tried_under_its_provider() {
|
||||
let registry = registry(wildcard_group());
|
||||
assert_eq!(wildcard_name(®istry, "gpt-4o", false), Some("gpt-4o"));
|
||||
assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false), None);
|
||||
assert_eq!(
|
||||
wildcard_name(®istry, "gpt-4o", false, None),
|
||||
Some("gpt-4o")
|
||||
);
|
||||
assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false, None), None);
|
||||
let caller = Caller {
|
||||
wildcard_request: Some("openai/gpt-4o"),
|
||||
..Caller::default()
|
||||
|
|
@ -564,6 +580,48 @@ mod tests {
|
|||
assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_bare_name_is_tried_under_its_provider_against_the_teams_own_wildcard() {
|
||||
let registry = registry(json!([
|
||||
{"model_name": "model_name_tw_1", "litellm_params": {"model": "openai/*"},
|
||||
"model_info": {"id": "tw", "team_id": "TW", "team_public_model_name": "openai/*"}},
|
||||
{"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "g"}},
|
||||
]));
|
||||
// No proxy-wide wildcard: only the team's patterns call for the provider spelling.
|
||||
assert_eq!(
|
||||
wildcard_name(®istry, "gpt-4o-mini", false, Some("TW")),
|
||||
Some("gpt-4o-mini")
|
||||
);
|
||||
assert_eq!(wildcard_name(®istry, "gpt-4o-mini", false, None), None);
|
||||
assert_eq!(
|
||||
wildcard_name(®istry, "openai/gpt-4o-mini", false, Some("TW")),
|
||||
None
|
||||
);
|
||||
let resolve = |team_id| {
|
||||
let caller = Caller {
|
||||
team_id,
|
||||
wildcard_request: Some("openai/gpt-4o-mini"),
|
||||
..Caller::default()
|
||||
};
|
||||
resolve_request(
|
||||
®istry,
|
||||
"gpt-4o-mini",
|
||||
false,
|
||||
caller,
|
||||
None,
|
||||
None,
|
||||
Timestamp::now(),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
let resolution = resolve(Some("TW")).unwrap();
|
||||
assert_eq!(resolution.kind, ResolveKind::Wildcard);
|
||||
assert_eq!(resolution.group, "gpt-4o-mini");
|
||||
assert_eq!(ids(Some(resolution.clone())), ["tw"]);
|
||||
assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o-mini");
|
||||
assert!(resolve(Some("OTHER")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() {
|
||||
let registry = registry(json!([
|
||||
|
|
|
|||
|
|
@ -281,8 +281,12 @@ impl Core {
|
|||
// No deployment for the name: route to the first default fallback.
|
||||
let default_target = crate::resolve::default_target(&snapshot.settings.dynamic);
|
||||
// `get_pattern`: a name no pattern matches is tried again under its inferred provider.
|
||||
let wildcard_name =
|
||||
crate::resolve::wildcard_name(registry, name, request.specific_deployment);
|
||||
let wildcard_name = crate::resolve::wildcard_name(
|
||||
registry,
|
||||
name,
|
||||
request.specific_deployment,
|
||||
request.team_id.as_deref(),
|
||||
);
|
||||
let wildcard_request = match wildcard_name {
|
||||
Some(model) => {
|
||||
let scope = StrategyScope::Default;
|
||||
|
|
@ -308,7 +312,7 @@ impl Core {
|
|||
caller,
|
||||
default_target,
|
||||
snapshot.settings.dynamic.default_ref(),
|
||||
jiff::Timestamp::now(),
|
||||
self.health.timestamp(),
|
||||
)?
|
||||
.ok_or_else(|| Error::UnknownModel {
|
||||
// The name `_common_checks_available_deployment` looked for, past the alias.
|
||||
|
|
|
|||
|
|
@ -1755,3 +1755,57 @@ async fn a_listed_pass_through_pick_names_the_group_an_alias_resolves_to(config:
|
|||
Error::NoPassThroughDeployments { model } if model == "primary"
|
||||
));
|
||||
}
|
||||
|
||||
struct StoppedClock(u64);
|
||||
|
||||
impl litellm_router_core::health::Clock for StoppedClock {
|
||||
fn now(&self) -> std::time::SystemTime {
|
||||
std::time::UNIX_EPOCH + Duration::from_secs(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn utc_hour(hour: u64) -> u64 {
|
||||
1_767_225_600 + hour * 3_600
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::inside_the_window(utc_hour(10), vec!["open"])]
|
||||
#[case::outside_the_window(utc_hour(20), vec!["open", "reserved"])]
|
||||
#[tokio::test]
|
||||
async fn access_windows_follow_the_routers_clock(#[case] now: u64, #[case] expected: Vec<&str>) {
|
||||
let config: RouterConfig = serde_json::from_value(json!({
|
||||
"model_list": [
|
||||
{"model_name": "chat", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "open"}},
|
||||
{"model_name": "chat", "litellm_params": {"model": "openai/b"}, "model_info": {
|
||||
"id": "reserved",
|
||||
"access_windows": [{"start": "09:00", "end": "17:00", "timezone": "UTC", "team_ids": ["batch"]}],
|
||||
}},
|
||||
]
|
||||
}))
|
||||
.unwrap();
|
||||
let router = Router::builder(
|
||||
config,
|
||||
RouterDynamicConfig::default(),
|
||||
Arc::new(ScriptedHost::default()),
|
||||
)
|
||||
.randomness(Arc::new(SeededRandomness::new(7)))
|
||||
.clock(Arc::new(StoppedClock(now)))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut picked = std::collections::BTreeSet::new();
|
||||
for _ in 0..40 {
|
||||
let deployment = router
|
||||
.select(&Op::from("acompletion"), &request_for("chat"))
|
||||
.await
|
||||
.unwrap();
|
||||
picked.insert(deployment.id.clone());
|
||||
}
|
||||
assert_eq!(picked.into_iter().collect::<Vec<_>>(), expected);
|
||||
}
|
||||
|
||||
fn request_for(model: &str) -> RouterRequest {
|
||||
RouterRequest {
|
||||
model: model.to_owned(),
|
||||
..RouterRequest::default()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -898,6 +898,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
def redacts_messages_itself(self) -> bool:
|
||||
return False
|
||||
|
||||
def exempt_from_message_redaction(self) -> bool:
|
||||
"""Router pre-call checks that key routing off the request and never log it return True, so
|
||||
`turn_off_message_logging` leaves the success event they receive unredacted."""
|
||||
return False
|
||||
|
||||
def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict:
|
||||
"""
|
||||
Redacts or excludes fields from StandardLoggingPayload before callbacks receive it.
|
||||
|
|
|
|||
|
|
@ -71,12 +71,12 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
_TERMINAL_STATUSES: Final = frozenset(
|
||||
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
|
||||
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action", "requires_continuation"}
|
||||
)
|
||||
|
||||
_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
|
||||
|
||||
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
|
||||
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action", "requires_continuation"})
|
||||
|
||||
SettlementOutcome: TypeAlias = Literal["billed", "released", "unsettled"]
|
||||
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ async def acreate(
|
|||
response_mime_type: str | None = None,
|
||||
# Continuation
|
||||
previous_interaction_id: str | None = None,
|
||||
continuation_token: str | None = None,
|
||||
# Extra params
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
|
|
@ -129,6 +130,7 @@ async def acreate(
|
|||
response_format: JSON schema for response format
|
||||
response_mime_type: MIME type of the response
|
||||
previous_interaction_id: ID of previous interaction for continuation
|
||||
continuation_token: Token from a `requires_continuation` interaction to resume its long decode
|
||||
extra_headers: Additional headers
|
||||
extra_body: Additional body parameters
|
||||
timeout: Request timeout
|
||||
|
|
@ -164,6 +166,7 @@ async def acreate(
|
|||
response_format=response_format,
|
||||
response_mime_type=response_mime_type,
|
||||
previous_interaction_id=previous_interaction_id,
|
||||
continuation_token=continuation_token,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
|
|
@ -224,6 +227,7 @@ def create(
|
|||
response_mime_type: str | None = None,
|
||||
# Continuation
|
||||
previous_interaction_id: str | None = None,
|
||||
continuation_token: str | None = None,
|
||||
# Extra params
|
||||
extra_headers: dict[str, str] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
|
|
@ -262,6 +266,7 @@ def create(
|
|||
response_format: JSON schema for response format
|
||||
response_mime_type: MIME type of the response
|
||||
previous_interaction_id: ID of previous interaction for continuation
|
||||
continuation_token: Token from a `requires_continuation` interaction to resume its long decode
|
||||
extra_headers: Additional headers
|
||||
extra_body: Additional body parameters
|
||||
timeout: Request timeout
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ INTERACTIONS_API_OPTIONAL_PARAMS: Final = {
|
|||
"response_format",
|
||||
"response_mime_type",
|
||||
"previous_interaction_id",
|
||||
"continuation_token",
|
||||
"agent_config",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4405,7 +4405,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if isinstance(callback, CustomLogger): # custom logger class
|
||||
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
|
||||
|
||||
model_call_details: dict = self.model_call_details
|
||||
model_call_details: dict = (
|
||||
self._model_call_details_with_request_messages()
|
||||
if callback.exempt_from_message_redaction()
|
||||
else self.model_call_details
|
||||
)
|
||||
##################################
|
||||
# call redaction hook for custom logger
|
||||
model_call_details = callback.redact_standard_logging_payload_from_model_call_details(
|
||||
|
|
@ -4969,6 +4973,21 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
) -> None:
|
||||
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
|
||||
|
||||
def _model_call_details_with_request_messages(self) -> dict[str, object]: # mutable-ok: loggers take a dict
|
||||
"""The success-event payload with the request messages put back where `turn_off_message_logging`
|
||||
redacted them in place, for a callback that routes on the request and never logs it."""
|
||||
request_messages: Final = cast(object, self.messages) # cast-ok: the request messages are only put back as-is
|
||||
standard_logging_object: Final = self.model_call_details.get("standard_logging_object")
|
||||
return {
|
||||
**self.model_call_details,
|
||||
"messages": request_messages,
|
||||
**(
|
||||
{"standard_logging_object": {**standard_logging_object, "messages": request_messages}}
|
||||
if isinstance(standard_logging_object, dict)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
def _get_completed_success_callbacks(
|
||||
self, *, asynchronous: bool
|
||||
) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
|
||||
|
|
|
|||
|
|
@ -5879,7 +5879,7 @@ def _bedrock_tools_pt(
|
|||
description = name
|
||||
|
||||
defs = parameters.pop("$defs", {})
|
||||
defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {}
|
||||
defs_copy = defs if isinstance(defs, dict) else {}
|
||||
# Expand $ref references in parameters using the definitions
|
||||
# Note: We don't pre-flatten defs as that causes exponential memory growth
|
||||
# with circular references (see issue #19098). unpack_defs handles nested
|
||||
|
|
|
|||
|
|
@ -827,11 +827,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
return cls._raise_invalid_reasoning_effort(model, value, llm_provider)
|
||||
|
||||
def get_supported_openai_params(self, model: str):
|
||||
sampling_params: Final = ("temperature", "top_p") if self._supports_sampling_params(model) else ()
|
||||
params: Final = [
|
||||
"stream",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
*sampling_params,
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"tools",
|
||||
|
|
|
|||
|
|
@ -126,6 +126,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
"response_format",
|
||||
"response_mime_type",
|
||||
"previous_interaction_id",
|
||||
"continuation_token",
|
||||
]
|
||||
|
||||
def validate_environment(
|
||||
|
|
@ -206,6 +207,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
"environment",
|
||||
"response_modalities",
|
||||
"previous_interaction_id",
|
||||
"continuation_token",
|
||||
]
|
||||
dropped_keys: Final = (
|
||||
frozenset({"agent_config"})
|
||||
|
|
@ -301,7 +303,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
|
|||
) -> InteractionsAPIStreamingResponse:
|
||||
"""Parse streaming chunk."""
|
||||
verbose_logger.debug("Google AI Interactions streaming chunk: %s", parsed_chunk)
|
||||
return InteractionsAPIStreamingResponse(**parsed_chunk)
|
||||
return InteractionsAPIStreamingResponse.model_validate(parsed_chunk)
|
||||
|
||||
# GET / DELETE / CANCEL - just build URLs, responses match spec directly
|
||||
|
||||
|
|
|
|||
|
|
@ -4046,6 +4046,8 @@
|
|||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"deprecation_date": "2027-09-01",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -4068,7 +4070,7 @@
|
|||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement"
|
||||
},
|
||||
"azure_ai/claude-opus-5": {
|
||||
"deprecation_date": "2027-07-08",
|
||||
|
|
@ -13458,13 +13460,16 @@
|
|||
"supports_reasoning": true
|
||||
},
|
||||
"azure_ai/mistral-document-ai-2512": {
|
||||
"deprecation_date": "2027-02-15",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"litellm_provider": "azure_ai",
|
||||
"ocr_cost_per_page": 0.003,
|
||||
"mode": "ocr",
|
||||
"supported_endpoints": [
|
||||
"/v1/ocr"
|
||||
],
|
||||
"source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"successor_model": "azure_ai/mistral-ocr-4-0"
|
||||
},
|
||||
"azure_ai/mistral-ocr-4-0": {
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -40893,6 +40898,22 @@
|
|||
"output_cost_per_token": 4e-07,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"jina-reranker-v3.5": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "jina_ai",
|
||||
"max_input_tokens": 131072,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://api.jina.ai/v1/models"
|
||||
},
|
||||
"jina-reranker-m0": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "jina_ai",
|
||||
"max_input_tokens": 10240,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://api.jina.ai/v1/models"
|
||||
},
|
||||
"jina-reranker-v2-base-multilingual": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "jina_ai",
|
||||
|
|
@ -43771,7 +43792,8 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"mistral/zai-glm-5-3": {
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
|
|
@ -45270,6 +45292,18 @@
|
|||
"source": "https://tokenfactory.nebius.com/api/public/models_info",
|
||||
"cache_read_input_token_cost": 1.5e-07
|
||||
},
|
||||
"nebius/moonshotai/Kimi-K3.1-Pro": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"litellm_provider": "nebius",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://tokenfactory.nebius.com/api/public/models_info"
|
||||
},
|
||||
"nebius/moonshotai/Kimi-K3": {
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1024000,
|
||||
|
|
@ -71189,6 +71223,8 @@
|
|||
},
|
||||
"gpt-5.5-cyber-preview": {
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"deprecation_date": "2027-02-01",
|
||||
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
|
||||
"input_cost_per_token": 1.25e-05,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
|
|
@ -72148,6 +72184,7 @@
|
|||
"output_cost_per_token": 4.5e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"tpm": 250000,
|
||||
"rpm": 10,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
|
|
@ -72170,6 +72207,7 @@
|
|||
"output_cost_per_token": 4.5e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"tpm": 250000,
|
||||
"rpm": 10,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
|
|
@ -76469,6 +76507,33 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"us-gov.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 6.6e-07,
|
||||
"cache_read_input_token_cost_flex": 3.3e-07,
|
||||
"cache_read_input_token_cost_priority": 1.155e-06,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"input_cost_per_token_batches": 1.32e-06,
|
||||
"input_cost_per_token_flex": 1.32e-06,
|
||||
"input_cost_per_token_priority": 4.62e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 524288,
|
||||
"max_output_tokens": 524288,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.92e-06,
|
||||
"output_cost_per_token_batches": 3.96e-06,
|
||||
"output_cost_per_token_flex": 3.96e-06,
|
||||
"output_cost_per_token_priority": 1.386e-05,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": {
|
||||
"cache_read_input_token_cost": 1.4e-08,
|
||||
"input_cost_per_token": 6e-07,
|
||||
|
|
@ -78437,7 +78502,7 @@
|
|||
"cache_creation_input_token_cost": 1e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 1.6e-05,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-10-01",
|
||||
"deprecation_date": "2027-01-15",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"input_cost_per_token": 8e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -78447,7 +78512,7 @@
|
|||
"output_cost_per_token": 4e-05,
|
||||
"prompt_cache_min_tokens": 512.0,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.02,"search_context_size_low":0.02,"search_context_size_medium":0.02},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models",
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
|
|
@ -82782,6 +82847,39 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"gpt-6-cyber": {
|
||||
"cache_creation_input_token_cost": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-05,
|
||||
"cache_read_input_token_cost": 2e-06,
|
||||
"input_cost_per_token": 2e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0001,
|
||||
"source": "https://developers.openai.com/api/docs/models/gpt-6-cyber",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_search": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gpt-6-astra-law": {
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
"input_cost_per_token": 1e-05,
|
||||
|
|
@ -83380,6 +83478,28 @@
|
|||
"supports_response_schema": true,
|
||||
"source": "https://empiriolabs.ai/models/kimi-k3"
|
||||
},
|
||||
"empiriolabs/kimi-k3-1-pro": {
|
||||
"litellm_provider": "empiriolabs",
|
||||
"mode": "chat",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
"cache_read_input_token_cost": 1.8e-06,
|
||||
"supports_system_messages": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_web_search": true,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.015,
|
||||
"search_context_size_medium": 0.015,
|
||||
"search_context_size_high": 0.015
|
||||
},
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_function_calling": true,
|
||||
"source": "https://empiriolabs.ai/models/kimi-k3-1-pro"
|
||||
},
|
||||
"empiriolabs/mimo-v2-6-flash": {
|
||||
"litellm_provider": "empiriolabs",
|
||||
"mode": "chat",
|
||||
|
|
@ -86383,5 +86503,32 @@
|
|||
"supports_system_messages": false,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"databricks/databricks-kimi-k3-1-pro": {
|
||||
"cache_creation_input_token_cost": 1.99997e-06,
|
||||
"cache_read_input_token_cost": 1.9999e-07,
|
||||
"input_cost_per_token": 1.99997e-06,
|
||||
"input_dbu_cost_per_token": 2.8571e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"metadata": {"notes":"Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.99999e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 100.0
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,14 @@ class PluginArchiveMetadata:
|
|||
description: str | None = None
|
||||
has_agent_skill_metadata: bool = False
|
||||
updated_at: datetime | None = None
|
||||
created_at: datetime | None = None
|
||||
created_by: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PluginReview:
|
||||
approved_by: str
|
||||
security_scan_passed: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -51,6 +59,7 @@ class PluginPublication:
|
|||
manifest: Mapping[str, object]
|
||||
created_by: str | None
|
||||
content: bytes | None = None
|
||||
review: PluginReview | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -66,6 +75,10 @@ class PublishedPlugin:
|
|||
version: str
|
||||
enabled: bool
|
||||
updated_at: datetime
|
||||
created_at: datetime | None = None
|
||||
created_by: str | None = None
|
||||
review: PluginReview | None = None
|
||||
published_by: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -7683,6 +7683,17 @@
|
|||
"PluginListItem": {
|
||||
"description": "Plugin item in list responses.",
|
||||
"properties": {
|
||||
"approved_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Approved By"
|
||||
},
|
||||
"author": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -7715,6 +7726,17 @@
|
|||
],
|
||||
"title": "Created At"
|
||||
},
|
||||
"created_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created By"
|
||||
},
|
||||
"description": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -7737,6 +7759,17 @@
|
|||
],
|
||||
"title": "Domain"
|
||||
},
|
||||
"download_count": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Download Count"
|
||||
},
|
||||
"enabled": {
|
||||
"title": "Enabled",
|
||||
"type": "boolean"
|
||||
|
|
@ -7801,6 +7834,22 @@
|
|||
],
|
||||
"title": "Namespace"
|
||||
},
|
||||
"published_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published By"
|
||||
},
|
||||
"security_scan_passed": {
|
||||
"default": false,
|
||||
"title": "Security Scan Passed",
|
||||
"type": "boolean"
|
||||
},
|
||||
"source": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
|
|
@ -59415,6 +59464,14 @@
|
|||
},
|
||||
"title": "Skills",
|
||||
"type": "array"
|
||||
},
|
||||
"tags": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Tags",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
@ -59690,6 +59747,17 @@
|
|||
},
|
||||
"SkillHubSkill": {
|
||||
"properties": {
|
||||
"approved_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Approved By"
|
||||
},
|
||||
"content_url": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -59701,6 +59769,29 @@
|
|||
],
|
||||
"title": "Content Url"
|
||||
},
|
||||
"created_at": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created At"
|
||||
},
|
||||
"created_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Created By"
|
||||
},
|
||||
"description": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -59732,7 +59823,7 @@
|
|||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Gateway ZIP downloads, included in proxy-admin catalog listings",
|
||||
"description": "Gateway ZIP downloads, including repeat downloads",
|
||||
"title": "Download Count"
|
||||
},
|
||||
"download_url": {
|
||||
|
|
@ -59779,6 +59870,22 @@
|
|||
],
|
||||
"title": "Namespace"
|
||||
},
|
||||
"published_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Published By"
|
||||
},
|
||||
"security_scan_passed": {
|
||||
"default": false,
|
||||
"title": "Security Scan Passed",
|
||||
"type": "boolean"
|
||||
},
|
||||
"source": {
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
|
|
@ -60434,6 +60541,66 @@
|
|||
"title": "Limit",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "tag",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": "",
|
||||
"title": "Tag",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "security_scan_passed",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Security Scan Passed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "sort_by",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": "name",
|
||||
"enum": [
|
||||
"name",
|
||||
"download_count",
|
||||
"updated_at",
|
||||
"created_at",
|
||||
"created_by",
|
||||
"published_by",
|
||||
"approved_by",
|
||||
"security_scan_passed"
|
||||
],
|
||||
"title": "Sort By",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"in": "query",
|
||||
"name": "sort_order",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"default": "asc",
|
||||
"enum": [
|
||||
"asc",
|
||||
"desc"
|
||||
],
|
||||
"title": "Sort Order",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ from litellm.proxy.common_utils.claude_code_marketplace_auth import (
|
|||
)
|
||||
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
|
||||
from litellm.proxy.deps.repositories import PluginUploadRepositoryDep
|
||||
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.types.proxy.auth.special_headers import SpecialHeaders
|
||||
from litellm.types.proxy.claude_code_endpoints import (
|
||||
|
|
@ -666,6 +667,7 @@ async def register_plugin(
|
|||
response_model=ListPluginsResponse,
|
||||
)
|
||||
async def list_plugins(
|
||||
uploads: PluginUploadRepositoryDep,
|
||||
enabled_only: bool = False,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> ListPluginsResponse:
|
||||
|
|
@ -678,6 +680,17 @@ async def list_plugins(
|
|||
Returns:
|
||||
List of plugins with their metadata.
|
||||
"""
|
||||
catalog: Final = await list_visible_plugins(enabled_only, user_api_key_dict)
|
||||
if uploads is None:
|
||||
raise _error_response(500, CommonProxyErrors.db_not_connected_error.value)
|
||||
counts: Final = await uploads.download_counts(tuple(plugin.id for plugin in catalog.plugins))
|
||||
return ListPluginsResponse(
|
||||
plugins=[plugin.model_copy(update={"download_count": counts.get(plugin.id, 0)}) for plugin in catalog.plugins],
|
||||
count=catalog.count,
|
||||
)
|
||||
|
||||
|
||||
async def list_visible_plugins(enabled_only: bool, user_api_key_dict: UserAPIKeyAuth) -> ListPluginsResponse:
|
||||
try:
|
||||
prisma_client: Final = await get_prisma_client()
|
||||
|
||||
|
|
@ -693,9 +706,14 @@ async def list_plugins(
|
|||
# Parse manifest to get additional fields
|
||||
manifest = json.loads(p.manifest_json) if p.manifest_json else {}
|
||||
|
||||
review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest))
|
||||
plugin_list.append(
|
||||
PluginListItem(
|
||||
id=p.id,
|
||||
created_by=p.created_by,
|
||||
approved_by=review.approved_by if review else None,
|
||||
published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)),
|
||||
security_scan_passed=review.security_scan_passed if review else False,
|
||||
name=p.name,
|
||||
version=p.version,
|
||||
description=p.description,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import hashlib
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Final
|
||||
from itertools import chain
|
||||
from typing import Annotated, Final, Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
|
@ -33,7 +34,7 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp
|
|||
enable_plugin,
|
||||
get_plugin_downloads,
|
||||
header_key_caller,
|
||||
list_plugins,
|
||||
list_visible_plugins,
|
||||
plugin_archive_source_matches,
|
||||
plugin_archive_url,
|
||||
plugin_downloader,
|
||||
|
|
@ -65,6 +66,7 @@ from litellm.proxy.discovery_endpoints.skill_import import (
|
|||
parse_skill_source,
|
||||
select_imported_skill,
|
||||
)
|
||||
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
|
||||
from litellm.types.proxy.claude_code_endpoints import PluginListItem
|
||||
from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import AgentSkillsIndexEntry
|
||||
from litellm.types.proxy.skill_hub import (
|
||||
|
|
@ -78,6 +80,7 @@ from litellm.types.proxy.skill_hub import (
|
|||
SkillHubSkill,
|
||||
SkillHubVersion,
|
||||
SkillHubVersions,
|
||||
SkillSortField,
|
||||
)
|
||||
|
||||
router: Final = APIRouter(prefix="/skill-hub/skills", tags=["Skills Hub"])
|
||||
|
|
@ -121,6 +124,11 @@ def _skill_response(request: Request, plugin: PluginListItem, download_count: in
|
|||
namespace=plugin.namespace,
|
||||
keywords=None if plugin.keywords is None else tuple(plugin.keywords),
|
||||
updated_at=None if plugin.updated_at is None else datetime.fromisoformat(plugin.updated_at),
|
||||
created_at=None if plugin.created_at is None else datetime.fromisoformat(plugin.created_at),
|
||||
created_by=plugin.created_by,
|
||||
approved_by=plugin.approved_by,
|
||||
published_by=plugin.published_by,
|
||||
security_scan_passed=plugin.security_scan_passed,
|
||||
visibility="public" if plugin.enabled else "private",
|
||||
source={"source": "archive", "url": _download_url(request, plugin.name)} if local_archive else plugin.source,
|
||||
download_url=_download_url(request, plugin.name) if local_archive else None,
|
||||
|
|
@ -137,28 +145,74 @@ async def list_skills(
|
|||
query: str = "",
|
||||
offset: Annotated[int, Query(ge=0)] = 0,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 50,
|
||||
tag: str = "",
|
||||
security_scan_passed: bool | None = None,
|
||||
sort_by: SkillSortField = "name",
|
||||
sort_order: Literal["asc", "desc"] = "asc",
|
||||
) -> SkillHubList:
|
||||
"""Browse the caller's public and granted private skills."""
|
||||
catalog: Final = await list_plugins(enabled_only=False, user_api_key_dict=caller)
|
||||
needle: Final = query.casefold()
|
||||
catalog: Final = await list_visible_plugins(enabled_only=False, user_api_key_dict=caller)
|
||||
needle: Final = query.strip().casefold()
|
||||
selected_tag: Final = tag.strip().casefold()
|
||||
matched: Final = tuple(
|
||||
plugin
|
||||
for plugin in sorted(catalog.plugins, key=lambda item: item.name)
|
||||
if needle in " ".join((plugin.name, plugin.description or "", *(plugin.keywords or ()))).casefold()
|
||||
if needle
|
||||
in " ".join(
|
||||
(
|
||||
plugin.name,
|
||||
plugin.description or "",
|
||||
plugin.domain or "",
|
||||
plugin.namespace or "",
|
||||
plugin.created_by or "",
|
||||
plugin.approved_by or "",
|
||||
plugin.published_by or "",
|
||||
*(plugin.keywords or ()),
|
||||
)
|
||||
).casefold()
|
||||
and (not selected_tag or selected_tag in {word.casefold() for word in plugin.keywords or ()})
|
||||
and (security_scan_passed is None or plugin.security_scan_passed == security_scan_passed)
|
||||
)
|
||||
page: Final = matched[offset : offset + limit]
|
||||
counts: Final[Mapping[str, int]] = (
|
||||
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page))
|
||||
if is_proxy_admin(caller)
|
||||
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in matched))
|
||||
if sort_by == "download_count"
|
||||
else {}
|
||||
)
|
||||
ordered: Final = sorted(
|
||||
matched, key=lambda plugin: _skill_sort_key(plugin, sort_by, counts), reverse=sort_order == "desc"
|
||||
)
|
||||
page: Final = ordered[offset : offset + limit]
|
||||
page_counts: Final = (
|
||||
await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page))
|
||||
if sort_by != "download_count"
|
||||
else counts
|
||||
)
|
||||
return SkillHubList(
|
||||
skills=tuple(_skill_response(request, plugin, counts.get(plugin.id)) for plugin in page),
|
||||
skills=tuple(_skill_response(request, plugin, page_counts.get(plugin.id)) for plugin in page),
|
||||
count=len(matched),
|
||||
next_offset=offset + limit if offset + limit < len(matched) else None,
|
||||
tags=tuple(
|
||||
sorted(set(chain.from_iterable(plugin.keywords or () for plugin in catalog.plugins)), key=str.casefold)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _skill_sort_key(plugin: PluginListItem, field: SkillSortField, counts: Mapping[str, int]) -> str | float:
|
||||
if field == "download_count":
|
||||
return counts.get(plugin.id, 0)
|
||||
if field == "security_scan_passed":
|
||||
return int(plugin.security_scan_passed)
|
||||
if field in {"created_at", "updated_at"}:
|
||||
value: Final = plugin.created_at if field == "created_at" else plugin.updated_at
|
||||
return datetime.fromisoformat(value).timestamp() if value else 0
|
||||
return {
|
||||
"name": plugin.name,
|
||||
"created_by": plugin.created_by or "",
|
||||
"published_by": plugin.published_by or "",
|
||||
"approved_by": plugin.approved_by or "",
|
||||
}[field].casefold()
|
||||
|
||||
|
||||
@router.get("/{name}", response_model=SkillHubSkill)
|
||||
async def get_skill(
|
||||
request: Request,
|
||||
|
|
@ -176,6 +230,7 @@ async def get_skill(
|
|||
|
||||
def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -> SkillHubSkill:
|
||||
manifest: Final = MANIFEST_FIELDS.validate_json(metadata.manifest_json or "{}")
|
||||
review: Final = read_plugin_review(manifest)
|
||||
return _skill_response(
|
||||
request,
|
||||
PluginListItem.model_validate(
|
||||
|
|
@ -187,7 +242,11 @@ def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -
|
|||
"source": dict(metadata.source),
|
||||
"has_agent_skill_metadata": metadata.has_agent_skill_metadata,
|
||||
"enabled": metadata.enabled,
|
||||
"created_at": None,
|
||||
"created_at": metadata.created_at.isoformat() if metadata.created_at is not None else None,
|
||||
"created_by": metadata.created_by,
|
||||
"approved_by": review.approved_by if review else None,
|
||||
"published_by": read_plugin_publisher(manifest),
|
||||
"security_scan_passed": review.security_scan_passed if review else False,
|
||||
"updated_at": metadata.updated_at.isoformat() if metadata.updated_at is not None else None,
|
||||
"domain": manifest.get("domain"),
|
||||
"namespace": manifest.get("namespace"),
|
||||
|
|
@ -416,7 +475,11 @@ def skill_publication_response(
|
|||
"version": published.version,
|
||||
"enabled": published.enabled,
|
||||
"has_agent_skill_metadata": has_agent_skill_metadata(publication.manifest),
|
||||
"created_at": None,
|
||||
"created_at": published.created_at.isoformat() if published.created_at is not None else None,
|
||||
"created_by": published.created_by,
|
||||
"approved_by": published.review.approved_by if published.review else None,
|
||||
"published_by": published.published_by,
|
||||
"security_scan_passed": published.review.security_scan_passed if published.review else False,
|
||||
"updated_at": published.updated_at.isoformat(),
|
||||
"domain": publication.manifest.get("domain"),
|
||||
"namespace": publication.manifest.get("namespace"),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@
|
|||
#
|
||||
# +-------------------------------------------------------------+
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from itertools import chain, dropwhile, takewhile
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -14,6 +15,10 @@ from litellm.integrations.custom_guardrail import (
|
|||
CustomGuardrail,
|
||||
log_guardrail_information,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
effective_scan_only_tool_results_for_guardrail,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -218,15 +223,14 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
zscaler_ai_guard_result = None
|
||||
direction: Final = "OUT" if input_type == "response" else "IN"
|
||||
verbose_proxy_logger.debug("direction: %s", direction)
|
||||
# Concatenate all texts and send to Zscaler AI Guard
|
||||
if texts:
|
||||
concatenated_text: Final = " ".join(texts)
|
||||
content: Final = self._content_to_scan(inputs, input_type)
|
||||
if content:
|
||||
zscaler_ai_guard_result = await self.make_zscaler_ai_guard_api_call(
|
||||
zscaler_ai_guard_url=self.zscaler_ai_guard_url,
|
||||
api_key=self.api_key,
|
||||
policy_id=policy_id,
|
||||
direction=direction,
|
||||
content=concatenated_text,
|
||||
content=content,
|
||||
user=user_for_payload,
|
||||
**kwargs,
|
||||
)
|
||||
|
|
@ -244,6 +248,37 @@ class ZscalerAIGuard(CustomGuardrail):
|
|||
verbose_proxy_logger.debug("ZscalerAIGuard: Successfully applied guardrail.")
|
||||
return inputs
|
||||
|
||||
def _content_to_scan(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
input_type: Literal["request", "response"],
|
||||
) -> str | None:
|
||||
structured_messages: Final = inputs.get("structured_messages") or []
|
||||
scans_last_user_turn: Final = (
|
||||
input_type == "request"
|
||||
and not effective_scan_only_tool_results_for_guardrail(self)
|
||||
and any("content" in message for message in structured_messages)
|
||||
)
|
||||
texts: Final = (
|
||||
self._last_user_turn_texts(structured_messages) if scans_last_user_turn else inputs.get("texts", [])
|
||||
)
|
||||
return " ".join(texts) if texts else None
|
||||
|
||||
@staticmethod
|
||||
def _last_user_turn_texts(messages: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
|
||||
from_last_user_message: Final = dropwhile(lambda message: message.get("role") != "user", reversed(messages))
|
||||
last_user_turn: Final = tuple(takewhile(lambda message: message.get("role") == "user", from_last_user_message))
|
||||
texts: Final = tuple(chain.from_iterable(map(ZscalerAIGuard._content_texts, reversed(last_user_turn))))
|
||||
return texts if any(text.strip() for text in texts) else ()
|
||||
|
||||
@staticmethod
|
||||
def _content_texts(message: Mapping[str, object]) -> tuple[str, ...]:
|
||||
content: Final = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return (content,)
|
||||
parts: Final = (as_str_mapping(part) for part in as_object_list(content) or ())
|
||||
return tuple(text for part in parts if part is not None and isinstance(text := part.get("text"), str))
|
||||
|
||||
def extract_blocking_info(self, response):
|
||||
"""
|
||||
Extracts transaction ID and blocking detector details from a response.
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ def _get_routing_strategies_from_router_class() -> list[str]:
|
|||
Dynamically extract routing strategies from the Router class __init__ method.
|
||||
"""
|
||||
# Get the __init__ signature
|
||||
sig: Final = inspect.signature(Router.__init__)
|
||||
sig: Final = inspect.signature(Router)
|
||||
|
||||
# Get the routing_strategy parameter
|
||||
routing_strategy_param: Final = sig.parameters.get("routing_strategy")
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ from litellm.proxy._types import (
|
|||
CommonProxyErrors,
|
||||
)
|
||||
from litellm.proxy.db.health_check_latest import latest_by_model_name
|
||||
from litellm.proxy.deps.repositories import PluginUploadRepositoryDep
|
||||
from litellm.proxy.utils import get_custom_url
|
||||
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
|
||||
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets
|
||||
from litellm.types.agents import AgentCard
|
||||
|
|
@ -71,6 +73,9 @@ class _EndpointEntry(TypedDict):
|
|||
|
||||
|
||||
class _PluginRow(Protocol):
|
||||
@property
|
||||
def created_by(self) -> str | None: ...
|
||||
|
||||
@property
|
||||
def id(self) -> str: ...
|
||||
|
||||
|
|
@ -366,7 +371,7 @@ async def get_mcp_servers():
|
|||
"/public/skill_hub",
|
||||
tags=["public", "Claude Code Marketplace"],
|
||||
)
|
||||
async def public_skill_hub():
|
||||
async def public_skill_hub(uploads: PluginUploadRepositoryDep):
|
||||
"""Return enabled (public) Claude Code skills, or none when the marketplace requires a key."""
|
||||
from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import (
|
||||
MANIFEST_FIELDS,
|
||||
|
|
@ -385,13 +390,22 @@ async def public_skill_hub():
|
|||
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
|
||||
try:
|
||||
plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True})
|
||||
counts: Final[Mapping[str, int]] = (
|
||||
await uploads.download_counts(tuple(plugin.id for plugin in plugins)) if uploads else {}
|
||||
)
|
||||
items: Final = []
|
||||
for plugin in plugins:
|
||||
raw = plugin.manifest_json or {}
|
||||
manifest = json.loads(raw) if isinstance(raw, str) else raw
|
||||
review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest))
|
||||
items.append(
|
||||
PluginListItem(
|
||||
id=plugin.id,
|
||||
download_count=counts.get(plugin.id),
|
||||
created_by=plugin.created_by,
|
||||
approved_by=review.approved_by if review else None,
|
||||
published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)),
|
||||
security_scan_passed=review.security_scan_passed if review else False,
|
||||
name=plugin.name,
|
||||
enabled=plugin.enabled,
|
||||
created_at=str(plugin.created_at) if plugin.created_at else None,
|
||||
|
|
|
|||
|
|
@ -271,6 +271,10 @@ REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = Map
|
|||
{"acreate_interaction": ("model", "agent"), "acreate_agent": ("id", "name")}
|
||||
)
|
||||
|
||||
BODY_PARAM_ALTERNATIVES_BY_ROUTE: Final[Mapping[str, Mapping[str, tuple[str, ...]]]] = MappingProxyType(
|
||||
{"acreate_interaction": MappingProxyType({"input": ("continuation_token",)})}
|
||||
)
|
||||
|
||||
JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_OPTIONAL_JSON_OBJECT_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(
|
||||
Mapping[str, object] | None
|
||||
|
|
@ -320,8 +324,11 @@ def _find_missing_required_body_param(
|
|||
)
|
||||
if missing_merge_base_param is not None:
|
||||
return MissingBodyParam(name=missing_merge_base_param, model_deployments_loaded=True)
|
||||
alternatives: Final = BODY_PARAM_ALTERNATIVES_BY_ROUTE.get(route_type, {})
|
||||
missing_present_params: Final = tuple(
|
||||
param for param in REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE.get(route_type, ()) if param not in data
|
||||
param
|
||||
for param in REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE.get(route_type, ())
|
||||
if param not in data and not any(alternative in data for alternative in alternatives.get(param, ()))
|
||||
)
|
||||
if not missing_present_params:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,15 +1,18 @@
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from semver import Version
|
||||
|
||||
from litellm.models.plugin_upload import (
|
||||
PluginPublication,
|
||||
PluginPublicationBase,
|
||||
PluginPublicationError,
|
||||
PluginReview,
|
||||
PublishedPlugin,
|
||||
)
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
|
|
@ -23,6 +26,22 @@ if TYPE_CHECKING:
|
|||
|
||||
_MANIFEST: Final = TypeAdapter(dict[str, object])
|
||||
_SOURCE: Final = TypeAdapter(dict[str, str])
|
||||
_REVIEW: Final = TypeAdapter(PluginReview)
|
||||
_PUBLISHER: Final = TypeAdapter[str | None](str | None)
|
||||
|
||||
|
||||
def read_plugin_review(manifest: Mapping[str, object]) -> PluginReview | None:
|
||||
try:
|
||||
return _REVIEW.validate_python(manifest.get("_litellm_review"))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def read_plugin_publisher(manifest: Mapping[str, object]) -> str | None:
|
||||
try:
|
||||
return _PUBLISHER.validate_python(manifest.get("_litellm_published_by"))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
class PluginPublicationTransaction(Protocol):
|
||||
|
|
@ -122,7 +141,22 @@ async def publish_plugin(
|
|||
)
|
||||
if isinstance(version, PluginPublicationError):
|
||||
return version
|
||||
manifest_json: Final = json.dumps({**publication.manifest, "version": version})
|
||||
current_manifest: Final = _MANIFEST.validate_json(current.manifest_json or "{}") if current is not None else {}
|
||||
same_source: Final = current is not None and current_manifest.get("source") == publication.manifest.get("source")
|
||||
review: Final = publication.review or (read_plugin_review(current_manifest) if same_source else None)
|
||||
publisher: Final = publication.review.approved_by if publication.review is not None else publication.created_by
|
||||
manifest_json: Final = json.dumps(
|
||||
{
|
||||
**{
|
||||
key: value
|
||||
for key, value in publication.manifest.items()
|
||||
if key not in {"_litellm_review", "_litellm_published_by"}
|
||||
},
|
||||
"version": version,
|
||||
**({"_litellm_published_by": publisher} if publisher is not None else {}),
|
||||
**({"_litellm_review": asdict(review)} if review is not None else {}),
|
||||
}
|
||||
)
|
||||
if publication.content is not None and verified_archive(publication.content, manifest_json) is None:
|
||||
return PluginPublicationError("conflict", "Uploaded archive does not match its published source digest")
|
||||
timestamp: Final = datetime.now(timezone.utc)
|
||||
|
|
@ -141,8 +175,18 @@ async def publish_plugin(
|
|||
await tx.litellm_claudecodepluginupload.create(
|
||||
data={"plugin_name": publication.name, "content": Base64.encode(publication.content)}
|
||||
)
|
||||
await _snapshot(tx, created, publication.content, publication.created_by)
|
||||
return PublishedPlugin(created.id, created.name, version, created.enabled, created.updated_at or timestamp)
|
||||
await _snapshot(tx, created, publication.content, publisher)
|
||||
return PublishedPlugin(
|
||||
created.id,
|
||||
created.name,
|
||||
version,
|
||||
created.enabled,
|
||||
created.updated_at or timestamp,
|
||||
created.created_at,
|
||||
created.created_by,
|
||||
review,
|
||||
publisher,
|
||||
)
|
||||
previous_archive: Final = await tx.litellm_claudecodepluginupload.find_unique(where={"plugin_name": current.name})
|
||||
previous_content: Final = (
|
||||
verified_archive(previous_archive.content.decode(), current.manifest_json)
|
||||
|
|
@ -152,11 +196,7 @@ async def publish_plugin(
|
|||
history_count: Final = await tx.litellm_claudecodepluginversion.count(where={"plugin_id": current.id}, take=1)
|
||||
if history_count == 0:
|
||||
await _snapshot(tx, current, previous_content, None)
|
||||
unchanged_content: Final = (
|
||||
previous_content
|
||||
if _MANIFEST.validate_json(current.manifest_json or "{}").get("source") == publication.manifest.get("source")
|
||||
else None
|
||||
)
|
||||
unchanged_content: Final = previous_content if same_source else None
|
||||
content: Final = publication.content if publication.content is not None else unchanged_content
|
||||
updated: Final = await tx.litellm_claudecodeplugintable.update(where={"id": current.id}, data=data)
|
||||
if updated is None:
|
||||
|
|
@ -171,5 +211,15 @@ async def publish_plugin(
|
|||
"update": {"content": Base64.encode(content)},
|
||||
},
|
||||
)
|
||||
await _snapshot(tx, updated, content, publication.created_by)
|
||||
return PublishedPlugin(updated.id, updated.name, version, updated.enabled, updated.updated_at or timestamp)
|
||||
await _snapshot(tx, updated, content, publisher)
|
||||
return PublishedPlugin(
|
||||
updated.id,
|
||||
updated.name,
|
||||
version,
|
||||
updated.enabled,
|
||||
updated.updated_at or timestamp,
|
||||
updated.created_at,
|
||||
updated.created_by,
|
||||
review,
|
||||
publisher,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -287,4 +287,6 @@ def _archive_metadata(plugin: "prisma_models.LiteLLM_ClaudeCodePluginTable") ->
|
|||
description=plugin.description,
|
||||
has_agent_skill_metadata=has_agent_skill_metadata(manifest),
|
||||
updated_at=plugin.updated_at,
|
||||
created_at=plugin.created_at,
|
||||
created_by=plugin.created_by,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,12 +7,13 @@ from datetime import datetime, timezone
|
|||
from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma's generated delegates are dynamic
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.models.plugin_upload import (
|
||||
PluginPublication,
|
||||
PluginPublicationBase,
|
||||
PluginPublicationError,
|
||||
PluginReview,
|
||||
PublishedPlugin,
|
||||
uploaded_plugin_source,
|
||||
)
|
||||
|
|
@ -312,7 +313,9 @@ class SkillSubmissionRepository:
|
|||
if approve and content is not None and publication_base_url
|
||||
else _SOURCE.validate_python(submission.proposal.manifest.get("source"))
|
||||
)
|
||||
publication: Final = await _publish_submission(transaction, row, source, content) if approve else None
|
||||
publication: Final = (
|
||||
await _publish_submission(transaction, row, source, content, reviewer) if approve else None
|
||||
)
|
||||
if isinstance(publication, PluginPublicationError):
|
||||
return publication
|
||||
reviewed_at: Final = datetime.now(timezone.utc)
|
||||
|
|
@ -346,6 +349,7 @@ async def _publish_submission(
|
|||
row: "prisma_models.LiteLLM_SkillSubmission",
|
||||
source: Mapping[str, str],
|
||||
content: bytes | None,
|
||||
reviewer: str,
|
||||
) -> PublishedPlugin | PluginPublicationError:
|
||||
stored: Final = _StoredProposal.model_validate_json(row.proposal_json)
|
||||
proposal: Final = stored.proposal(row.plugin_name)
|
||||
|
|
@ -363,6 +367,24 @@ async def _publish_submission(
|
|||
manifest={**stored.manifest, "source": dict(source)},
|
||||
content=content,
|
||||
created_by=row.submitted_by,
|
||||
review=PluginReview(reviewer, scan_passed_for_archive(row.scan_json, source.get("sha256"))),
|
||||
),
|
||||
base,
|
||||
)
|
||||
|
||||
|
||||
def scan_passed_for_archive(scan_json: str | None, digest: str | None) -> bool:
|
||||
if not scan_json or not digest:
|
||||
return False
|
||||
try:
|
||||
scan: Final = SkillScan.model_validate_json(scan_json)
|
||||
except ValidationError:
|
||||
return False
|
||||
return (
|
||||
scan.status == "completed"
|
||||
and scan.archive_sha256 == digest
|
||||
and scan.report is not None
|
||||
and scan.report.complete
|
||||
and scan.report.risk_score == 0
|
||||
and not scan.report.findings
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import asyncio
|
||||
from collections.abc import Callable, Coroutine, Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Final, TypeAlias
|
||||
|
|
@ -297,12 +297,8 @@ class StreamedRoundBilling:
|
|||
self._close(parked)
|
||||
self._bank((self._dispatch(row, billed, self._logging_obj.attempt_scoped_copy()),), row)
|
||||
|
||||
def _dispatch(
|
||||
self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj
|
||||
) -> BilledRoundsDispatch | Coroutine[object, object, None]:
|
||||
def _dispatch(self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj) -> BilledRoundsDispatch:
|
||||
dispatch: Final = self._log_success(_SuccessLog(logger, row, billed.chat_usage))
|
||||
if billed.chat_usage is None:
|
||||
return dispatch
|
||||
return BilledRoundsDispatch(dispatch, billed.spend)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
|
|||
requests_background,
|
||||
without_budget_reservation,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
|
||||
|
|
@ -638,6 +638,24 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self._handle_failure(e)
|
||||
raise
|
||||
|
||||
def _deferred_logging_dispatch(
|
||||
self, logging_coroutine: Coroutine[object, object, None], logging_response: object
|
||||
) -> BilledRoundsDispatch | Coroutine[object, object, None]:
|
||||
response: Final[object] = getattr(logging_response, "response", None)
|
||||
if not isinstance(response, ResponsesAPIResponse):
|
||||
return logging_coroutine
|
||||
price: Final = (
|
||||
UncachedPrice(0.0, None)
|
||||
if self._completed_response_cache_hit
|
||||
else self.logging_obj.uncached_price(response)
|
||||
)
|
||||
spend: Final = BilledRoundsSpend(
|
||||
ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response.usage),
|
||||
price.cost or 0.0,
|
||||
price.cost_breakdown,
|
||||
)
|
||||
return BilledRoundsDispatch(logging_coroutine, spend)
|
||||
|
||||
def _log_completed_response(self, *, is_async: bool) -> None:
|
||||
if self._completed_response_logged:
|
||||
return
|
||||
|
|
@ -662,7 +680,9 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if deferred_dispatch_armed:
|
||||
# End-of-stream guardrail scans write guardrail_information after
|
||||
# the terminal event; dispatching now would snapshot metadata early.
|
||||
self.logging_obj._deferred_stream_complete_args = (logging_coroutine,)
|
||||
self.logging_obj._deferred_stream_complete_args = (
|
||||
self._deferred_logging_dispatch(logging_coroutine, logging_response),
|
||||
)
|
||||
else:
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging"
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ if TYPE_CHECKING:
|
|||
# ``Router(...)``'s parameters, their defaults and its docstring: ``router_arguments`` binds a call
|
||||
# against them and ``Router.__init__`` reports them. The body is never run.
|
||||
def _router_init(
|
||||
self,
|
||||
self: object,
|
||||
model_list: list[DeploymentTypedDict] | list[dict[str, Any]] | None = None,
|
||||
## ASSISTANTS API ##
|
||||
assistants_config: AssistantsTypedDict | None = None,
|
||||
|
|
@ -222,17 +222,22 @@ def _class_descriptor(cls: type, name: str) -> bool:
|
|||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import overload
|
||||
from typing import ParamSpec, overload
|
||||
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject
|
||||
from litellm.types.utils import EmbeddingResponse, LiteLLMBatch, ModelResponse
|
||||
|
||||
_P: Final = ParamSpec("_P")
|
||||
|
||||
def _takes_parameters_of(source: Callable[_P, None]) -> Callable[[Callable[..., None]], Callable[_P, None]]: ...
|
||||
|
||||
class Router(RustCore, DeprecatedRouterMembers):
|
||||
"""What type checkers see: the constructor's parameters and the endpoints' return types. The
|
||||
endpoints are served by the core at run time (litellm/router_rust/invoke/endpoints.py)."""
|
||||
|
||||
__init__: Callable[..., None] = _router_init # pyright: ignore[reportIncompatibleMethodOverride, reportIncompatibleVariableOverride, reportUnknownVariableType] # a bare assignment would read as RustCore's constructor
|
||||
@_takes_parameters_of(_router_init)
|
||||
def __init__(self) -> None: ... # pyright: ignore[reportMissingSuperCall] # a type-checking stub
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: RouterConfig, host: PythonHost = python_host) -> "Router": ...
|
||||
|
|
|
|||
|
|
@ -1907,10 +1907,9 @@ class RustEndpoints:
|
|||
if self._scheduler is None:
|
||||
from litellm.scheduler import Scheduler
|
||||
|
||||
self._scheduler = (
|
||||
Scheduler(polling_interval=self._polling_interval) # pyright: ignore[reportArgumentType] # as configured
|
||||
if self._polling_interval is not None
|
||||
else Scheduler()
|
||||
self._scheduler = Scheduler(
|
||||
polling_interval=self._polling_interval, # pyright: ignore[reportArgumentType] # as configured
|
||||
redis_cache=self._core.routing_cache.redis_cache,
|
||||
)
|
||||
return self._scheduler
|
||||
|
||||
|
|
|
|||
|
|
@ -264,10 +264,18 @@ class WalkMetadata:
|
|||
|
||||
def hop_kwargs(self, event: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a fresh dict for the loggers
|
||||
"""The kwargs the level held after a hop (``kwargs["model"] = target``), as the
|
||||
fallback loggers see them."""
|
||||
fallback loggers see them: the request's own metadata (the proxy's ``user_api_key_*``
|
||||
labels) with the level's walk stamps laid over it, as ``run_async_fallback``'s
|
||||
``_hop_metadata`` copies it."""
|
||||
scope: Final = self._scope(event.get("scope"))
|
||||
model: Final = scope.metadata.get("model_group", event.get("to"))
|
||||
return {**self._request, **scope.kwargs, "model": model, self._field: dict(scope.metadata)}
|
||||
request_bucket: Final = as_str_mapping(self._request.get(self._field)) or {}
|
||||
return {
|
||||
**self._request,
|
||||
**scope.kwargs,
|
||||
"model": model,
|
||||
self._field: {**request_bucket, **scope.metadata},
|
||||
}
|
||||
|
||||
def log_retry(self, event: Mapping[str, object], error: BaseException | None) -> None:
|
||||
"""A ``retry_logged`` event: the failure joins the scope's ``previous_models``."""
|
||||
|
|
|
|||
|
|
@ -279,6 +279,7 @@ class RustHealthCallbacks:
|
|||
is_caller_scoped_auth_failure,
|
||||
is_caller_timeout_408,
|
||||
is_failure_outside_selected_deployment,
|
||||
is_response_lookup_not_found,
|
||||
is_video_lookup_not_found,
|
||||
)
|
||||
|
||||
|
|
@ -288,6 +289,7 @@ class RustHealthCallbacks:
|
|||
is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None)
|
||||
or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params))
|
||||
or is_video_lookup_not_found(exception, logged)
|
||||
or is_response_lookup_not_found(exception, logged)
|
||||
or is_caller_timeout_408(logged, status) # pyright: ignore[reportArgumentType] # the status as raised
|
||||
or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status) # pyright: ignore[reportArgumentType] # see above
|
||||
)
|
||||
|
|
|
|||
|
|
@ -108,6 +108,26 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob
|
|||
return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES
|
||||
|
||||
|
||||
# The calls on an existing response id. ``compact_responses`` is left out: it sends a model and input,
|
||||
# so its 404 can mean a missing model rather than a missing response.
|
||||
RESPONSE_ID_CALL_TYPES: Final = frozenset(
|
||||
{
|
||||
"aget_responses",
|
||||
"get_responses",
|
||||
"adelete_responses",
|
||||
"delete_responses",
|
||||
"acancel_responses",
|
||||
"cancel_responses",
|
||||
"alist_input_items",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def is_response_lookup_not_found(exception: object, failure_kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether a call on an existing response id failed with a provider 404: the response is not on that account."""
|
||||
return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in RESPONSE_ID_CALL_TYPES
|
||||
|
||||
|
||||
_EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = (
|
||||
# ContentPolicyViolationError subclasses BadRequestError, so it must be checked first.
|
||||
(litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"),
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ async def send_llm_exception_alert(
|
|||
ctx: "RoutingContext | None",
|
||||
request_kwargs: dict,
|
||||
error_traceback_str: str,
|
||||
original_exception,
|
||||
original_exception: Exception,
|
||||
):
|
||||
"""
|
||||
Only runs if router.slack_alerting_logger is set
|
||||
|
|
@ -34,7 +34,7 @@ async def send_llm_exception_alert(
|
|||
|
||||
Parameters:
|
||||
ctx (RoutingContext): The router's routing context.
|
||||
original_exception (Any): The original exception that occurred.
|
||||
original_exception (Exception): The original exception that occurred.
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
|
|
|||
|
|
@ -163,6 +163,9 @@ class PromptCachingDeploymentCheck(CustomLogger):
|
|||
|
||||
return healthy_deployments
|
||||
|
||||
def exempt_from_message_redaction(self) -> bool:
|
||||
return True
|
||||
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: _PromptCachingSuccessEventKwargs,
|
||||
|
|
|
|||
|
|
@ -176,11 +176,16 @@ class Status1(Enum):
|
|||
incomplete = "incomplete"
|
||||
budget_exceeded = "budget_exceeded"
|
||||
queued = "queued"
|
||||
requires_continuation = "requires_continuation"
|
||||
|
||||
|
||||
class InteractionStatusUpdate(LiteLLMBaseModel):
|
||||
interaction_id: str | None = None
|
||||
status: Status1 | None = None
|
||||
continuation_token: str | None = Field(
|
||||
None,
|
||||
description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
)
|
||||
event_type: Literal["interaction.status_update"] = "interaction.status_update"
|
||||
event_id: str | None = Field(
|
||||
None,
|
||||
|
|
@ -343,6 +348,7 @@ class Status3(Enum):
|
|||
INCOMPLETE = "incomplete"
|
||||
BUDGET_EXCEEDED = "budget_exceeded"
|
||||
QUEUED = "queued"
|
||||
REQUIRES_CONTINUATION = "requires_continuation"
|
||||
|
||||
|
||||
class ModelOption(RootModel[str]):
|
||||
|
|
@ -604,6 +610,9 @@ class ToolChoice(RootModel[ToolChoiceType | ToolChoiceConfig]):
|
|||
|
||||
class Usage(LiteLLMBaseModel):
|
||||
total_input_tokens: int | None = Field(None, description="Number of tokens in the prompt (context).")
|
||||
initial_input_tokens: int | None = Field(
|
||||
None, description="Number of tokens in the prompt of the first hop of a long decode."
|
||||
)
|
||||
input_tokens_by_modality: list[ModalityTokens] | None = Field(
|
||||
None, description="A breakdown of input token usage by modality."
|
||||
)
|
||||
|
|
@ -625,6 +634,7 @@ class Usage(LiteLLMBaseModel):
|
|||
None, description="A breakdown of tool-use token usage by modality."
|
||||
)
|
||||
total_reasoning_tokens: int | None = Field(None, description="Number of tokens of thoughts for thinking models.")
|
||||
total_thought_tokens: int | None = Field(None, description="Number of tokens of thoughts for thinking models.")
|
||||
total_tokens: int | None = Field(
|
||||
None,
|
||||
description="Total token count for the interaction request (prompt + responses + other\ninternal tokens).",
|
||||
|
|
@ -787,6 +797,10 @@ class Interaction(LiteLLMBaseModel):
|
|||
description="The mime type of the response. This is required if response_format is set.",
|
||||
)
|
||||
previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.")
|
||||
continuation_token: str | None = Field(
|
||||
None,
|
||||
description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
)
|
||||
input: str | list[Content] | list[Turn] | Content | None = Field(
|
||||
None, description="The inputs for the interaction."
|
||||
)
|
||||
|
|
@ -843,6 +857,10 @@ class CreateModelInteractionParams(LiteLLMBaseModel):
|
|||
description="The mime type of the response. This is required if response_format is set.",
|
||||
)
|
||||
previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.")
|
||||
continuation_token: str | None = Field(
|
||||
None,
|
||||
description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
)
|
||||
input: str | list[Content] | list[Turn] | Content = Field(..., description="The inputs for the interaction.")
|
||||
generation_config: GenerationConfig | None = Field(
|
||||
None,
|
||||
|
|
@ -894,6 +912,10 @@ class CreateAgentInteractionParams(LiteLLMBaseModel):
|
|||
description="The mime type of the response. This is required if response_format is set.",
|
||||
)
|
||||
previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.")
|
||||
continuation_token: str | None = Field(
|
||||
None,
|
||||
description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
)
|
||||
input: str | list[Content] | list[Turn] | Content = Field(..., description="The inputs for the interaction.")
|
||||
agent_config: DynamicAgentConfig | DeepResearchAgentConfig | None = Field(
|
||||
None, description="Configuration for the agent.", discriminator="type"
|
||||
|
|
@ -1080,6 +1102,7 @@ class InteractionsAPIResponse(BaseLiteLLMOpenAIResponseObject):
|
|||
# New schema field (Api-Revision: 2026-05-20).
|
||||
steps: list[dict[str, Any]] | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
continuation_token: str | None = None
|
||||
|
||||
_hidden_params: dict = PrivateAttr(default_factory=dict)
|
||||
|
||||
|
|
@ -1126,6 +1149,7 @@ class InteractionsAPIStreamingResponse(BaseLiteLLMOpenAIResponseObject):
|
|||
step: dict[str, Any] | None = None
|
||||
interaction_id: str | None = None
|
||||
interaction: dict[str, Any] | None = None
|
||||
continuation_token: str | None = None
|
||||
|
||||
_hidden_params: dict = PrivateAttr(default_factory=dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -154,6 +154,11 @@ class PluginListItem(LiteLLMBaseModel):
|
|||
enabled: bool
|
||||
created_at: str | None
|
||||
updated_at: str | None
|
||||
created_by: str | None = None
|
||||
approved_by: str | None = None
|
||||
published_by: str | None = None
|
||||
security_scan_passed: bool = False
|
||||
download_count: int | None = None
|
||||
has_agent_skill_metadata: bool = Field(default=False, exclude=True)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,17 @@ from pydantic import ConfigDict, Field
|
|||
from litellm.models.skill_submission import SubmissionOperation, SubmissionStatus
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
SkillSortField: TypeAlias = Literal[
|
||||
"name",
|
||||
"download_count",
|
||||
"updated_at",
|
||||
"created_at",
|
||||
"created_by",
|
||||
"published_by",
|
||||
"approved_by",
|
||||
"security_scan_passed",
|
||||
]
|
||||
|
||||
|
||||
class SkillHubSkill(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
|
@ -16,6 +27,11 @@ class SkillHubSkill(LiteLLMBaseModel):
|
|||
description: str | None = None
|
||||
version: str | None = None
|
||||
updated_at: datetime | None = None
|
||||
created_at: datetime | None = None
|
||||
created_by: str | None = None
|
||||
approved_by: str | None = None
|
||||
published_by: str | None = None
|
||||
security_scan_passed: bool = False
|
||||
visibility: Literal["public", "private"]
|
||||
source: Mapping[str, str]
|
||||
download_url: str | None = None
|
||||
|
|
@ -23,12 +39,11 @@ class SkillHubSkill(LiteLLMBaseModel):
|
|||
domain: str | None = None
|
||||
namespace: str | None = None
|
||||
keywords: tuple[str, ...] | None = None
|
||||
download_count: int | None = Field(
|
||||
None, description="Gateway ZIP downloads, included in proxy-admin catalog listings"
|
||||
)
|
||||
download_count: int | None = Field(None, description="Gateway ZIP downloads, including repeat downloads")
|
||||
|
||||
|
||||
class SkillHubList(LiteLLMBaseModel):
|
||||
tags: tuple[str, ...] = ()
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
skills: tuple[SkillHubSkill, ...]
|
||||
|
|
|
|||
|
|
@ -703,6 +703,8 @@ class CallTypes(str, Enum):
|
|||
adelete_responses = "adelete_responses"
|
||||
cancel_responses = "cancel_responses"
|
||||
acancel_responses = "acancel_responses"
|
||||
compact_responses = "compact_responses"
|
||||
acompact_responses = "acompact_responses"
|
||||
alist_input_items = "alist_input_items"
|
||||
llm_passthrough_route = "llm_passthrough_route"
|
||||
allm_passthrough_route = "allm_passthrough_route"
|
||||
|
|
@ -806,6 +808,8 @@ CallTypesLiteral = Literal[
|
|||
"send_message",
|
||||
"aresponses",
|
||||
"responses",
|
||||
"compact_responses",
|
||||
"acompact_responses",
|
||||
"acreate_skill",
|
||||
"acreate_realtime_client_secret",
|
||||
"arealtime_calls",
|
||||
|
|
|
|||
|
|
@ -4046,6 +4046,8 @@
|
|||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"deprecation_date": "2027-09-01",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -4068,7 +4070,7 @@
|
|||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement"
|
||||
},
|
||||
"azure_ai/claude-opus-5": {
|
||||
"deprecation_date": "2027-07-08",
|
||||
|
|
@ -13458,13 +13460,16 @@
|
|||
"supports_reasoning": true
|
||||
},
|
||||
"azure_ai/mistral-document-ai-2512": {
|
||||
"deprecation_date": "2027-02-15",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"litellm_provider": "azure_ai",
|
||||
"ocr_cost_per_page": 0.003,
|
||||
"mode": "ocr",
|
||||
"supported_endpoints": [
|
||||
"/v1/ocr"
|
||||
],
|
||||
"source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"successor_model": "azure_ai/mistral-ocr-4-0"
|
||||
},
|
||||
"azure_ai/mistral-ocr-4-0": {
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -40893,6 +40898,22 @@
|
|||
"output_cost_per_token": 4e-07,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"jina-reranker-v3.5": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "jina_ai",
|
||||
"max_input_tokens": 131072,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://api.jina.ai/v1/models"
|
||||
},
|
||||
"jina-reranker-m0": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "jina_ai",
|
||||
"max_input_tokens": 10240,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://api.jina.ai/v1/models"
|
||||
},
|
||||
"jina-reranker-v2-base-multilingual": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"litellm_provider": "jina_ai",
|
||||
|
|
@ -43771,7 +43792,8 @@
|
|||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"mistral/zai-glm-5-3": {
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
|
|
@ -45270,6 +45292,18 @@
|
|||
"source": "https://tokenfactory.nebius.com/api/public/models_info",
|
||||
"cache_read_input_token_cost": 1.5e-07
|
||||
},
|
||||
"nebius/moonshotai/Kimi-K3.1-Pro": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"litellm_provider": "nebius",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://tokenfactory.nebius.com/api/public/models_info"
|
||||
},
|
||||
"nebius/moonshotai/Kimi-K3": {
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1024000,
|
||||
|
|
@ -71189,6 +71223,8 @@
|
|||
},
|
||||
"gpt-5.5-cyber-preview": {
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"deprecation_date": "2027-02-01",
|
||||
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
|
||||
"input_cost_per_token": 1.25e-05,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
|
|
@ -72148,6 +72184,7 @@
|
|||
"output_cost_per_token": 4.5e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"tpm": 250000,
|
||||
"rpm": 10,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
|
|
@ -72170,6 +72207,7 @@
|
|||
"output_cost_per_token": 4.5e-06,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"tpm": 250000,
|
||||
"rpm": 10,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
|
|
@ -76469,6 +76507,33 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"us-gov.xai.grok-4.7": {
|
||||
"supports_regex_lookaround": false,
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
|
||||
"supports_bedrock_runtime_chat_completions_response_format": true,
|
||||
"cache_read_input_token_cost": 6.6e-07,
|
||||
"cache_read_input_token_cost_flex": 3.3e-07,
|
||||
"cache_read_input_token_cost_priority": 1.155e-06,
|
||||
"input_cost_per_token": 2.64e-06,
|
||||
"input_cost_per_token_batches": 1.32e-06,
|
||||
"input_cost_per_token_flex": 1.32e-06,
|
||||
"input_cost_per_token_priority": 4.62e-06,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 524288,
|
||||
"max_output_tokens": 524288,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.92e-06,
|
||||
"output_cost_per_token_batches": 3.96e-06,
|
||||
"output_cost_per_token_flex": 3.96e-06,
|
||||
"output_cost_per_token_priority": 1.386e-05,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": {
|
||||
"cache_read_input_token_cost": 1.4e-08,
|
||||
"input_cost_per_token": 6e-07,
|
||||
|
|
@ -78437,7 +78502,7 @@
|
|||
"cache_creation_input_token_cost": 1e-05,
|
||||
"cache_creation_input_token_cost_above_1hr": 1.6e-05,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-10-01",
|
||||
"deprecation_date": "2027-01-15",
|
||||
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"input_cost_per_token": 8e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -78447,7 +78512,7 @@
|
|||
"output_cost_per_token": 4e-05,
|
||||
"prompt_cache_min_tokens": 512.0,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.02,"search_context_size_low":0.02,"search_context_size_medium":0.02},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models",
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
|
|
@ -82782,6 +82847,39 @@
|
|||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"gpt-6-cyber": {
|
||||
"cache_creation_input_token_cost": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-05,
|
||||
"cache_read_input_token_cost": 2e-06,
|
||||
"input_cost_per_token": 2e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.0001,
|
||||
"source": "https://developers.openai.com/api/docs/models/gpt-6-cyber",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_search": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gpt-6-astra-law": {
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
"input_cost_per_token": 1e-05,
|
||||
|
|
@ -83380,6 +83478,28 @@
|
|||
"supports_response_schema": true,
|
||||
"source": "https://empiriolabs.ai/models/kimi-k3"
|
||||
},
|
||||
"empiriolabs/kimi-k3-1-pro": {
|
||||
"litellm_provider": "empiriolabs",
|
||||
"mode": "chat",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
"cache_read_input_token_cost": 1.8e-06,
|
||||
"supports_system_messages": true,
|
||||
"supports_vision": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_web_search": true,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.015,
|
||||
"search_context_size_medium": 0.015,
|
||||
"search_context_size_high": 0.015
|
||||
},
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_function_calling": true,
|
||||
"source": "https://empiriolabs.ai/models/kimi-k3-1-pro"
|
||||
},
|
||||
"empiriolabs/mimo-v2-6-flash": {
|
||||
"litellm_provider": "empiriolabs",
|
||||
"mode": "chat",
|
||||
|
|
@ -86383,5 +86503,32 @@
|
|||
"supports_system_messages": false,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"databricks/databricks-kimi-k3-1-pro": {
|
||||
"cache_creation_input_token_cost": 1.99997e-06,
|
||||
"cache_read_input_token_cost": 1.9999e-07,
|
||||
"input_cost_per_token": 1.99997e-06,
|
||||
"input_dbu_cost_per_token": 2.8571e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"metadata": {"notes":"Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.99999e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 100.0
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,11 +83,6 @@ id = "GHSA-45c3-73f7-pv4m"
|
|||
ignoreUntil = 2026-11-27
|
||||
reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-xcx6-jx8p-x3h5"
|
||||
ignoreUntil = 2026-11-24
|
||||
reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0"
|
||||
|
||||
[[PackageOverrides]]
|
||||
name = "pillow"
|
||||
version = "12.3.0"
|
||||
|
|
|
|||
22
packaging/litellm-proxy/pyproject.toml
Normal file
22
packaging/litellm-proxy/pyproject.toml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
[project]
|
||||
name = "litellm-proxy"
|
||||
version = "1.109.0"
|
||||
description = "Opt-in Rust HTTP forwarding server for the LiteLLM Python proxy"
|
||||
requires-python = ">=3.10, <3.16"
|
||||
license = "MIT"
|
||||
|
||||
[build-system]
|
||||
requires = ["maturin==1.15.0"]
|
||||
build-backend = "maturin"
|
||||
|
||||
[tool.maturin]
|
||||
manifest-path = "../../litellm-rust/crates/gateway/Cargo.toml"
|
||||
bindings = "bin"
|
||||
targets = [{ name = "litellm-gateway", kind = "bin" }]
|
||||
profile = "release"
|
||||
editable-profile = "dev"
|
||||
include = [
|
||||
{ path = "rust-toolchain.toml", format = "sdist" },
|
||||
{ path = ".cargo/config.toml", format = "sdist" },
|
||||
{ path = "model_prices_and_context_window.json", format = "sdist" },
|
||||
]
|
||||
|
|
@ -359,6 +359,7 @@ exclude = [
|
|||
]
|
||||
|
||||
[tool.uv]
|
||||
concurrent-builds = 1
|
||||
cache-keys = [
|
||||
{ file = "pyproject.toml" },
|
||||
{ file = "rust-toolchain.toml" },
|
||||
|
|
@ -395,11 +396,12 @@ brotli = "2026-10-22T14:01:00Z"
|
|||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy = { workspace = true }
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
litellm-enterprise = { workspace = true }
|
||||
|
||||
[tool.uv.workspace]
|
||||
members = ["enterprise", "litellm-proxy-extras"]
|
||||
members = ["enterprise", "litellm-proxy-extras", "packaging/litellm-proxy"]
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.109.0"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Build the independent core distribution from shared repository sources."""
|
||||
"""Build standalone core or gateway distributions from shared repository sources."""
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
|
|
@ -19,8 +19,16 @@ SOURCES: Final = (
|
|||
"LICENSE",
|
||||
)
|
||||
|
||||
IN_TREE_RUST_PATH: Final = "../../litellm-rust/"
|
||||
|
||||
def stage_core_distribution(source: Path, destination: Path) -> None:
|
||||
|
||||
class BuildOptions(argparse.Namespace):
|
||||
package: str
|
||||
out_dir: Path
|
||||
sdist_only: bool
|
||||
|
||||
|
||||
def stage_core_distribution(source: Path, destination: Path, *, package: str = "litellm-core") -> None:
|
||||
version: Final = subprocess.run(
|
||||
["uv", "version", "--short"], cwd=source, check=True, capture_output=True, text=True
|
||||
).stdout.strip()
|
||||
|
|
@ -53,8 +61,7 @@ def stage_core_distribution(source: Path, destination: Path) -> None:
|
|||
)(directory, names)
|
||||
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
for name in SOURCES:
|
||||
path: Final = source / name
|
||||
for name, path in ((name, source / name) for name in SOURCES if package == "litellm-core" or name != "litellm"):
|
||||
if path.is_dir():
|
||||
shutil.copytree(
|
||||
path,
|
||||
|
|
@ -63,15 +70,19 @@ def stage_core_distribution(source: Path, destination: Path) -> None:
|
|||
)
|
||||
else:
|
||||
shutil.copy2(path, destination / name)
|
||||
shutil.copy2(source / "packaging/litellm-core/pyproject.toml", destination / "pyproject.toml")
|
||||
template: Final = (source / "packaging" / package / "pyproject.toml").read_text()
|
||||
if package == "litellm-proxy" and IN_TREE_RUST_PATH not in template:
|
||||
raise SystemExit(f"packaging/{package}/pyproject.toml no longer points at {IN_TREE_RUST_PATH}")
|
||||
metadata: Final = template.replace(IN_TREE_RUST_PATH, "litellm-rust/")
|
||||
(destination / "pyproject.toml").write_text(metadata)
|
||||
subprocess.run(["uv", "version", version, "--frozen"], cwd=destination, check=True)
|
||||
|
||||
|
||||
def build_core_distribution(output: Path, *, sdist_only: bool = False) -> None:
|
||||
def build_core_distribution(output: Path, *, sdist_only: bool = False, package: str = "litellm-core") -> None:
|
||||
output_path: Final = output.resolve()
|
||||
with tempfile.TemporaryDirectory(prefix="litellm-core-") as temporary:
|
||||
stage: Final = Path(temporary)
|
||||
stage_core_distribution(ROOT, stage)
|
||||
stage_core_distribution(ROOT, stage, package=package)
|
||||
subprocess.run(
|
||||
["uv", "build", "--python", sys.executable, "--out-dir", str(output_path)]
|
||||
+ (["--sdist"] if sdist_only else []),
|
||||
|
|
@ -82,12 +93,13 @@ def build_core_distribution(output: Path, *, sdist_only: bool = False) -> None:
|
|||
|
||||
def main() -> None:
|
||||
parser: Final = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--package", choices=("litellm-core", "litellm-proxy"), default="litellm-core")
|
||||
parser.add_argument("--out-dir", type=Path, default=ROOT / "dist/core")
|
||||
parser.add_argument(
|
||||
"--sdist-only", action="store_true", help="Build only the source archive, without compiling a wheel"
|
||||
)
|
||||
args: Final = parser.parse_args()
|
||||
build_core_distribution(Path(args.out_dir), sdist_only=args.sdist_only)
|
||||
args: Final = parser.parse_args(namespace=BuildOptions())
|
||||
build_core_distribution(Path(args.out_dir), sdist_only=args.sdist_only, package=args.package)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -20,12 +20,21 @@ else:
|
|||
ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("package_cli_args", "package"),
|
||||
[
|
||||
(("--package", "litellm-proxy"), "litellm-proxy"),
|
||||
((), "litellm-core"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sdist_only", [False, True])
|
||||
def test_build_command_selects_requested_distributions(tmp_path: Path, sdist_only: bool) -> None:
|
||||
def test_build_command_selects_requested_distributions(
|
||||
tmp_path: Path, sdist_only: bool, package_cli_args: tuple[str, ...], package: str
|
||||
) -> None:
|
||||
from scripts.build_core_distribution import main
|
||||
|
||||
output: Final = tmp_path / "dist"
|
||||
arguments: Final = ["build_core_distribution.py", "--out-dir", str(output)] + (
|
||||
arguments: Final = ["build_core_distribution.py", *package_cli_args, "--out-dir", str(output)] + (
|
||||
["--sdist-only"] if sdist_only else []
|
||||
)
|
||||
with (
|
||||
|
|
@ -42,6 +51,7 @@ def test_build_command_selects_requested_distributions(tmp_path: Path, sdist_onl
|
|||
assert command[command.index("--out-dir") + 1] == str(output)
|
||||
assert run.call_args.kwargs["check"] is True
|
||||
assert run.call_args.kwargs["cwd"] == stage.call_args.args[1]
|
||||
assert stage.call_args.kwargs["package"] == package
|
||||
assert not stage.call_args.args[1].exists()
|
||||
|
||||
|
||||
|
|
@ -53,7 +63,10 @@ def source_repository(tmp_path: Path) -> Path:
|
|||
return source
|
||||
|
||||
|
||||
def test_staging_stamps_release_version_without_modifying_sources(tmp_path: Path, source_repository: Path) -> None:
|
||||
@pytest.mark.parametrize("package", ["litellm-core", "litellm-proxy"])
|
||||
def test_staging_stamps_release_version_without_modifying_sources(
|
||||
tmp_path: Path, source_repository: Path, package: str
|
||||
) -> None:
|
||||
from scripts.build_core_distribution import SOURCES, stage_core_distribution
|
||||
|
||||
source: Final = source_repository
|
||||
|
|
@ -61,16 +74,44 @@ def test_staging_stamps_release_version_without_modifying_sources(tmp_path: Path
|
|||
path: Final = source / name
|
||||
path.write_text(f"shared {name}")
|
||||
(source / "pyproject.toml").write_text('[project]\nname = "litellm"\nversion = "9.8.7rc1"\n')
|
||||
manifest: Final = source / "packaging/litellm-core/pyproject.toml"
|
||||
manifest: Final = source / "packaging" / package / "pyproject.toml"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_bytes((ROOT / "packaging/litellm-core/pyproject.toml").read_bytes())
|
||||
manifest.write_bytes((ROOT / "packaging" / package / "pyproject.toml").read_bytes())
|
||||
original: Final = manifest.read_bytes()
|
||||
stage: Final = tmp_path / "stage"
|
||||
stage_core_distribution(source, stage)
|
||||
assert tomllib.loads((stage / "pyproject.toml").read_text())["project"]["version"] == "9.8.7rc1"
|
||||
stage_core_distribution(source, stage, package=package)
|
||||
staged_project: Final = tomllib.loads((stage / "pyproject.toml").read_text())
|
||||
assert staged_project["project"]["name"] == package
|
||||
assert staged_project["project"]["version"] == "9.8.7rc1"
|
||||
if package == "litellm-proxy":
|
||||
assert staged_project["tool"]["maturin"]["targets"] == [{"name": "litellm-gateway", "kind": "bin"}]
|
||||
assert not (stage / "litellm").exists()
|
||||
assert all(
|
||||
(stage / name).exists()
|
||||
for name in ("litellm-rust", ".cargo", "rust-toolchain.toml", "model_prices_and_context_window.json")
|
||||
)
|
||||
assert manifest.read_bytes() == original
|
||||
assert (source / "pyproject.toml").read_text() == '[project]\nname = "litellm"\nversion = "9.8.7rc1"\n'
|
||||
assert all((stage / name).read_bytes() == (source / name).read_bytes() for name in SOURCES)
|
||||
expected: Final = tuple(name for name in SOURCES if package == "litellm-core" or name != "litellm")
|
||||
assert all((stage / name).read_bytes() == (source / name).read_bytes() for name in expected)
|
||||
|
||||
|
||||
def test_gateway_staging_refuses_manifest_without_in_tree_rust_path(tmp_path: Path, source_repository: Path) -> None:
|
||||
from scripts.build_core_distribution import SOURCES, stage_core_distribution
|
||||
|
||||
source: Final = source_repository
|
||||
for name in SOURCES:
|
||||
(source / name).write_text(f"shared {name}")
|
||||
(source / "pyproject.toml").write_text('[project]\nname = "litellm"\nversion = "9.8.7"\n')
|
||||
manifest: Final = source / "packaging" / "litellm-proxy" / "pyproject.toml"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_text(
|
||||
(ROOT / "packaging" / "litellm-proxy" / "pyproject.toml")
|
||||
.read_text()
|
||||
.replace("../../litellm-rust/", "../../rust/")
|
||||
)
|
||||
with pytest.raises(SystemExit, match="no longer points at"):
|
||||
stage_core_distribution(source, tmp_path / "stage", package="litellm-proxy")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
|
@ -99,7 +140,9 @@ def test_core_wheel_metadata_and_resources(distributions: tuple[Path, Path]) ->
|
|||
assert metadata["Version"] == tomllib.loads((ROOT / "pyproject.toml").read_text())["project"]["version"]
|
||||
assert not metadata.get_all("Provides-Extra")
|
||||
assert not any(n.endswith(".dist-info/entry_points.txt") for n in names)
|
||||
requirements: Final = tomllib.loads((ROOT / "packaging/litellm-core/pyproject.toml").read_text())["project"]["dependencies"]
|
||||
requirements: Final = tomllib.loads((ROOT / "packaging/litellm-core/pyproject.toml").read_text())["project"][
|
||||
"dependencies"
|
||||
]
|
||||
for python_version in ("3.10", "3.11", "3.12", "3.13", "3.14"):
|
||||
environment: Final = {"python_version": python_version, "python_full_version": python_version + ".0"}
|
||||
assert {
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ importlib-metadata: >=6.8.0 # Apache 2.0 License
|
|||
tokenizers: >=0.20.2 # Apache 2.0 License
|
||||
jinja2: >=3.1.4 # BSD 3-Clause License
|
||||
litellm-proxy-extras: >=0.1.1 # MIT License
|
||||
litellm-proxy: >=1.109.0 # MIT, verified in repository LICENSE for litellm-rust and packaging/litellm-proxy
|
||||
litellm-enterprise: >=0.1.1 # LiteLLM Enterprise License
|
||||
a2a-sdk: >=0.3.22 # Apache 2.0 license
|
||||
anyio: >=4.5.0 # Unknown license
|
||||
|
|
|
|||
|
|
@ -272,16 +272,32 @@ def assert_key_reported(
|
|||
date: str,
|
||||
metadata: Mapping[str, JsonValue],
|
||||
metrics: Mapping[str, float],
|
||||
*,
|
||||
entity_rows: Mapping[str, int] | None = None,
|
||||
) -> None:
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = object_value(response.json())
|
||||
records: Final = tuple(object_value(record) for record in records_of_key(body, api_key))
|
||||
assert records, response.text
|
||||
assert all(record["metadata"] == metadata for record in records), response.text
|
||||
assert all(counted(record["metrics"]) == pytest.approx(metrics) for record in records), response.text
|
||||
days: Final = body["results"]
|
||||
assert isinstance(days, list) and len(days) == 1, response.text
|
||||
day: Final = object_value(days[0])
|
||||
breakdown: Final = object_value(day["breakdown"])
|
||||
if entity_rows is None:
|
||||
assert all(counted(record["metrics"]) == pytest.approx(metrics) for record in records), response.text
|
||||
else:
|
||||
totals: Final = records_of_key({name: value for name, value in breakdown.items() if name != "entities"}, api_key)
|
||||
assert totals, response.text
|
||||
assert all(counted(object_value(record)["metrics"]) == pytest.approx(metrics) for record in totals), response.text
|
||||
entities: Final = object_value(breakdown["entities"])
|
||||
assert set(entities) == set(entity_rows), response.text
|
||||
for entity, rows in entity_rows.items():
|
||||
bucket: Final = object_value(entities[entity])
|
||||
assert counted(bucket["metrics"]) == pytest.approx(seeded_metrics(rows)), response.text
|
||||
keys: Final = object_value(bucket["api_key_breakdown"])
|
||||
assert set(keys) == {api_key}, response.text
|
||||
assert counted(object_value(keys[api_key])["metrics"]) == pytest.approx(seeded_metrics(rows)), response.text
|
||||
assert day["date"] == date, response.text
|
||||
assert counted(day["metrics"]) == pytest.approx(metrics), response.text
|
||||
assert object_value(body["metadata"])["total_spend"] == pytest.approx(metrics["spend"]), response.text
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from pydantic import JsonValue
|
|||
CHATGPT_AUTH_HOST: Final = "auth.openai.com"
|
||||
GITHUB_HOST: Final = "github.com"
|
||||
GITHUB_API_HOST: Final = "api.github.com"
|
||||
COPILOT_API_HOST: Final = "api.githubcopilot.com"
|
||||
AUTH_HOSTS: Final = (CHATGPT_AUTH_HOST, GITHUB_HOST, GITHUB_API_HOST)
|
||||
CHATGPT_ACCOUNT: Final = "acct-device-login"
|
||||
_FAR_FUTURE: Final = 4102444800
|
||||
|
|
@ -79,7 +80,17 @@ COPILOT_MINTED_KEY: Final = "tid=device-login;minted"
|
|||
COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected"
|
||||
CONTROL_KEY: Final = "device-login-control-key"
|
||||
ACCEPTED_BEARERS: Final = frozenset(
|
||||
{CHATGPT_STORED, CHATGPT_REFRESHED, CHATGPT_FIRST_LOGIN, COPILOT_STORED_KEY, COPILOT_MINTED_KEY, CONTROL_KEY}
|
||||
{
|
||||
CHATGPT_STORED,
|
||||
CHATGPT_REFRESHED,
|
||||
CHATGPT_FIRST_LOGIN,
|
||||
COPILOT_ACCESS,
|
||||
COPILOT_FIRST_LOGIN_ACCESS,
|
||||
COPILOT_UNSAVED_KEY_ACCESS,
|
||||
COPILOT_STORED_KEY,
|
||||
COPILOT_MINTED_KEY,
|
||||
CONTROL_KEY,
|
||||
}
|
||||
)
|
||||
REJECTED_DETAIL: Final = "the scripted provider rejected this bearer token"
|
||||
_DEVICE_AUTH_ID: Final = "device-auth-synthetic"
|
||||
|
|
@ -180,6 +191,10 @@ def _auth_hosts(api_url: str, grant: threading.Event) -> Callable[[Request], Rep
|
|||
return _json(200, {"access_token": COPILOT_FIRST_LOGIN_ACCESS, "token_type": "bearer"})
|
||||
case ("api.github.com", "GET", "/copilot_internal/v2/token"):
|
||||
return _copilot_api_key(request, api_url)
|
||||
case ("api.github.com", "GET", "/copilot_internal/user"):
|
||||
if request.headers.get("authorization") == f"token {COPILOT_REJECTED_ACCESS}":
|
||||
return _json(401, {"message": "Bad credentials"})
|
||||
return _json(200, {"endpoints": {"api": f"https://{COPILOT_API_HOST}"}})
|
||||
case _:
|
||||
return _switched_off()
|
||||
|
||||
|
|
@ -204,9 +219,22 @@ class Switches:
|
|||
switch.clear()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProviderAPI:
|
||||
plain: Wire
|
||||
secure: Wire
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.plain.url
|
||||
|
||||
def drain(self) -> tuple[Request, ...]:
|
||||
return (*self.plain.drain(), *self.secure.drain())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Peers:
|
||||
api: Wire
|
||||
api: _ProviderAPI
|
||||
auth: Wire
|
||||
tunnel: str
|
||||
cert: Path
|
||||
|
|
@ -228,6 +256,14 @@ class Peers:
|
|||
def auth_connections(self) -> tuple[str, ...]:
|
||||
return tuple(self.authorities.get_nowait() for _ in range(self.authorities.qsize()))
|
||||
|
||||
def assert_only_copilot_discovery(self, access_token: str) -> None:
|
||||
assert set(self.auth_connections()) <= {f"{GITHUB_API_HOST}:443"}
|
||||
requests: Final = self.auth.drain()
|
||||
assert {
|
||||
(request.method, request.target, request.headers.get("host"), request.headers.get("authorization"))
|
||||
for request in requests
|
||||
} <= {("GET", "/copilot_internal/user", GITHUB_API_HOST, f"token {access_token}")}, requests
|
||||
|
||||
def dropped(self) -> tuple[Hangup, ...]:
|
||||
return tuple(self.hangups.get_nowait() for _ in range(self.hangups.qsize()))
|
||||
|
||||
|
|
@ -279,9 +315,8 @@ class _TunnelServer(socketserver.ThreadingTCPServer):
|
|||
|
||||
@contextmanager
|
||||
def _tunnel(
|
||||
destination: Wire, switches: Switches, authorities: SimpleQueue[str], hangups: SimpleQueue[Hangup]
|
||||
destination: Wire, provider: Wire, switches: Switches, authorities: SimpleQueue[str], hangups: SimpleQueue[Hangup]
|
||||
) -> Generator[str]:
|
||||
destination_port: Final = int(destination.url.rsplit(":", 1)[1])
|
||||
|
||||
class Tunnel(socketserver.StreamRequestHandler):
|
||||
rbufsize = 0
|
||||
|
|
@ -306,20 +341,24 @@ def _tunnel(
|
|||
if len(request_line) < 2:
|
||||
return
|
||||
authority: Final = request_line[1]
|
||||
if authority.rsplit(":", 1)[0] not in AUTH_HOSTS:
|
||||
host: Final = authority.rsplit(":", 1)[0]
|
||||
if host not in (*AUTH_HOSTS, COPILOT_API_HOST):
|
||||
self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n")
|
||||
return
|
||||
authorities.put(authority)
|
||||
if switches.refuse_connect.is_set():
|
||||
is_auth: Final = host in AUTH_HOSTS
|
||||
if is_auth:
|
||||
authorities.put(authority)
|
||||
if is_auth and switches.refuse_connect.is_set():
|
||||
self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\ncontent-length: 0\r\n\r\n")
|
||||
return
|
||||
if self.hold(authority, opened, switches.hold_connect) is None:
|
||||
if is_auth and self.hold(authority, opened, switches.hold_connect) is None:
|
||||
return
|
||||
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||
client_hello: Final = self.hold(authority, opened, switches.stall_tls)
|
||||
client_hello: Final = self.hold(authority, opened, switches.stall_tls) if is_auth else b""
|
||||
if client_hello is None:
|
||||
return
|
||||
self.request.settimeout(10)
|
||||
destination_port: Final = int((destination if is_auth else provider).url.rsplit(":", 1)[1])
|
||||
with socket.create_connection(("127.0.0.1", destination_port), timeout=10) as upstream:
|
||||
upstream.sendall(client_hello)
|
||||
outbound: Final = threading.Thread(target=_pipe, args=(self.request, upstream))
|
||||
|
|
@ -340,16 +379,17 @@ def _tunnel(
|
|||
|
||||
@contextmanager
|
||||
def device_login_peers(directory: Path) -> Generator[Peers]:
|
||||
cert, key = write_self_signed_cert(directory, AUTH_HOSTS)
|
||||
cert, key = write_self_signed_cert(directory, (*AUTH_HOSTS, COPILOT_API_HOST))
|
||||
switches: Final = Switches(threading.Event(), threading.Event(), threading.Event(), threading.Event())
|
||||
authorities: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
hangups: Final[SimpleQueue[Hangup]] = SimpleQueue()
|
||||
with (
|
||||
wire_server(_provider_api(rv.ResponsesVendor())) as api,
|
||||
wire_server(_provider_api(rv.ResponsesVendor()), tls=server_context(cert, key)) as copilot_api,
|
||||
wire_server(_auth_hosts(api.url, switches.grant), tls=server_context(cert, key)) as auth,
|
||||
_tunnel(auth, switches, authorities, hangups) as tunnel,
|
||||
_tunnel(auth, copilot_api, switches, authorities, hangups) as tunnel,
|
||||
):
|
||||
yield Peers(api, auth, tunnel, cert, authorities, hangups, switches)
|
||||
yield Peers(_ProviderAPI(api, copilot_api), auth, tunnel, cert, authorities, hangups, switches)
|
||||
|
||||
|
||||
def chatgpt_record(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,114 @@
|
|||
"""GET, compact, DELETE and cancel on /v1/responses answer 200 when a during_call guardrail is on by default
|
||||
(LIT-9770): the hook used to raise `'acompact_responses' is not a valid CallTypes` and the proxy returned 500"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, string_value
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration._support.wire import Reply
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_MODEL: Final = "openai/gpt-5.4-mini"
|
||||
|
||||
|
||||
def _input() -> list[JsonValue]:
|
||||
return [
|
||||
{"role": "user", "content": f"Remember the number {uuid.uuid4().hex}."},
|
||||
{"role": "assistant", "content": "Got it."},
|
||||
]
|
||||
|
||||
|
||||
def _response(response_id: str) -> Reply:
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"id": response_id,
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": _MODEL.removeprefix("openai/"),
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": f"msg_{uuid.uuid4().hex}",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Got it.", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 12, "output_tokens": 5, "total_tokens": 17},
|
||||
}
|
||||
return Reply(body=json.dumps(body).encode())
|
||||
|
||||
|
||||
def _deleted(response_id: str) -> Reply:
|
||||
return Reply(body=json.dumps({"id": response_id, "object": "response.deleted", "deleted": True}).encode())
|
||||
|
||||
|
||||
def _guarded_config(path: Path) -> Path:
|
||||
base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
guardrail: Final[dict[str, JsonValue]] = {
|
||||
"guardrail_name": f"content-filter-{uuid.uuid4().hex}",
|
||||
"litellm_params": {
|
||||
"guardrail": "litellm_content_filter",
|
||||
"mode": "during_call",
|
||||
"default_on": True,
|
||||
"blocked_words": [{"keyword": "forbiddenword", "action": "BLOCK"}],
|
||||
},
|
||||
}
|
||||
path.write_text(yaml.safe_dump({**base, "guardrails": [guardrail]}))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def guarded(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
directory: Final = tmp_path_factory.mktemp("during-call-guardrail")
|
||||
with owned_proxy(session_gateway, directory, {}, config=_guarded_config(directory / "config.yaml")) as candidate:
|
||||
yield candidate
|
||||
|
||||
|
||||
def _create(guarded: Gateway, provider: SharedProvider, upstream_id: str) -> str:
|
||||
provider.expect(_response(upstream_id))
|
||||
created: Final = guarded.request("POST", "/v1/responses", {"model": _MODEL, "input": _input(), "store": True})
|
||||
assert created.status_code == 200, created.text
|
||||
(upstream,) = provider.received()
|
||||
assert upstream.target.endswith("/responses"), upstream.target
|
||||
return string_value(_JSON_OBJECT.validate_json(created.content)["id"])
|
||||
|
||||
|
||||
def _lifecycle(guarded: Gateway, provider: SharedProvider, operation: str) -> tuple[httpx.Response, str]:
|
||||
if operation == "compact":
|
||||
provider.expect(_response(f"resp_{uuid.uuid4().hex}"))
|
||||
return guarded.request(
|
||||
"POST", "/v1/responses/compact", {"model": _MODEL, "input": _input()}
|
||||
), "/responses/compact"
|
||||
upstream_id: Final = f"resp_{uuid.uuid4().hex}"
|
||||
managed_id: Final = _create(guarded, provider, upstream_id)
|
||||
provider.expect(_deleted(upstream_id) if operation == "delete" else _response(upstream_id))
|
||||
if operation == "get":
|
||||
return guarded.request("GET", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}"
|
||||
if operation == "delete":
|
||||
return guarded.request("DELETE", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}"
|
||||
return guarded.request("POST", f"/v1/responses/{managed_id}/cancel"), f"/responses/{upstream_id}/cancel"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["get", "compact", "delete", "cancel"])
|
||||
def test_responses_lifecycle_answers_200_with_a_during_call_guardrail_on(
|
||||
guarded: Gateway, provider: SharedProvider, operation: str
|
||||
) -> None:
|
||||
response, upstream_target = _lifecycle(guarded, provider, operation)
|
||||
assert response.status_code == 200, response.text
|
||||
(upstream,) = provider.received()
|
||||
assert upstream.target.endswith(upstream_target), upstream.target
|
||||
|
|
@ -134,6 +134,8 @@ async def test_approval_commits_version_archive_history_and_review(pending_updat
|
|||
assert current is not None
|
||||
published_manifest: Final = {
|
||||
**case.proposal.manifest,
|
||||
"_litellm_review": {"approved_by": "integration-reviewer", "security_scan_passed": False},
|
||||
"_litellm_published_by": "integration-reviewer",
|
||||
"source": {
|
||||
**TypeAdapter(dict[str, str]).validate_python(case.submission.proposal.manifest["source"]),
|
||||
"url": f"https://skills.example.test/claude-code/plugins/{case.original.name}/archive/"
|
||||
|
|
@ -148,17 +150,22 @@ async def test_approval_commits_version_archive_history_and_review(pending_updat
|
|||
)
|
||||
metadata: Final = await case.publications.get_metadata(case.original.name)
|
||||
assert metadata is not None
|
||||
assert metadata.created_by == before.created_by == case.original.created_by
|
||||
assert metadata.created_at == before.created_at
|
||||
assert metadata.updated_at is not None and before.updated_at is not None
|
||||
assert metadata.updated_at > before.updated_at
|
||||
assert await case.publications.get_content(metadata) == case.proposal.content
|
||||
history: Final = await case.publications.versions(metadata, 10, 0)
|
||||
assert tuple((version.version, version.description, version.archive_available) for version in history) == (
|
||||
("1.0.1", case.proposal.description, True),
|
||||
("1.0.0", case.original.description, True),
|
||||
)
|
||||
assert history[0].published_by == "integration-reviewer"
|
||||
assert history[1].published_by == case.original.created_by
|
||||
for version, publication, manifest in zip(
|
||||
history, (case.proposal, case.original), (published_manifest, case.original.manifest), strict=True
|
||||
history, (case.proposal, case.original), (published_manifest, {**case.original.manifest, "_litellm_published_by": case.original.created_by}), strict=True
|
||||
):
|
||||
assert version.manifest == {**manifest, "version": version.version}
|
||||
assert version.published_by == publication.created_by
|
||||
assert await case.publications.version_content(metadata, version.id) == publication.content
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from collections.abc import Mapping
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -248,7 +249,7 @@ def test_duplicate_json_key_keeps_the_last_value(gateway: Gateway) -> None:
|
|||
assert stored_manifest(name)["installation_preference"] == "available"
|
||||
|
||||
|
||||
def test_repeated_identical_update_is_idempotent(gateway: Gateway) -> None:
|
||||
def test_repeated_identical_updates_publish_patch_versions_without_changing_preference(gateway: Gateway) -> None:
|
||||
name: Final = plugin_name()
|
||||
with gateway.scenario() as scenario:
|
||||
registered(scenario, name, installation_preference="required")
|
||||
|
|
@ -257,7 +258,9 @@ def test_repeated_identical_update_is_idempotent(gateway: Gateway) -> None:
|
|||
replaced(gateway, name, installation_preference="required")
|
||||
second: Final = stored_rows(name)
|
||||
assert len(first) == 1, first
|
||||
assert second == first
|
||||
first_manifest: Final = JSON_OBJECT.validate_json(string_value(first[0]["manifest_json"]))
|
||||
assert first_manifest["version"] == "1.0.1", first
|
||||
assert second == [{**first[0], "manifest_json": json.dumps({**first_manifest, "version": "1.0.2"})}]
|
||||
assert_preference_served(gateway, name, "required")
|
||||
|
||||
|
||||
|
|
@ -442,14 +445,16 @@ def test_burst_reads_across_processes_while_the_preference_flips(gateway: Gatewa
|
|||
name: Final = plugin_name()
|
||||
targets: Final = (gateway, peer)
|
||||
flips: Final = tuple(PREFERENCES[(index + 1) % len(PREFERENCES)] for index in range(13))
|
||||
writer: Final = Lock()
|
||||
with gateway.scenario() as scenario:
|
||||
registered(scenario, name, installation_preference="available")
|
||||
|
||||
def job(index: int) -> tuple[str, JsonValue]:
|
||||
if index % 4 == 3:
|
||||
return "flip", replace(
|
||||
gateway, name, replacement(name, installation_preference=flips[index // 4])
|
||||
).status_code
|
||||
with writer:
|
||||
return "flip", replace(
|
||||
gateway, name, replacement(name, installation_preference=flips[index // 4])
|
||||
).status_code
|
||||
return "read", marketplace_entry(targets[index % 2], name).get("installationPreference")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ from openai.types.chat import ChatCompletionMessageParam
|
|||
from openai.types.responses.tool_param import Mcp
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
|
||||
pytestmark: Final = [pytest.mark.xdist_group("mcp_llm_private_gateway"), pytest.mark.shares_worker_state]
|
||||
|
||||
|
||||
|
|
@ -589,7 +588,11 @@ def _upstream_model(surface: Surface) -> str:
|
|||
}[surface]
|
||||
|
||||
|
||||
PRICES: Final[Mapping[str, JsonValue]] = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "num_retries": 0}
|
||||
PRICES: Final[Mapping[str, JsonValue]] = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"num_retries": 0,
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
@ -917,7 +920,9 @@ def test_an_open_server_in_another_gateway_cannot_widen_this_gateways_tools(gate
|
|||
register_mcp(scenario, local, "open" + uuid.uuid4().hex[:8], allow_all_keys=True)
|
||||
listed, called = official_client_outcomes(shared, key, "/mcp", f"{foreign_alias}-add", ADD)
|
||||
assert set(listed.tools) == {
|
||||
f"{foreign_alias}-add", f"{foreign_alias}-multiply", f"{foreign_alias}-fail"
|
||||
f"{foreign_alias}-add",
|
||||
f"{foreign_alias}-multiply",
|
||||
f"{foreign_alias}-fail",
|
||||
}, listed.tools
|
||||
assert called.ok and called.text == str(ADD["a"] + ADD["b"]), called
|
||||
assert [call["body"]["params"]["name"] for call in _peer_add_calls(foreign)] == ["add"]
|
||||
|
|
@ -1528,6 +1533,7 @@ def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_
|
|||
one_round_rows: Final = _model_rows(
|
||||
one_round.headers["x-litellm-call-id"], _logs_a_failure_billing(_price(FIRST_ROUND) + FIXED_MARGIN)
|
||||
)
|
||||
assert _statuses(one_round_rows) == ["failure"], one_round_rows
|
||||
rig.wire.drain()
|
||||
response: Final = rig.send(key, [AUTO], stream=True, guardrails=[STREAM_BLOCK])
|
||||
assert STREAM_BLOCK_DETAIL in response.text, response.text
|
||||
|
|
|
|||
|
|
@ -67,9 +67,9 @@ class _TokenDirs:
|
|||
def environment(self) -> Mapping[str, str]:
|
||||
return MappingProxyType({"CHATGPT_TOKEN_DIR": str(self.chatgpt), "GITHUB_COPILOT_TOKEN_DIR": str(self.copilot)})
|
||||
|
||||
def store_valid_tokens(self, api_url: str) -> None:
|
||||
def store_valid_tokens(self) -> None:
|
||||
dl.write_chatgpt(self.chatgpt, dl.chatgpt_record(dl.CHATGPT_STORED))
|
||||
dl.write_copilot_key(self.copilot, dl.COPILOT_STORED_KEY, api_url)
|
||||
dl.write_copilot_access(self.copilot, dl.COPILOT_ACCESS)
|
||||
|
||||
|
||||
def _token_dirs(directory: Path) -> _TokenDirs:
|
||||
|
|
@ -242,16 +242,16 @@ def test_booting_without_tokens_lists_login_deployments_and_serves_only_after_to
|
|||
assert peers.auth_connections() == ()
|
||||
assert dl.bearer(peers.api.drain()[-1]) == dl.CONTROL_KEY
|
||||
|
||||
dirs.store_valid_tokens(peers.api.url)
|
||||
dirs.store_valid_tokens()
|
||||
_reload(owned)
|
||||
_every_worker_lists(owned, workers)
|
||||
for served_logins, name in enumerate(_LOGIN_NAMES, start=1):
|
||||
served: Final = _chat(_base_url(owned), key, name, uuid.uuid4().hex)
|
||||
assert served.status == 200, served.text
|
||||
_successes_logged(key, at_least=_CONTROL_CALLS_BEFORE_THE_RELOAD + served_logins)
|
||||
assert peers.auth_connections() == ()
|
||||
peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS)
|
||||
forwarded: Final = peers.api.drain()
|
||||
assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_STORED, dl.COPILOT_STORED_KEY}
|
||||
assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_STORED, dl.COPILOT_ACCESS}
|
||||
|
||||
dl.write_chatgpt(dirs.chatgpt, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.GOOD_REFRESH))
|
||||
peers.switches.hold_connect.set()
|
||||
|
|
@ -287,7 +287,7 @@ def test_a_token_removed_after_boot_turns_every_request_into_the_refusal(
|
|||
) -> None:
|
||||
dirs: Final = _token_dirs(tmp_path)
|
||||
with dl.device_login_peers(tmp_path) as peers:
|
||||
dirs.store_valid_tokens(peers.api.url)
|
||||
dirs.store_valid_tokens()
|
||||
with owned_proxy_process(
|
||||
gateway,
|
||||
tmp_path,
|
||||
|
|
@ -310,7 +310,7 @@ def test_a_token_removed_after_boot_turns_every_request_into_the_refusal(
|
|||
refused: Final = _chat(_base_url(owned), key, name, uuid.uuid4().hex)
|
||||
assert refused.status == 400, refused.text
|
||||
assert refusal in refused.message(), refused.text
|
||||
assert peers.auth_connections() == ()
|
||||
peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS)
|
||||
tail: Final = owned.log.read_text()
|
||||
assert dl.CHATGPT_USER_CODE not in tail and dl.COPILOT_USER_CODE not in tail, tail[-3000:]
|
||||
control: Final = _chat(_base_url(owned), key, _CONTROL, uuid.uuid4().hex)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ import asyncio
|
|||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
|
@ -26,9 +25,10 @@ from integration._support.database import read_rows
|
|||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.timing import assert_cut_short, timed
|
||||
from integration._support.wire import Request
|
||||
from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS
|
||||
|
||||
pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
|
||||
|
||||
Provider: TypeAlias = Literal["chatgpt", "copilot"]
|
||||
|
|
@ -52,7 +52,7 @@ _REFUSALS: Final[Mapping[Provider, str]] = MappingProxyType(
|
|||
{"chatgpt": dl.CHATGPT_REFUSAL, "copilot": dl.COPILOT_REFUSAL}
|
||||
)
|
||||
_STORED_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType(
|
||||
{"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_STORED_KEY}
|
||||
{"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_ACCESS}
|
||||
)
|
||||
_CONTROL_MODEL: Final = "device-login-control"
|
||||
_EXTRA: Final[Mapping[str, JsonValue]] = MappingProxyType({"num_retries": 0, "cache": {"no-cache": True}})
|
||||
|
|
@ -154,7 +154,7 @@ class _Scene:
|
|||
if provider == "chatgpt":
|
||||
dl.write_chatgpt(self.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_STORED))
|
||||
return
|
||||
dl.write_copilot_key(self.rig.copilot_dir, dl.COPILOT_STORED_KEY, self.rig.peers.api.url)
|
||||
dl.write_copilot_access(self.rig.copilot_dir, dl.COPILOT_ACCESS)
|
||||
|
||||
|
||||
def _config(peers: dl.Peers, directory: Path) -> Path:
|
||||
|
|
@ -403,7 +403,7 @@ def test_missing_token_is_refused_with_the_login_instructions_before_any_auth_ho
|
|||
|
||||
|
||||
@pytest.mark.parametrize("cell", _CELLS, ids=_Cell.name)
|
||||
def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene, cell: _Cell) -> None:
|
||||
def test_stored_token_serves_the_request_without_a_device_login(scene: _Scene, cell: _Cell) -> None:
|
||||
scene.store_valid_token(cell.provider)
|
||||
call: Final = scene.call(cell)
|
||||
served: Final = _serve(call)
|
||||
|
|
@ -412,7 +412,10 @@ def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene
|
|||
received: Final = scene.rig.peers.api.drain()
|
||||
(forwarded,) = _carrying(received, call.marker)
|
||||
assert dl.bearer(forwarded) == _STORED_BEARERS[cell.provider], _describe(received)
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
if cell.provider == "copilot":
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS)
|
||||
else:
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
(row,) = _spend_rows(scene.key)
|
||||
assert row["status"] == "success", row
|
||||
|
||||
|
|
@ -487,7 +490,7 @@ def test_missing_token_is_refused_on_the_other_model_routes(scene: _Scene, route
|
|||
assert _describe(scene.rig.peers.api.drain()) == ()
|
||||
|
||||
|
||||
def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) -> None:
|
||||
def test_copilot_embeddings_are_served_with_the_stored_access_token(scene: _Scene) -> None:
|
||||
scene.store_valid_token("copilot")
|
||||
served: Final = scene.post(
|
||||
"/v1/embeddings", {"model": "github_copilot/text-embedding-3-small", "input": "login", **_EXTRA}
|
||||
|
|
@ -496,9 +499,9 @@ def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) ->
|
|||
(item,) = rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(served.text)["data"])
|
||||
assert item["embedding"] == [0.25, 0.5, 0.75], served.text
|
||||
(forwarded,) = scene.rig.peers.api.drain()
|
||||
assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_STORED_KEY), forwarded.target
|
||||
assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_ACCESS), forwarded.target
|
||||
assert forwarded.target.endswith("/embeddings"), forwarded.target
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", _PROVIDERS)
|
||||
|
|
@ -562,71 +565,67 @@ def test_stored_token_the_provider_rejects_surfaces_its_401_without_a_login(
|
|||
if provider == "chatgpt":
|
||||
dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_REJECTED))
|
||||
else:
|
||||
dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url)
|
||||
dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS)
|
||||
call: Final = scene.call(_Cell(provider, endpoint, False, "httpx"))
|
||||
served: Final = _serve(call)
|
||||
assert served.status == 401, served.text
|
||||
assert dl.REJECTED_DETAIL in served.text, served.text
|
||||
received: Final = scene.rig.peers.api.drain()
|
||||
assert {dl.bearer(request) for request in _carrying(received, call.marker)} == {
|
||||
dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_KEY
|
||||
dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_ACCESS
|
||||
}, _describe(received)
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
if provider == "copilot":
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_REJECTED_ACCESS)
|
||||
else:
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stale_key", (False, True), ids=("missing-api-key", "expired-api-key"))
|
||||
def test_copilot_api_key_is_minted_once_from_the_stored_access_token(scene: _Scene, stale_key: bool) -> None:
|
||||
def test_copilot_access_token_is_forwarded_without_minting_an_api_key(scene: _Scene, stale_key: bool) -> None:
|
||||
dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_ACCESS)
|
||||
if stale_key:
|
||||
dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url, -3600)
|
||||
legacy_key: Final = scene.rig.copilot_dir / "api-key.json"
|
||||
stored_key: Final = legacy_key.read_bytes() if stale_key else None
|
||||
call: Final = scene.call(_Cell("copilot", "chat", False, "httpx"))
|
||||
served: Final = _serve(call)
|
||||
assert served.status == 200, served.text
|
||||
assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text
|
||||
(minted,) = scene.rig.peers.auth.drain()
|
||||
assert (minted.method, minted.target, minted.headers.get("host"), minted.headers.get("authorization")) == (
|
||||
"GET",
|
||||
"/copilot_internal/v2/token",
|
||||
dl.GITHUB_API_HOST,
|
||||
f"token {dl.COPILOT_ACCESS}",
|
||||
)
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS)
|
||||
(forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker)
|
||||
assert dl.bearer(forwarded) == dl.COPILOT_MINTED_KEY
|
||||
assert json.loads((scene.rig.copilot_dir / "api-key.json").read_text())["token"] == dl.COPILOT_MINTED_KEY
|
||||
assert dl.bearer(forwarded) == dl.COPILOT_ACCESS
|
||||
assert (legacy_key.read_bytes() if legacy_key.exists() else None) == stored_key
|
||||
|
||||
|
||||
def test_copilot_api_key_that_cannot_be_saved_is_minted_at_most_once_per_worker(scene: _Scene) -> None:
|
||||
def test_copilot_access_token_serves_when_the_legacy_api_key_path_is_a_directory(scene: _Scene) -> None:
|
||||
dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_UNSAVED_KEY_ACCESS)
|
||||
unwritable_key_file: Final = scene.rig.copilot_dir / "api-key.json"
|
||||
unwritable_key_file.mkdir()
|
||||
first: Final = scene.call(_Cell("copilot", "chat", False, "httpx"))
|
||||
first_served: Final = _serve(first)
|
||||
assert first_served.status == 200, first_served.text
|
||||
(minted,) = scene.rig.peers.auth.drain()
|
||||
assert minted.headers.get("authorization") == f"token {dl.COPILOT_UNSAVED_KEY_ACCESS}"
|
||||
later: Final = tuple(scene.call(_Cell("copilot", "chat", False, "httpx")) for _ in range(3))
|
||||
later_served: Final = tuple(_serve(call) for call in later)
|
||||
assert [served.status for served in later_served] == [200, 200, 200], [served.text for served in later_served]
|
||||
assert len(scene.rig.peers.auth.drain()) <= 1
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_UNSAVED_KEY_ACCESS)
|
||||
received: Final = scene.rig.peers.api.drain()
|
||||
forwarded: Final = tuple(_carrying(received, call.marker) for call in (first, *later))
|
||||
assert [tuple(dl.bearer(request) for request in requests) for requests in forwarded] == [
|
||||
(dl.COPILOT_MINTED_KEY,)
|
||||
(dl.COPILOT_UNSAVED_KEY_ACCESS,)
|
||||
] * 4, _describe(received)
|
||||
assert unwritable_key_file.is_dir()
|
||||
|
||||
|
||||
def test_copilot_access_token_github_rejects_fails_the_request_without_a_device_login(scene: _Scene) -> None:
|
||||
dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS)
|
||||
served: Final = _serve(scene.call(_Cell("copilot", "chat", False, "httpx")))
|
||||
assert served.status == 400, served.text
|
||||
assert "Failed to refresh API key" in _error_message(served.text), served.text
|
||||
call: Final = scene.call(_Cell("copilot", "chat", False, "httpx"))
|
||||
served: Final = _serve(call)
|
||||
assert served.status == 401, served.text
|
||||
assert dl.REJECTED_DETAIL in _error_message(served.text), served.text
|
||||
_failure_logged(served)
|
||||
reached: Final = scene.rig.peers.auth.drain()
|
||||
assert {(request.method, request.target) for request in reached} == {("GET", "/copilot_internal/v2/token")}, (
|
||||
_describe(reached)
|
||||
)
|
||||
assert _describe(scene.rig.peers.api.drain()) == ()
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_REJECTED_ACCESS)
|
||||
(forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker)
|
||||
assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_REJECTED_ACCESS)
|
||||
|
||||
|
||||
def test_expired_copilot_api_key_with_no_access_token_is_refused_before_any_auth_host_call(scene: _Scene) -> None:
|
||||
|
|
@ -727,7 +726,10 @@ def test_a_token_mounted_after_a_refusal_serves_without_a_restart_and_removing_i
|
|||
after: Final = _serve(scene.call(cell))
|
||||
assert after.status == 400, after.text
|
||||
assert _REFUSALS[provider] in _error_message(after.text), after.text
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
if provider == "copilot":
|
||||
scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS)
|
||||
else:
|
||||
assert scene.rig.peers.auth_connections() == ()
|
||||
|
||||
|
||||
_COOLDOWN_SECONDS_LEFT: Final = 45
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -147,6 +146,7 @@ def test_interaction_forwards_native_body_to_gemini(gateway: Gateway, prefix: st
|
|||
"agent": None,
|
||||
"updated": None,
|
||||
"outputs": None,
|
||||
"continuation_token": None,
|
||||
}, response.text
|
||||
assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text
|
||||
|
||||
|
|
@ -179,6 +179,7 @@ def test_interaction_authenticates_google_key_header(gateway: Gateway, prefix: s
|
|||
"agent": None,
|
||||
"updated": None,
|
||||
"outputs": None,
|
||||
"continuation_token": None,
|
||||
}, response.text
|
||||
assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text
|
||||
|
||||
|
|
@ -218,6 +219,7 @@ def test_agent_interaction_uses_env_gemini_credentials(gateway: Gateway, tmp_pat
|
|||
"agent": "deep-research-pro-preview-12-2025",
|
||||
"updated": None,
|
||||
"outputs": None,
|
||||
"continuation_token": None,
|
||||
}, response.text
|
||||
assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text
|
||||
|
||||
|
|
@ -277,3 +279,78 @@ def test_stream_interaction_ends_without_openai_done_terminator(gateway: Gateway
|
|||
expected: Final = b"".join(_proxy_stream_event(payload, model) for _event_type, payload in _STREAM_EVENTS)
|
||||
assert response.content == expected, response.text
|
||||
assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions?alt=sse")]
|
||||
|
||||
|
||||
_LONG_DECODE_MODEL: Final = "gemini/gemini-4-argon"
|
||||
_LONG_DECODE_BODY: Final = {
|
||||
"input": "prove the prime number theorem",
|
||||
"generation_config": {"max_output_tokens": 120000, "thinking_level": "high"},
|
||||
}
|
||||
|
||||
|
||||
def _hop(hop_id: str, status: str, text: str, token: str | None, input_tokens: int, output_tokens: int) -> dict:
|
||||
return {
|
||||
"id": hop_id,
|
||||
"object": "interaction",
|
||||
"model": "gemini-4-argon",
|
||||
"status": status,
|
||||
"created": "2026-11-05T00:00:00Z",
|
||||
"steps": [{"type": "model_output", "content": [{"type": "text", "text": text}]}],
|
||||
"usage": {
|
||||
"total_input_tokens": input_tokens,
|
||||
"total_output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
"initial_input_tokens": 79,
|
||||
},
|
||||
"continuation_token": token,
|
||||
}
|
||||
|
||||
|
||||
_FIRST_HOP: Final = _hop("interaction-hop-1", "requires_continuation", "Lemma 1.", "tok-1", 79, 32768)
|
||||
_SECOND_HOP: Final = _hop("interaction-hop-2", "completed", " QED.", None, 32847, 1533)
|
||||
|
||||
|
||||
def test_interaction_keeps_requires_continuation_status_and_token_for_the_caller(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.target == "/v1beta/interactions"
|
||||
return Reply(body=json.dumps(_FIRST_HOP).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_LONG_DECODE_MODEL, api_base=wire.url, api_key=_GEMINI_DEPLOYMENT_KEY)
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1beta/interactions", {"model": model, **_LONG_DECODE_BODY}, key=key
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
sent: Final = [json.loads(r.body) for r in wire.drain()]
|
||||
|
||||
assert sent == [{"model": "gemini-4-argon", **_LONG_DECODE_BODY}], sent
|
||||
payload: Final = response.json()
|
||||
assert payload["id"] == "interaction-hop-1", response.text
|
||||
assert payload["status"] == "requires_continuation", response.text
|
||||
assert payload["continuation_token"] == "tok-1", response.text
|
||||
assert payload["usage"]["initial_input_tokens"] == 79, response.text
|
||||
|
||||
|
||||
def test_interaction_forwards_a_caller_continuation_token_to_resume_a_long_decode(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.target == "/v1beta/interactions"
|
||||
return Reply(body=json.dumps(_SECOND_HOP).encode())
|
||||
|
||||
resume: Final = {
|
||||
"generation_config": _LONG_DECODE_BODY["generation_config"],
|
||||
"previous_interaction_id": "interaction-hop-1",
|
||||
"continuation_token": "tok-1",
|
||||
}
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_LONG_DECODE_MODEL, api_base=wire.url, api_key=_GEMINI_DEPLOYMENT_KEY)
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = gateway.request("POST", "/v1beta/interactions", {"model": model, **resume}, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
sent: Final = [json.loads(r.body) for r in wire.drain()]
|
||||
|
||||
assert sent == [{"model": "gemini-4-argon", **resume}], sent
|
||||
payload: Final = response.json()
|
||||
assert payload["id"] == "interaction-hop-2", response.text
|
||||
assert payload["status"] == "completed", response.text
|
||||
assert payload["continuation_token"] is None, response.text
|
||||
|
|
|
|||
|
|
@ -299,14 +299,14 @@ def cooldown_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Cooldown
|
|||
pytest.param("chat", _BRIDGED, "pause_after_content", id="chat-bridged-to-responses"),
|
||||
),
|
||||
)
|
||||
def test_a_deployment_whose_stream_stalls_stays_out_of_cooldown(
|
||||
def test_a_deployment_whose_stream_stalls_is_cooled_down(
|
||||
cooldown_rig: _CooldownRig, endpoint: Endpoint, backend: str, shape: Shape
|
||||
) -> None:
|
||||
with _peer(shape) as wire, cooldown_rig.gateway.scenario() as scenario:
|
||||
group, identity = _deployment(scenario, wire, backend)
|
||||
response: Final = _streamed(cooldown_rig.gateway, endpoint, group)
|
||||
error: Final = _failure_row(cooldown_rig.gateway, _ROWS, (group,))
|
||||
assert not cooled(cooldown_rig.cache, identity), (error, response.text)
|
||||
assert eventually(lambda: cooled(cooldown_rig.cache, identity), bool, _ROW_SECONDS), (error, response.text)
|
||||
|
||||
|
||||
@_OWNED_PROXY
|
||||
|
|
|
|||
|
|
@ -61,10 +61,10 @@ _PROVIDERS: Final = MappingProxyType(
|
|||
)
|
||||
_STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream"))
|
||||
_NON_BOOLEAN_FLAGS: Final = (
|
||||
pytest.param(1, True, id="int"),
|
||||
pytest.param("", False, id="empty-string"),
|
||||
pytest.param([], False, id="list"),
|
||||
pytest.param("x" * 5120, True, id="five-kilobytes"),
|
||||
pytest.param(1, id="int"),
|
||||
pytest.param("", id="empty-string"),
|
||||
pytest.param([], id="list"),
|
||||
pytest.param("x" * 5120, id="five-kilobytes"),
|
||||
)
|
||||
_OPENAI_UNAUTHORIZED: Final = Reply(
|
||||
status=401,
|
||||
|
|
@ -326,10 +326,8 @@ def test_default_gateway_keeps_the_flag_off_the_wire_on_a_fallback(
|
|||
assert observed.attempted == "1", observed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS)
|
||||
def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire(
|
||||
registered: _Registered, value: JsonValue, errors_reported: bool
|
||||
) -> None:
|
||||
@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS)
|
||||
def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire(registered: _Registered, value: JsonValue) -> None:
|
||||
sent: Final = _send(
|
||||
registered.gateway,
|
||||
_PATHS["chat"],
|
||||
|
|
@ -497,9 +495,9 @@ def test_exposed_chat_without_a_fallback_keeps_the_flag_off_the_wire_and_the_err
|
|||
assert observed.errors_header is None, observed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS)
|
||||
def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_header_follows_its_truthiness(
|
||||
exposed: _Exposed, value: JsonValue, errors_reported: bool
|
||||
@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS)
|
||||
def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_reports_no_errors(
|
||||
exposed: _Exposed, value: JsonValue
|
||||
) -> None:
|
||||
sent: Final = _send(
|
||||
exposed.proxy,
|
||||
|
|
@ -515,7 +513,7 @@ def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_heade
|
|||
assert observed.leaks == (), observed
|
||||
assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed
|
||||
assert observed.attempted == "1", observed
|
||||
assert _errors_mention_the_outage(observed.errors_header) is errors_reported, observed
|
||||
assert observed.errors_header is None, observed
|
||||
|
||||
|
||||
def test_exposed_proxy_rejects_an_unauthenticated_flagged_request_before_the_wire(exposed: _Exposed) -> None:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
import collections
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import eventually, gateway_from_environment, object_value
|
||||
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.redis_process import owned_redis
|
||||
from integration._support.wire import Reply, Wire, wire_server
|
||||
from integration.routing._prompt_cache_pin_support import (
|
||||
DEFAULT_MARK,
|
||||
anthropic_deployment,
|
||||
ask_turn,
|
||||
chat_body,
|
||||
claude_peer,
|
||||
new_marker,
|
||||
one_new_pin,
|
||||
pin_config,
|
||||
post_chat,
|
||||
read_pins,
|
||||
system_turn,
|
||||
)
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from redis import Redis
|
||||
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 300)
|
||||
|
||||
_GROUP: Final = "pin-logging-off"
|
||||
_LEFT: Final = "pin-logging-off-left"
|
||||
_RIGHT: Final = "pin-logging-off-right"
|
||||
_FOLLOW_UPS: Final = 20
|
||||
_REDACTED_MESSAGES: Final = [{"role": "user", "content": REDACTED_BY_LITELLM}]
|
||||
_LOGGED_EVENT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
owned: OwnedProxy
|
||||
redis: "Redis[bytes]"
|
||||
logger_sink: Wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("prompt-cache-pin-message-logging-off")
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_redis(directory) as cache,
|
||||
wire_server(claude_peer) as left,
|
||||
wire_server(claude_peer) as right,
|
||||
wire_server(lambda request: Reply()) as logger_sink,
|
||||
):
|
||||
config: Final = pin_config(
|
||||
directory,
|
||||
[anthropic_deployment(_GROUP, _LEFT, left.url), anthropic_deployment(_GROUP, _RIGHT, right.url)],
|
||||
litellm_settings={
|
||||
"turn_off_message_logging": True,
|
||||
"callbacks": ["integration._support.redaction_probe.probe"],
|
||||
"cache": False,
|
||||
"enable_redis_auth_cache": False,
|
||||
},
|
||||
)
|
||||
environment: Final = {
|
||||
"REDIS_HOST": cache.host,
|
||||
"REDIS_PORT": str(cache.port),
|
||||
"REDACTION_PROBE_URL": logger_sink.url,
|
||||
}
|
||||
with (
|
||||
owned_proxy_process(gateway, directory, environment, config=config, workers=1) as owned,
|
||||
Redis(host=cache.host, port=cache.port, socket_timeout=2) as redis,
|
||||
):
|
||||
yield _Rig(owned, redis, logger_sink)
|
||||
|
||||
|
||||
def test_follow_ups_stay_on_the_first_deployment_while_the_logger_sees_redacted_messages(rig: _Rig) -> None:
|
||||
before: Final = frozenset(read_pins(rig.redis))
|
||||
body: Final = chat_body(_GROUP, [system_turn(new_marker(), DEFAULT_MARK), ask_turn(new_marker())])
|
||||
|
||||
_, first_served = post_chat(rig.owned.gateway, body)
|
||||
_, pin = one_new_pin(rig.redis, before)
|
||||
assert pin.model_id == first_served, (pin, first_served)
|
||||
|
||||
served: Final = tuple(post_chat(rig.owned.gateway, body)[1] for _ in range(_FOLLOW_UPS))
|
||||
assert served == (first_served,) * _FOLLOW_UPS, collections.Counter(served)
|
||||
|
||||
eventually(rig.logger_sink.received.qsize, lambda count: count >= 1 + _FOLLOW_UPS, seconds=30)
|
||||
logged: Final = tuple(
|
||||
object_value(_LOGGED_EVENT.validate_json(request.body)["standard"])["messages"]
|
||||
for request in rig.logger_sink.drain()
|
||||
)
|
||||
assert len(logged) == 1 + _FOLLOW_UPS, len(logged)
|
||||
assert all(messages == _REDACTED_MESSAGES for messages in logged), logged
|
||||
|
|
@ -184,7 +184,7 @@ def clean(bench: _Bench) -> Iterator[_Bench]:
|
|||
|
||||
|
||||
_FIRST_LOGIN_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType(
|
||||
{"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_MINTED_KEY}
|
||||
{"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_FIRST_LOGIN_ACCESS}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -205,7 +205,7 @@ def test_sync_call_on_the_main_thread_keeps_the_interactive_first_login(
|
|||
assert {dl.bearer(request) for request in forwarded} == {_FIRST_LOGIN_BEARERS[provider]}, [
|
||||
(request.method, request.target) for request in forwarded
|
||||
]
|
||||
expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token", "api-key.json")
|
||||
expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token",)
|
||||
assert clean.token_files() == expected_files
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -36,11 +36,11 @@ _BURST: Final = 12
|
|||
_CALLBACK_WINDOW: Final = 15.0
|
||||
_STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream"))
|
||||
_NON_BOOLEAN_FLAGS: Final = (
|
||||
pytest.param(1, True, id="int"),
|
||||
pytest.param("", False, id="empty-string"),
|
||||
pytest.param([], False, id="list"),
|
||||
pytest.param("x" * 5120, True, id="five-kilobytes"),
|
||||
pytest.param(False, False, id="false"),
|
||||
pytest.param(1, id="int"),
|
||||
pytest.param("", id="empty-string"),
|
||||
pytest.param([], id="list"),
|
||||
pytest.param("x" * 5120, id="five-kilobytes"),
|
||||
pytest.param(False, id="false"),
|
||||
)
|
||||
_UNAUTHORIZED: Final = Reply(
|
||||
status=401,
|
||||
|
|
@ -263,10 +263,8 @@ def test_without_a_fallback_the_flag_still_stays_off_the_wire() -> None:
|
|||
assert outcome.errors == (), outcome
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS)
|
||||
def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its_truthiness(
|
||||
value: object, errors_reported: bool
|
||||
) -> None:
|
||||
@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS)
|
||||
def test_a_flag_other_than_true_stays_off_the_wire_and_reports_no_errors(value: object) -> None:
|
||||
with wire_server(_peer) as wire:
|
||||
router: Final = _router(wire)
|
||||
outcome: Final = _outcome(wire, _complete(router, "primary", stream=False, include_fallback_errors=value))
|
||||
|
|
@ -274,7 +272,7 @@ def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its
|
|||
assert outcome.hit == ("primary", "backup"), outcome
|
||||
assert outcome.text == f"{_ANSWER} backup", outcome
|
||||
assert outcome.attempted == 1, outcome
|
||||
assert (len(outcome.errors) == 1) is errors_reported, outcome
|
||||
assert outcome.errors == (), outcome
|
||||
|
||||
|
||||
class _Recorder(CustomLogger):
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -57,6 +57,7 @@ def test_key_whose_daily_spend_names_two_users_is_reported_with_no_owner(gateway
|
|||
DAY,
|
||||
key_metadata(),
|
||||
seeded_metrics(2),
|
||||
entity_rows={first: 1, second: 1},
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -74,6 +75,7 @@ def test_daily_spend_rows_naming_no_user_do_not_hide_the_one_user_the_others_nam
|
|||
DAY,
|
||||
key_metadata(user=owner, email=email),
|
||||
seeded_metrics(2),
|
||||
entity_rows={owner: 1, "Unassigned": 1},
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ def test_key_stops_being_reported_with_an_owner_once_a_second_user_spends_with_i
|
|||
with daily_rows((user_row(second, api_key, DAY),)):
|
||||
shared: Final = activity_of_key(gateway, AGGREGATED_USER_ACTIVITY, api_key)
|
||||
assert_key_reported(alone, api_key, DAY, key_metadata(user=first, email=email), seeded_metrics(1))
|
||||
assert_key_reported(shared, api_key, DAY, key_metadata(), seeded_metrics(2))
|
||||
assert_key_reported(shared, api_key, DAY, key_metadata(), seeded_metrics(2), entity_rows={first: 1, second: 1})
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
|
|
|
|||
|
|
@ -756,7 +756,7 @@
|
|||
{
|
||||
"label": "list",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -834,12 +834,12 @@
|
|||
{
|
||||
"label": "create",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n"
|
||||
},
|
||||
{
|
||||
"label": "copy",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -967,12 +967,12 @@
|
|||
{
|
||||
"label": "list_files",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n"
|
||||
},
|
||||
{
|
||||
"label": "get_file",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1036,7 +1036,7 @@
|
|||
{
|
||||
"label": "delete",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -1093,7 +1093,7 @@
|
|||
{
|
||||
"label": "get",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1171,7 +1171,7 @@
|
|||
"deep_research": {
|
||||
"summary": "Deep Research",
|
||||
"value": {
|
||||
"agent": "deep-research-pro-preview-12-2025",
|
||||
"agent": "deep-research-preview-04-2026",
|
||||
"created": "2025-11-26T12:22:47Z",
|
||||
"id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg",
|
||||
"object": "interaction",
|
||||
|
|
@ -1472,7 +1472,7 @@
|
|||
{
|
||||
"label": "deep_research",
|
||||
"lang": "sh",
|
||||
"source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-pro-preview-12-2025\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n"
|
||||
"source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-preview-04-2026\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n"
|
||||
},
|
||||
{
|
||||
"label": "antigravity",
|
||||
|
|
@ -1517,7 +1517,7 @@
|
|||
{
|
||||
"label": "deep_research",
|
||||
"lang": "python",
|
||||
"source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-pro-preview-12-2025\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n"
|
||||
"source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-preview-04-2026\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n"
|
||||
},
|
||||
{
|
||||
"label": "antigravity",
|
||||
|
|
@ -1562,7 +1562,7 @@
|
|||
{
|
||||
"label": "deep_research",
|
||||
"lang": "javascript",
|
||||
"source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-pro-preview-12-2025',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n"
|
||||
"source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-preview-04-2026',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n"
|
||||
},
|
||||
{
|
||||
"label": "antigravity",
|
||||
|
|
@ -1607,7 +1607,7 @@
|
|||
{
|
||||
"label": "deep_research",
|
||||
"lang": "java",
|
||||
"source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-pro-preview-12-2025\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n"
|
||||
"source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-preview-04-2026\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n"
|
||||
},
|
||||
{
|
||||
"label": "antigravity",
|
||||
|
|
@ -1652,7 +1652,7 @@
|
|||
{
|
||||
"label": "deep_research",
|
||||
"lang": "go",
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-pro-preview-12-2025\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n"
|
||||
"source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-preview-04-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n"
|
||||
},
|
||||
{
|
||||
"label": "antigravity",
|
||||
|
|
@ -3041,6 +3041,22 @@
|
|||
"description": "An agent definition for the CreateAgent API.\nThis message is the target for annotation-parser-based JSON parsing.\nNew format:\n {\n \"id\": \"customer-sentinel\",\n \"base_agent\": \"\",\n \"system_instruction\": \"...\",\n \"base_environment\": { \"type\": \"remote\", \"sources\": [...] },\n \"tools\": [ {\"type\": \"code_execution\"} ]\n }",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"description": "The unique identifier for the agent.",
|
||||
"type": "string"
|
||||
},
|
||||
"base_agent": {
|
||||
"description": "The base agent to extend.",
|
||||
"type": "string"
|
||||
},
|
||||
"system_instruction": {
|
||||
"description": "System instruction for the agent.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Agent description for developers to quickly read and understand.",
|
||||
"type": "string"
|
||||
},
|
||||
"agent_config": {
|
||||
"description": "Configuration parameters for the agent.",
|
||||
"oneOf": [
|
||||
|
|
@ -3060,22 +3076,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"description": "The unique identifier for the agent.",
|
||||
"type": "string"
|
||||
},
|
||||
"base_agent": {
|
||||
"description": "The base agent to extend.",
|
||||
"type": "string"
|
||||
},
|
||||
"system_instruction": {
|
||||
"description": "System instruction for the agent.",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"description": "Agent description for developers to quickly read and understand.",
|
||||
"type": "string"
|
||||
},
|
||||
"tools": {
|
||||
"description": "The tools available to the agent.",
|
||||
"type": "array",
|
||||
|
|
@ -3091,13 +3091,11 @@
|
|||
"description": "The agent to interact with.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"deep-research-pro-preview-12-2025",
|
||||
"deep-research-preview-04-2026",
|
||||
"deep-research-max-preview-04-2026",
|
||||
"antigravity-preview-05-2026"
|
||||
],
|
||||
"x-speakeasy-enum-descriptions": [
|
||||
"Gemini Deep Research Agent",
|
||||
"Gemini Deep Research Agent",
|
||||
"Gemini Deep Research Max Agent",
|
||||
"Use the Antigravity managed agent to perform multi-step tasks that require reasoning, file operations, and tool use."
|
||||
|
|
@ -3330,14 +3328,6 @@
|
|||
"type": {
|
||||
"const": "audio"
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
|
|
@ -3371,6 +3361,14 @@
|
|||
"WEBM audio format"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"rate": {
|
||||
"description": "Deprecated. Use sample_rate instead. The value is ignored.",
|
||||
"type": "integer",
|
||||
|
|
@ -3817,32 +3815,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"response_format": {
|
||||
"description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
},
|
||||
{
|
||||
"title": "ResponseFormatList",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"environment": {
|
||||
"description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/EnvironmentConfig"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"description": "Required. Output only. A unique identifier for the interaction completion.",
|
||||
"type": "string",
|
||||
|
|
@ -3858,7 +3830,8 @@
|
|||
"failed",
|
||||
"cancelled",
|
||||
"incomplete",
|
||||
"budget_exceeded"
|
||||
"budget_exceeded",
|
||||
"requires_continuation"
|
||||
],
|
||||
"readOnly": true,
|
||||
"x-google-enum-deprecated": [
|
||||
|
|
@ -3868,7 +3841,8 @@
|
|||
false,
|
||||
false,
|
||||
false,
|
||||
true
|
||||
true,
|
||||
false
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"The interaction is in progress.",
|
||||
|
|
@ -3877,7 +3851,8 @@
|
|||
"The interaction failed.",
|
||||
"The interaction was cancelled.",
|
||||
"The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).",
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)."
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).",
|
||||
"The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request."
|
||||
]
|
||||
},
|
||||
"created": {
|
||||
|
|
@ -3909,6 +3884,21 @@
|
|||
"$ref": "#/components/schemas/ResponseModality"
|
||||
}
|
||||
},
|
||||
"response_format": {
|
||||
"description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
},
|
||||
{
|
||||
"title": "ResponseFormatList",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"response_mime_type": {
|
||||
"description": "The mime type of the response. This is required if response_format is set.",
|
||||
"type": "string",
|
||||
|
|
@ -3918,6 +3908,17 @@
|
|||
"description": "The ID of the previous interaction, if any.",
|
||||
"type": "string"
|
||||
},
|
||||
"environment": {
|
||||
"description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/EnvironmentConfig"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
"environment_id": {
|
||||
"description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.",
|
||||
"type": "string",
|
||||
|
|
@ -3945,6 +3946,12 @@
|
|||
"type": "string"
|
||||
}
|
||||
},
|
||||
"continuation_token": {
|
||||
"description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"stream": {
|
||||
"description": "Input only. Whether the interaction will be streamed.",
|
||||
"type": "boolean",
|
||||
|
|
@ -4018,32 +4025,6 @@
|
|||
"$ref": "#/components/schemas/GenerationConfig",
|
||||
"writeOnly": true
|
||||
},
|
||||
"response_format": {
|
||||
"description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
},
|
||||
{
|
||||
"title": "ResponseFormatList",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"environment": {
|
||||
"description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/EnvironmentConfig"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"description": "Required. Output only. A unique identifier for the interaction completion.",
|
||||
"type": "string",
|
||||
|
|
@ -4059,7 +4040,8 @@
|
|||
"failed",
|
||||
"cancelled",
|
||||
"incomplete",
|
||||
"budget_exceeded"
|
||||
"budget_exceeded",
|
||||
"requires_continuation"
|
||||
],
|
||||
"readOnly": true,
|
||||
"x-google-enum-deprecated": [
|
||||
|
|
@ -4069,7 +4051,8 @@
|
|||
false,
|
||||
false,
|
||||
false,
|
||||
true
|
||||
true,
|
||||
false
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"The interaction is in progress.",
|
||||
|
|
@ -4078,7 +4061,8 @@
|
|||
"The interaction failed.",
|
||||
"The interaction was cancelled.",
|
||||
"The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).",
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)."
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).",
|
||||
"The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request."
|
||||
]
|
||||
},
|
||||
"created": {
|
||||
|
|
@ -4110,6 +4094,21 @@
|
|||
"$ref": "#/components/schemas/ResponseModality"
|
||||
}
|
||||
},
|
||||
"response_format": {
|
||||
"description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
},
|
||||
{
|
||||
"title": "ResponseFormatList",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ResponseFormat"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"response_mime_type": {
|
||||
"description": "The mime type of the response. This is required if response_format is set.",
|
||||
"type": "string",
|
||||
|
|
@ -4119,6 +4118,17 @@
|
|||
"description": "The ID of the previous interaction, if any.",
|
||||
"type": "string"
|
||||
},
|
||||
"environment": {
|
||||
"description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/EnvironmentConfig"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
"environment_id": {
|
||||
"description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.",
|
||||
"type": "string",
|
||||
|
|
@ -4146,6 +4156,12 @@
|
|||
"type": "string"
|
||||
}
|
||||
},
|
||||
"continuation_token": {
|
||||
"description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"stream": {
|
||||
"description": "Input only. Whether the interaction will be streamed.",
|
||||
"type": "boolean",
|
||||
|
|
@ -4348,11 +4364,31 @@
|
|||
"type": "string",
|
||||
"enum": [
|
||||
"application/pdf",
|
||||
"text/csv"
|
||||
"text/csv",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.oasis.opendocument.spreadsheet"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"PDF document format",
|
||||
"CSV document format"
|
||||
"CSV document format",
|
||||
"PPTX presentation format",
|
||||
"PPT presentation format",
|
||||
"ODP presentation format",
|
||||
"DOCX document format",
|
||||
"DOC document format",
|
||||
"DOTX document template format",
|
||||
"ODT document format",
|
||||
"XLSX spreadsheet format",
|
||||
"XLS spreadsheet format",
|
||||
"ODS spreadsheet format"
|
||||
]
|
||||
}
|
||||
},
|
||||
|
|
@ -4379,6 +4415,37 @@
|
|||
"type": {
|
||||
"const": "document"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"application/pdf",
|
||||
"text/csv",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.oasis.opendocument.spreadsheet"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"PDF document format",
|
||||
"CSV document format",
|
||||
"PPTX presentation format",
|
||||
"PPT presentation format",
|
||||
"ODP presentation format",
|
||||
"DOCX document format",
|
||||
"DOC document format",
|
||||
"DOTX document template format",
|
||||
"ODT document format",
|
||||
"XLSX spreadsheet format",
|
||||
"XLS spreadsheet format",
|
||||
"ODS spreadsheet format"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
|
|
@ -4386,17 +4453,6 @@
|
|||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"application/pdf",
|
||||
"text/csv"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"PDF document format",
|
||||
"CSV document format"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
@ -4474,23 +4530,6 @@
|
|||
"type": "string",
|
||||
"readOnly": true
|
||||
},
|
||||
"network": {
|
||||
"description": "Network configuration for the environment.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"disabled"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"All network egress is blocked."
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"sources": {
|
||||
"description": "Sources to be mounted into the environment.",
|
||||
"type": "array",
|
||||
|
|
@ -4526,6 +4565,23 @@
|
|||
""
|
||||
]
|
||||
},
|
||||
"network": {
|
||||
"description": "Network configuration for the environment.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"disabled"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"All network egress is blocked."
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"file_count": {
|
||||
"description": "Output only. The number of files in the environment, output only.",
|
||||
"type": "string",
|
||||
|
|
@ -4551,6 +4607,12 @@
|
|||
"type": {
|
||||
"const": "remote"
|
||||
},
|
||||
"sources": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/Source"
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"description": "Network configuration for the environment.",
|
||||
"oneOf": [
|
||||
|
|
@ -4568,12 +4630,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"sources": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/Source"
|
||||
}
|
||||
},
|
||||
"environment_id": {
|
||||
"description": "Optional. The environment ID for the interaction. If specified, the request will\nupdate the existing environment instead of creating a new one.",
|
||||
"type": "string"
|
||||
|
|
@ -5347,30 +5403,6 @@
|
|||
"description": "Configuration parameters for model interactions.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tool_choice": {
|
||||
"description": "The tool choice configuration.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ToolChoiceConfig"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"auto",
|
||||
"any",
|
||||
"none",
|
||||
"validated"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"Auto tool choice.",
|
||||
"Any tool choice.",
|
||||
"No tool choice.",
|
||||
"Validated tool choice."
|
||||
],
|
||||
"x-speakeasy-model-namespace": "interactions"
|
||||
}
|
||||
]
|
||||
},
|
||||
"temperature": {
|
||||
"description": "Controls the randomness of the output.",
|
||||
"type": "number",
|
||||
|
|
@ -5395,6 +5427,30 @@
|
|||
"type": "string"
|
||||
}
|
||||
},
|
||||
"tool_choice": {
|
||||
"description": "The tool choice configuration.",
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ToolChoiceConfig"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"auto",
|
||||
"any",
|
||||
"none",
|
||||
"validated"
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"Auto tool choice.",
|
||||
"Any tool choice.",
|
||||
"No tool choice.",
|
||||
"Validated tool choice."
|
||||
],
|
||||
"x-speakeasy-model-namespace": "interactions"
|
||||
}
|
||||
]
|
||||
},
|
||||
"thinking_level": {
|
||||
"description": "The level of thought tokens that the model should generate.",
|
||||
"$ref": "#/components/schemas/ThinkingLevel"
|
||||
|
|
@ -6211,14 +6267,6 @@
|
|||
"type": {
|
||||
"const": "image"
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
|
|
@ -6242,6 +6290,14 @@
|
|||
"TIFF image format"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"resolution": {
|
||||
"description": "The resolution of the media.",
|
||||
"$ref": "#/components/schemas/MediaResolution"
|
||||
|
|
@ -6385,7 +6441,8 @@
|
|||
"failed",
|
||||
"cancelled",
|
||||
"incomplete",
|
||||
"budget_exceeded"
|
||||
"budget_exceeded",
|
||||
"requires_continuation"
|
||||
],
|
||||
"readOnly": true,
|
||||
"x-google-enum-deprecated": [
|
||||
|
|
@ -6395,7 +6452,8 @@
|
|||
false,
|
||||
false,
|
||||
false,
|
||||
true
|
||||
true,
|
||||
false
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"The interaction is in progress.",
|
||||
|
|
@ -6404,7 +6462,8 @@
|
|||
"The interaction failed.",
|
||||
"The interaction was cancelled.",
|
||||
"The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).",
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)."
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).",
|
||||
"The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request."
|
||||
]
|
||||
},
|
||||
"created": {
|
||||
|
|
@ -6434,9 +6493,33 @@
|
|||
"$ref": "#/components/schemas/GenerationConfig",
|
||||
"writeOnly": true
|
||||
},
|
||||
"system_instruction": {
|
||||
"description": "System instruction for the interaction.",
|
||||
"type": "string"
|
||||
},
|
||||
"tools": {
|
||||
"description": "A list of tool declarations the model may call during interaction.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/Tool"
|
||||
}
|
||||
},
|
||||
"input": {
|
||||
"$ref": "#/components/schemas/InteractionsInput"
|
||||
},
|
||||
"usage": {
|
||||
"description": "Output only. Statistics on the interaction request's token usage.",
|
||||
"$ref": "#/components/schemas/Usage",
|
||||
"readOnly": true
|
||||
},
|
||||
"response_modalities": {
|
||||
"description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).",
|
||||
"type": "array",
|
||||
"deprecated": true,
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ResponseModality"
|
||||
}
|
||||
},
|
||||
"response_format": {
|
||||
"description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.",
|
||||
"oneOf": [
|
||||
|
|
@ -6452,6 +6535,15 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"response_mime_type": {
|
||||
"description": "The mime type of the response. This is required if response_format is set.",
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"previous_interaction_id": {
|
||||
"description": "The ID of the previous interaction, if any.",
|
||||
"type": "string"
|
||||
},
|
||||
"environment": {
|
||||
"description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.",
|
||||
"oneOf": [
|
||||
|
|
@ -6463,39 +6555,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"system_instruction": {
|
||||
"description": "System instruction for the interaction.",
|
||||
"type": "string"
|
||||
},
|
||||
"tools": {
|
||||
"description": "A list of tool declarations the model may call during interaction.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/Tool"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"description": "Output only. Statistics on the interaction request's token usage.",
|
||||
"$ref": "#/components/schemas/Usage",
|
||||
"readOnly": true
|
||||
},
|
||||
"response_modalities": {
|
||||
"description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).",
|
||||
"type": "array",
|
||||
"deprecated": true,
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ResponseModality"
|
||||
}
|
||||
},
|
||||
"response_mime_type": {
|
||||
"description": "The mime type of the response. This is required if response_format is set.",
|
||||
"type": "string",
|
||||
"deprecated": true
|
||||
},
|
||||
"previous_interaction_id": {
|
||||
"description": "The ID of the previous interaction, if any.",
|
||||
"type": "string"
|
||||
},
|
||||
"environment_id": {
|
||||
"description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.",
|
||||
"type": "string",
|
||||
|
|
@ -6531,6 +6590,12 @@
|
|||
},
|
||||
"readOnly": true
|
||||
},
|
||||
"continuation_token": {
|
||||
"description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.",
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"stream": {
|
||||
"description": "Input only. Whether the interaction will be streamed.",
|
||||
"type": "boolean",
|
||||
|
|
@ -6813,16 +6878,16 @@
|
|||
},
|
||||
"readOnly": true
|
||||
},
|
||||
"agent": {
|
||||
"description": "The agent to interact with.",
|
||||
"type": "string"
|
||||
},
|
||||
"continuation_token": {
|
||||
"description": "Opaque token to resume a long decode.\nOutput: set when status is INCOMPLETE and decoding can be resumed.\nInput: pass the latest token back unchanged in CreateInteraction to\ncontinue decoding.",
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"agent": {
|
||||
"description": "The agent to interact with.",
|
||||
"type": "string"
|
||||
},
|
||||
"object": {
|
||||
"description": "Output only. The resource type.",
|
||||
"type": "string",
|
||||
|
|
@ -6863,7 +6928,8 @@
|
|||
"failed",
|
||||
"cancelled",
|
||||
"incomplete",
|
||||
"budget_exceeded"
|
||||
"budget_exceeded",
|
||||
"requires_continuation"
|
||||
],
|
||||
"x-google-enum-deprecated": [
|
||||
false,
|
||||
|
|
@ -6872,7 +6938,8 @@
|
|||
false,
|
||||
false,
|
||||
false,
|
||||
true
|
||||
true,
|
||||
false
|
||||
],
|
||||
"x-google-enum-descriptions": [
|
||||
"The interaction is in progress.",
|
||||
|
|
@ -6881,7 +6948,8 @@
|
|||
"The interaction failed.",
|
||||
"The interaction was cancelled.",
|
||||
"The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).",
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)."
|
||||
"Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).",
|
||||
"The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request."
|
||||
]
|
||||
},
|
||||
"event_type": {
|
||||
|
|
@ -7076,14 +7144,14 @@
|
|||
"type": {
|
||||
"const": "mcp_server"
|
||||
},
|
||||
"url": {
|
||||
"description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"",
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"description": "The name of the MCPServer.",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"",
|
||||
"type": "string"
|
||||
},
|
||||
"headers": {
|
||||
"description": "Optional: Fields for authentication headers, timeouts, etc., if needed.",
|
||||
"type": "object",
|
||||
|
|
@ -7362,7 +7430,7 @@
|
|||
},
|
||||
"ModelOption": {
|
||||
"title": "Model",
|
||||
"description": "The model that will complete your prompt.\\n\\nSee [models](https://ai.google.dev/gemini-api/docs/models) for additional details.",
|
||||
"description": "The model that will complete your prompt.\\n\\nSee [models](https://aistudio.google.com/docs/models) for additional details.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"gemma-4-26b-a4b-it",
|
||||
|
|
@ -8695,6 +8763,20 @@
|
|||
"description": "Configuration for speech recognition (transcription).",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"language_codes": {
|
||||
"description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"custom_vocabulary": {
|
||||
"description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"mode": {
|
||||
"description": "Discriminated transcription mode options or enum.",
|
||||
"oneOf": [
|
||||
|
|
@ -8714,20 +8796,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"language_codes": {
|
||||
"description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"custom_vocabulary": {
|
||||
"description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"timestamp_granularities": {
|
||||
"description": "Optional. The granularity of timestamps to include in the transcription output.\nSupported values: \"word\". If empty, no timestamps are generated.",
|
||||
"type": "array",
|
||||
|
|
@ -9332,6 +9400,11 @@
|
|||
"type": "integer",
|
||||
"format": "int32"
|
||||
},
|
||||
"initial_input_tokens": {
|
||||
"description": "Number of input tokens used in the first turn of this interaction.",
|
||||
"type": "integer",
|
||||
"format": "int32"
|
||||
},
|
||||
"input_tokens_by_modality": {
|
||||
"description": "A breakdown of input token usage by modality.",
|
||||
"type": "array",
|
||||
|
|
@ -9497,6 +9570,10 @@
|
|||
"description": "The URI of the video.",
|
||||
"type": "string"
|
||||
},
|
||||
"resolution": {
|
||||
"description": "The resolution of the media.",
|
||||
"$ref": "#/components/schemas/MediaResolution"
|
||||
},
|
||||
"processing": {
|
||||
"description": "How the model processes this video for understanding.",
|
||||
"oneOf": [
|
||||
|
|
@ -9516,10 +9593,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"resolution": {
|
||||
"description": "The resolution of the media.",
|
||||
"$ref": "#/components/schemas/MediaResolution"
|
||||
},
|
||||
"name": {
|
||||
"description": "A user-defined name for this content block. Can be referenced by the model\nin the final response.",
|
||||
"type": "string"
|
||||
|
|
@ -9573,14 +9646,6 @@
|
|||
"type": {
|
||||
"const": "video"
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
|
|
@ -9608,6 +9673,14 @@
|
|||
"JPEG 2000 video format"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"format": "byte",
|
||||
"x-speakeasy-base64-input-mode": "file"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string"
|
||||
},
|
||||
"resolution": {
|
||||
"description": "The resolution of the media.",
|
||||
"$ref": "#/components/schemas/MediaResolution"
|
||||
|
|
|
|||
|
|
@ -559,28 +559,34 @@ class TestInteractionsAcreateOffline:
|
|||
|
||||
@pytest.mark.usefixtures("fake_provider_credentials")
|
||||
@pytest.mark.asyncio
|
||||
async def test_acreate_gemini_keeps_continuation_token_out_of_the_request(self, respx_mock: MockRouter) -> None:
|
||||
async def test_acreate_gemini_sends_a_caller_continuation_token_to_resume_a_decode(
|
||||
self, respx_mock: MockRouter
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "interaction-offline",
|
||||
"object": "interaction",
|
||||
"model": "gemini-3.8-flash",
|
||||
"model": "gemini-4-argon",
|
||||
"status": "completed",
|
||||
"steps": [{"type": "model_output", "content": [{"type": "text", "text": "Hi"}]}],
|
||||
"usage": {"input_tokens": 6, "output_tokens": 3},
|
||||
"usage": {"total_input_tokens": 6, "total_output_tokens": 3},
|
||||
},
|
||||
)
|
||||
)
|
||||
response: Final = await interactions.acreate(
|
||||
model="gemini/gemini-3.8-flash",
|
||||
input="Prove it",
|
||||
model="gemini/gemini-4-argon",
|
||||
previous_interaction_id="interaction-hop-1",
|
||||
continuation_token="dG9rLTE=",
|
||||
api_key="gemini-offline",
|
||||
)
|
||||
# The Interactions API answered 400 "Unknown parameter 'continuation_token'." on 2026-10-13
|
||||
assert json.loads(route.calls.last.request.content) == {"model": "gemini-3.8-flash", "input": "Prove it"}
|
||||
# Google's spec (2026-11-05) takes continuation_token on CreateModelInteractionParams; a 2026-10-13 400 is gone
|
||||
assert json.loads(route.calls.last.request.content) == {
|
||||
"model": "gemini-4-argon",
|
||||
"previous_interaction_id": "interaction-hop-1",
|
||||
"continuation_token": "dG9rLTE=",
|
||||
}
|
||||
assert response.status == "completed"
|
||||
|
||||
@pytest.mark.usefixtures("fake_provider_credentials")
|
||||
|
|
@ -746,9 +752,7 @@ def test_create_with_system_instruction_for_google_and_responses(
|
|||
def test_create_streaming_returns_interaction_events(
|
||||
respx_mock: MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(
|
||||
"https://generativelanguage.googleapis.com/v1beta/interactions?alt=sse"
|
||||
).mock(
|
||||
route: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/interactions?alt=sse").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
|
|
@ -856,3 +860,65 @@ def test_create_streaming_hands_completed_interaction_to_success_callback(
|
|||
)
|
||||
|
||||
assert recorder.logged == (_GEMINI_COMPLETED_INTERACTION,)
|
||||
|
||||
|
||||
_GEMINI_URL: Final = "https://generativelanguage.googleapis.com/v1beta/interactions"
|
||||
_LONG_DECODE_REQUEST: Final = {
|
||||
"model": "gemini/gemini-4-argon",
|
||||
"input": "Prove the prime number theorem.",
|
||||
"generation_config": {"max_output_tokens": 120000, "thinking_level": "high"},
|
||||
}
|
||||
_FIRST_HOP: Final = {
|
||||
"id": "hop-1",
|
||||
"object": "interaction",
|
||||
"model": "gemini-4-argon",
|
||||
"status": "requires_continuation",
|
||||
"steps": [{"type": "model_output", "content": [{"type": "text", "text": "Lemma 1."}]}],
|
||||
"usage": {
|
||||
"total_input_tokens": 79,
|
||||
"total_output_tokens": 32668,
|
||||
"total_thought_tokens": 100,
|
||||
"total_tokens": 32847,
|
||||
"initial_input_tokens": 79,
|
||||
},
|
||||
"continuation_token": "tok-1",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("fake_provider_credentials")
|
||||
def test_create_keeps_requires_continuation_status_and_token_for_the_caller(respx_mock: MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_GEMINI_URL).mock(return_value=httpx.Response(200, json=_FIRST_HOP))
|
||||
|
||||
response: Final = interactions.create(**_LONG_DECODE_REQUEST, api_key="gemini-offline")
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.status == "requires_continuation"
|
||||
assert response.continuation_token == "tok-1"
|
||||
assert response.usage == _FIRST_HOP["usage"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("fake_provider_credentials")
|
||||
def test_create_sends_a_caller_continuation_token_to_resume_a_long_decode(respx_mock: MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_GEMINI_URL).mock(
|
||||
return_value=httpx.Response(
|
||||
200, json={**_FIRST_HOP, "id": "hop-2", "status": "completed", "continuation_token": None}
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = interactions.create(
|
||||
model="gemini/gemini-4-argon",
|
||||
previous_interaction_id="hop-1",
|
||||
continuation_token="tok-1",
|
||||
generation_config=_LONG_DECODE_REQUEST["generation_config"],
|
||||
api_key="gemini-offline",
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent == {
|
||||
"model": "gemini-4-argon",
|
||||
"previous_interaction_id": "hop-1",
|
||||
"continuation_token": "tok-1",
|
||||
"generation_config": _LONG_DECODE_REQUEST["generation_config"],
|
||||
}, sent
|
||||
assert response.status == "completed"
|
||||
assert response.continuation_token is None
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
OpenAPI compliance tests for Google Interactions API.
|
||||
|
||||
Validates that our SDK requests/responses match the OpenAPI spec snapshot in
|
||||
interactions.openapi.json, taken from https://ai.google.dev/static/api/interactions.openapi.json
|
||||
interactions.openapi.json, taken from https://aistudio.google.com/api-reference/interactions.openapi.json
|
||||
(litellm/types/interactions/README.md says how to refresh it). The scheduled
|
||||
interactions-spec-canary workflow runs the same tests against the live spec by passing
|
||||
--interactions-spec-url, so a change Google makes shows up there, not in every PR.
|
||||
|
|
@ -23,7 +23,7 @@ import pytest
|
|||
from openapi_core import OpenAPI
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json"
|
||||
OPENAPI_SPEC_URL = "https://aistudio.google.com/api-reference/interactions.openapi.json"
|
||||
OPENAPI_SPEC_SNAPSHOT: Final = Path(__file__).with_name("interactions.openapi.json")
|
||||
SPEC_URL_OPTION: Final = TypeAdapter(str | None)
|
||||
|
||||
|
|
@ -47,10 +47,10 @@ def _requested_spec_url(request: pytest.FixtureRequest) -> str | None:
|
|||
def _load_openapi_spec(live_url: str | None) -> LoadedSpec:
|
||||
if live_url is None:
|
||||
return LoadedSpec(source=str(OPENAPI_SPEC_SNAPSHOT), spec=json.loads(OPENAPI_SPEC_SNAPSHOT.read_text()))
|
||||
with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0) as client:
|
||||
with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0, follow_redirects=True) as client:
|
||||
response: Final = client.get(live_url)
|
||||
response.raise_for_status()
|
||||
return LoadedSpec(source=str(response.url), spec=response.json())
|
||||
return LoadedSpec(source=live_url, spec=response.json())
|
||||
|
||||
|
||||
def _model_create_request_schema(spec_dict: Dict[str, Any]) -> Dict[str, Any]:
|
||||
|
|
|
|||
|
|
@ -11047,3 +11047,24 @@ def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_
|
|||
|
||||
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["claude-opus-4-7", "claude-haiku-4-5-20251001"])
|
||||
def test_get_supported_params_reports_sampling_params_only_when_the_request_keeps_them(
|
||||
local_model_cost_map, model: str
|
||||
):
|
||||
"""What get_supported_openai_params says about temperature/top_p must match what
|
||||
map_openai_params does with them under drop_params, so a caller that pre-flights
|
||||
from the list is never told a param is fine that the request path then drops."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
supported = config.get_supported_openai_params(model=model)
|
||||
sent = config.map_openai_params(
|
||||
non_default_params={"temperature": 0.1, "top_p": 0.5},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert ("temperature" in supported) == ("temperature" in sent), (model, supported, sent)
|
||||
assert ("top_p" in supported) == ("top_p" in sent), (model, supported, sent)
|
||||
|
|
|
|||
|
|
@ -477,6 +477,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
|
|||
"aget_responses",
|
||||
"adelete_responses",
|
||||
"acancel_responses",
|
||||
"acompact_responses",
|
||||
"aresponses_websocket",
|
||||
"alist_input_items",
|
||||
"acreate_fine_tuning_job",
|
||||
|
|
|
|||
|
|
@ -2161,7 +2161,7 @@ class TestVertexClaudeBatch:
|
|||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5)
|
||||
|
||||
with pytest.raises(VertexAIError, match=r"line 1 .*top_p=0.5.*drop_params") as exc_info:
|
||||
with pytest.raises(VertexAIError, match=r"line 1 .*top_p.*drop_params") as exc_info:
|
||||
_uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"}, [entry])
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ def _make_mock_prisma():
|
|||
record.description = data.get("description")
|
||||
record.manifest_json = data.get("manifest_json", "{}")
|
||||
record.enabled = data.get("enabled", True)
|
||||
record.created_by = data.get("created_by")
|
||||
record.created_at = data.get("created_at")
|
||||
record.updated_at = data.get("updated_at")
|
||||
store[data["name"]] = record
|
||||
|
|
@ -221,7 +222,7 @@ async def test_metadata_edits_preserve_uploaded_source_and_reject_a_changed_http
|
|||
await table.create(data={"name": _UPLOAD_NAME, "manifest_json": json.dumps({"source": _UPLOAD_SOURCE})})
|
||||
request: Final = UpdatePluginRequest(source=_UPLOAD_SOURCE, description="Updated description", version="1.0.0")
|
||||
response: Final = await update_plugin(uploads=_uploads(), plugin_name=_UPLOAD_NAME, request=request, user_api_key_dict=_USER)
|
||||
expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description", "version": "1.0.0"}
|
||||
expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description", "version": "1.0.0", "_litellm_published_by": _USER.user_id}
|
||||
assert response.plugin.description == "Updated description"
|
||||
assert await _read_stored_manifest(_UPLOAD_NAME) == expected
|
||||
with pytest.raises(HTTPException) as error:
|
||||
|
|
@ -354,7 +355,7 @@ async def _register_public_and_private_plugins() -> None:
|
|||
|
||||
|
||||
async def _listed_names(user: UserAPIKeyAuth) -> set[str]:
|
||||
response = await list_plugins(user_api_key_dict=user)
|
||||
response = await list_plugins(uploads=_uploads(), user_api_key_dict=user)
|
||||
return {plugin.name for plugin in response.plugins}
|
||||
|
||||
|
||||
|
|
@ -474,7 +475,7 @@ async def test_update_plugin_propagates_installation_preference_to_get_and_list(
|
|||
)
|
||||
|
||||
plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER)
|
||||
listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0]
|
||||
listed_plugin = (await list_plugins(uploads=_uploads(), user_api_key_dict=_USER)).plugins[0]
|
||||
|
||||
assert response.plugin.installation_preference == "auto_install"
|
||||
assert plugin["installation_preference"] == "auto_install"
|
||||
|
|
@ -514,7 +515,7 @@ async def test_stored_unknown_installation_preference_is_not_served(stored_value
|
|||
)
|
||||
|
||||
marketplace = json.loads((await get_marketplace(request=MagicMock())).body)
|
||||
listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0]
|
||||
listed_plugin = (await list_plugins(uploads=_uploads(), user_api_key_dict=_USER)).plugins[0]
|
||||
plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER)
|
||||
|
||||
assert marketplace["plugins"] == [{"name": name, "source": _ARCHIVE_SOURCE}]
|
||||
|
|
@ -1288,7 +1289,7 @@ async def test_skill_upload_creates_metadata_and_bytes_atomically_and_rejects_du
|
|||
"url": f"http://test/claude-code/plugins/{_UPLOAD_NAME}/archive/{digest}.zip",
|
||||
"sha256": digest,
|
||||
}
|
||||
assert json.loads(stored["manifest_json"]) == {**metadata, "source": source}
|
||||
assert json.loads(stored["manifest_json"]) == {**metadata, "source": source, "_litellm_published_by": stored["created_by"]}
|
||||
assert stored == {
|
||||
"name": _UPLOAD_NAME, "description": "My notes", "version": "2.0.0",
|
||||
"manifest_json": stored["manifest_json"], "created_by": litellm.proxy.proxy_server.litellm_proxy_admin_name,
|
||||
|
|
|
|||
|
|
@ -196,25 +196,25 @@ def _assert_portable(response: httpx.Response) -> None:
|
|||
assert stat.S_IMODE(archive.getinfo("scripts/check.sh").external_attr >> 16) == 0o755
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key, expected", ((MASTER_KEY, 7), (PLAIN_KEY, None)))
|
||||
async def test_catalog_counts_are_admin_only_and_batched_for_the_page(
|
||||
hub: HubClient, key: str, expected: int | None,
|
||||
@pytest.mark.parametrize("route", (f"{HUB}?limit=1", "/claude-code/plugins"))
|
||||
@pytest.mark.parametrize("key", (MASTER_KEY, PLAIN_KEY))
|
||||
async def test_catalog_counts_are_batched_for_the_page(
|
||||
hub: HubClient, key: str, route: str,
|
||||
) -> None:
|
||||
assert (await _upload(hub, _zip(FILES))).status_code == 200
|
||||
record: Final = _stored_upload(hub)
|
||||
hub.plugins.find_many.return_value = [record, record.model_copy(update={"id": "other", "name": "zzz-other"})]
|
||||
hub.database.query_raw.reset_mock()
|
||||
hub.database.query_raw.return_value = [{"id": record.id, "download_count": 7}]
|
||||
response: Final = await hub.client.get(f"{HUB}?limit=1", headers=_headers(key))
|
||||
response: Final = await hub.client.get(route, headers=_headers(key))
|
||||
assert response.status_code == 200, response.text
|
||||
assert [(item["id"], item["download_count"]) for item in response.json()["skills"]] == [(record.id, expected)]
|
||||
if expected is None:
|
||||
hub.database.query_raw.assert_not_awaited()
|
||||
else:
|
||||
assert hub.database.query_raw.await_count == 1
|
||||
assert hub.database.query_raw.await_args.args[1] == (record.id,)
|
||||
assert (await hub.client.get(f"{HUB}?offset=2", headers=_headers(key))).json()["skills"] == []
|
||||
assert hub.database.query_raw.await_count == 1
|
||||
items: Final = response.json()["plugins" if route == "/claude-code/plugins" else "skills"]
|
||||
expected: Final = {record.id: 7, "other": 0} if route == "/claude-code/plugins" else {record.id: 7}
|
||||
assert {item["id"]: item["download_count"] for item in items} == expected
|
||||
assert hub.database.query_raw.await_count == 1
|
||||
assert set(hub.database.query_raw.await_args.args[1]) == set(expected)
|
||||
assert (await hub.client.get(f"{HUB}?offset=2", headers=_headers(key))).json()["skills"] == []
|
||||
assert hub.database.query_raw.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ("upload", "import"))
|
||||
|
|
@ -806,3 +806,98 @@ async def test_configured_scan_persists_result_without_changing_manual_approval(
|
|||
assert (await hub.client.get(path, headers=_headers(MASTER_KEY))).json() == before
|
||||
approved: Final = await hub.client.put(path + "/approve", headers=_headers(MASTER_KEY))
|
||||
assert approved.status_code == 200 and approved.json()["submission"]["status"] == "approved"
|
||||
|
||||
|
||||
async def test_catalog_filters_metadata_before_pagination_and_preserves_tag_facets(hub: HubClient) -> None:
|
||||
hub.plugins.find_many.return_value = [
|
||||
prisma_models.LiteLLM_ClaudeCodePluginTable(
|
||||
id=name, name=name, enabled=True, anonymous_download_count=0, created_by="author@example.test",
|
||||
created_at=datetime(2026, 1, 2), updated_at=datetime(2026, 2, 3),
|
||||
manifest_json=json.dumps({
|
||||
"source": {"source": "github", "repo": "example/skills"},
|
||||
"keywords": tags,
|
||||
"_litellm_published_by": "publisher@example.test",
|
||||
**({"_litellm_review": {"approved_by": "reviewer@example.test", "security_scan_passed": scanned}} if approved else {}),
|
||||
}),
|
||||
)
|
||||
for name, tags, approved, scanned in (
|
||||
("a-other", ["support"], False, False),
|
||||
("b-review", ["engineering"], True, False),
|
||||
("c-review", ["engineering"], True, True),
|
||||
)
|
||||
]
|
||||
response = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={
|
||||
"query": " REVIEWER ", "tag": "ENGINEERING", "limit": 1, "offset": 1,
|
||||
})
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["count"] == 2
|
||||
assert body["tags"] == ["engineering", "support"]
|
||||
assert [skill["name"] for skill in body["skills"]] == ["c-review"]
|
||||
skill = body["skills"][0]
|
||||
assert skill["created_by"] == "author@example.test"
|
||||
assert skill["published_by"] == "publisher@example.test"
|
||||
assert skill["approved_by"] == "reviewer@example.test"
|
||||
assert skill["created_at"].startswith("2026-01-02")
|
||||
assert skill["updated_at"].startswith("2026-02-03")
|
||||
assert skill["security_scan_passed"] is True
|
||||
scanned = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={"security_scan_passed": "true"})
|
||||
assert [skill["name"] for skill in scanned.json()["skills"]] == ["c-review"]
|
||||
unscanned = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={"security_scan_passed": "false"})
|
||||
assert [skill["name"] for skill in unscanned.json()["skills"]] == ["a-other", "b-review"]
|
||||
assert unscanned.json()["skills"][0]["approved_by"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_issues", (False, True))
|
||||
async def test_admin_approval_does_not_override_scan_result(pending_hub: HubClient, has_issues: bool) -> None:
|
||||
hub: Final = pending_hub
|
||||
pending: Final = await _upload(hub, _zip(FILES), MEMBER_KEY, version=None)
|
||||
receipt: Final = pending.json()
|
||||
submission: Final = hub.database.litellm_skillsubmission.rows[0]
|
||||
digest: Final = hashlib.sha256(submission.archive_content.decode()).hexdigest()
|
||||
await hub.database.litellm_skillsubmission.update(
|
||||
where={"id": submission.id}, data={"scan_json": json.dumps({
|
||||
"id": "scan", "requested_by": "admin", "started_at": "2026-01-01T00:00:00Z",
|
||||
"expires_at": "2026-01-01T00:03:00Z", "status": "completed", "archive_sha256": digest,
|
||||
"report": {"scanner_version": "test", "scan_mode": "static-only", "complete": True,
|
||||
"risk_score": 0, "findings": [{"rule_id": "rule", "severity": "LOW",
|
||||
"message": "Issue", "file": "SKILL.md"}] if has_issues else []},
|
||||
})},
|
||||
)
|
||||
approved: Final = await hub.client.put(
|
||||
f"/skill-hub/submissions/{receipt['submission_id']}/approve", headers=_headers(MASTER_KEY),
|
||||
)
|
||||
assert approved.status_code == 200
|
||||
skill: Final = approved.json()["skill"]
|
||||
assert skill["security_scan_passed"] is not has_issues
|
||||
assert skill["created_by"] == MEMBER_KEY
|
||||
assert skill["approved_by"] == approved.json()["submission"]["reviewed_by"]
|
||||
assert skill["published_by"] == skill["approved_by"]
|
||||
_stored_upload(hub)
|
||||
listed: Final = (await hub.client.get(HUB, headers=_headers(MASTER_KEY))).json()["skills"][0]
|
||||
assert listed["security_scan_passed"] is not has_issues
|
||||
assert listed["published_by"] == skill["published_by"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field,order,expected", [
|
||||
("name", "asc", "a"), ("name", "desc", "c"),
|
||||
("download_count", "desc", "a"), ("download_count", "asc", "b"),
|
||||
("updated_at", "desc", "c"), ("updated_at", "asc", "b"),
|
||||
])
|
||||
async def test_catalog_sorts_all_matches_before_pagination(
|
||||
hub: HubClient, field: str, order: str, expected: str,
|
||||
) -> None:
|
||||
hub.plugins.find_many.return_value = [
|
||||
prisma_models.LiteLLM_ClaudeCodePluginTable(
|
||||
id=name, name=name, enabled=True, anonymous_download_count=0, updated_at=datetime(2026, 1, day),
|
||||
manifest_json=json.dumps({"source": {"source": "github", "repo": "example/skills"}}),
|
||||
) for name, day in (("a", 2), ("b", 1), ("c", 3))
|
||||
]
|
||||
hub.database.query_raw.return_value = [
|
||||
{"id": name, "download_count": count} for name, count in (("a", 8), ("b", 0), ("c", 3))
|
||||
]
|
||||
response = await hub.client.get(HUB, headers=_headers(PLAIN_KEY), params={
|
||||
"sort_by": field, "sort_order": order, "limit": 1, "offset": 0,
|
||||
})
|
||||
assert response.status_code == 200, response.text
|
||||
assert [skill["name"] for skill in response.json()["skills"]] == [expected]
|
||||
|
|
|
|||
|
|
@ -3291,3 +3291,21 @@ async def test_pass_through_guardrail_text_is_the_client_body_without_the_inject
|
|||
texts: Final = [call["inputs"]["texts"] for call in guardrail.apply_calls]
|
||||
assert [[json.loads(text) for text in call_texts] for call_texts in texts] == [[body]], texts
|
||||
assert data["litellm_metadata"]["user_api_key_user_email"] == "someone@example.com", data["litellm_metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_moderation_hook_lets_responses_compact_through_without_a_translation() -> None:
|
||||
handler: Final = UnifiedLLMGuardrails()
|
||||
guardrail: Final = RecordingGuardrail()
|
||||
data: Final = {
|
||||
"guardrail_to_apply": guardrail,
|
||||
"model": "gpt-5.4-mini",
|
||||
"input": [{"role": "user", "content": "Remember the number 42."}],
|
||||
}
|
||||
|
||||
result: Final = await handler.async_moderation_hook(
|
||||
data=data, user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), call_type="acompact_responses"
|
||||
)
|
||||
|
||||
assert result is data
|
||||
assert guardrail.apply_calls == []
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ from fastapi import HTTPException
|
|||
|
||||
import litellm
|
||||
from litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard import ZscalerAIGuard
|
||||
from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
|
||||
|
|
@ -141,6 +143,216 @@ async def test_apply_guardrail_text_concatenation(mock_api_call):
|
|||
assert call_args.kwargs["content"] == "Hello world"
|
||||
|
||||
|
||||
_CONVERSATION: Final = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "What is two plus two?"},
|
||||
{"role": "assistant", "content": "Four."},
|
||||
]
|
||||
|
||||
|
||||
async def _execute_policy_bodies(
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
input_type: Literal["request", "response"],
|
||||
scan_only_tool_results: bool = False,
|
||||
) -> tuple[object, ...]:
|
||||
received: Final = asyncio.Queue[object]()
|
||||
|
||||
async def execute_policy(request: web.Request) -> web.Response:
|
||||
await received.put(await request.json())
|
||||
return web.json_response({"statusCode": 200, "action": "ALLOW"})
|
||||
|
||||
app: Final = web.Application()
|
||||
app.router.add_post("/v1/detection/execute-policy", execute_policy)
|
||||
handler: Final = InMemoryGuardrailHandler()
|
||||
async with TestServer(app) as server:
|
||||
guardrail_id: Final = handler.initialize_guardrail(
|
||||
guardrail={
|
||||
"guardrail_name": "zscaler-last-user-turn",
|
||||
"litellm_params": {
|
||||
"guardrail": "zscaler_ai_guard",
|
||||
"mode": "pre_call",
|
||||
"api_key": "test_key",
|
||||
"api_base": str(server.make_url("/v1/detection/execute-policy")),
|
||||
"policy_id": 1,
|
||||
"scan_only_tool_results": scan_only_tool_results,
|
||||
},
|
||||
},
|
||||
)["guardrail_id"]
|
||||
try:
|
||||
guardrail: Final = handler.guardrail_id_to_custom_guardrail[guardrail_id]
|
||||
assert guardrail is not None
|
||||
returned: Final = await guardrail.apply_guardrail(inputs, {}, input_type)
|
||||
finally:
|
||||
handler.delete_in_memory_guardrail(guardrail_id)
|
||||
|
||||
assert returned is inputs
|
||||
return tuple(received.get_nowait() for _ in range(received.qsize()))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("inputs", "expected_content"),
|
||||
[
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message", "flattened"],
|
||||
"structured_messages": [*_CONVERSATION, {"role": "user", "content": "And three plus three?"}],
|
||||
},
|
||||
"And three plus three?",
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message", "flattened"],
|
||||
"structured_messages": [
|
||||
*_CONVERSATION,
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "And three"}, {"type": "text", "text": "plus three?"}],
|
||||
},
|
||||
],
|
||||
},
|
||||
"And three plus three?",
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message", "flattened"],
|
||||
"structured_messages": [
|
||||
*_CONVERSATION,
|
||||
{"role": "user", "content": "And three plus three?"},
|
||||
{"role": "user", "content": "Answer in words."},
|
||||
],
|
||||
},
|
||||
"And three plus three? Answer in words.",
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["Summarize", "this report"],
|
||||
"structured_messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [{"kind": "text", "text": "Summarize"}, {"kind": "text", "text": "this report"}],
|
||||
"messageId": "msg-1",
|
||||
}
|
||||
],
|
||||
},
|
||||
"Summarize this report",
|
||||
),
|
||||
({"texts": ["first input", "second input"]}, "first input second input"),
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message"],
|
||||
"structured_messages": [*_CONVERSATION, {"role": "assistant", "content": "Prefilled answer:"}],
|
||||
},
|
||||
"What is two plus two?",
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message"],
|
||||
"structured_messages": [
|
||||
*_CONVERSATION,
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "tool output"},
|
||||
],
|
||||
},
|
||||
"What is two plus two?",
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message"],
|
||||
"structured_messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "assistant", "content": "Prefilled answer:"},
|
||||
],
|
||||
},
|
||||
None,
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["You are a helpful assistant.", " "],
|
||||
"structured_messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": " "},
|
||||
],
|
||||
},
|
||||
None,
|
||||
),
|
||||
(
|
||||
{
|
||||
"texts": ["every", "message", "\n"],
|
||||
"structured_messages": [
|
||||
*_CONVERSATION,
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "\n"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
None,
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"string-content",
|
||||
"content-parts",
|
||||
"consecutive-user-messages",
|
||||
"a2a-message-parts",
|
||||
"texts-only-route",
|
||||
"ends-on-assistant",
|
||||
"ends-on-tool",
|
||||
"no-user-message",
|
||||
"whitespace-only-latest-turn",
|
||||
"whitespace-text-beside-image",
|
||||
],
|
||||
)
|
||||
async def test_request_scan_sends_only_the_last_user_turn(
|
||||
inputs: GenericGuardrailAPIInputs, expected_content: str | None
|
||||
) -> None:
|
||||
bodies: Final = await _execute_policy_bodies(inputs, "request")
|
||||
|
||||
assert bodies == (
|
||||
() if expected_content is None else ({"direction": "IN", "content": expected_content, "policyId": 1},)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_scan_sends_every_text() -> None:
|
||||
inputs: Final = GenericGuardrailAPIInputs(
|
||||
texts=["First choice.", "Second choice."],
|
||||
structured_messages=[*_CONVERSATION, {"role": "user", "content": "And three plus three?"}],
|
||||
)
|
||||
|
||||
bodies: Final = await _execute_policy_bodies(inputs, "response")
|
||||
|
||||
assert bodies == ({"direction": "OUT", "content": "First choice. Second choice.", "policyId": 1},)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_only_tool_results_sends_the_tool_results() -> None:
|
||||
inputs: Final = GenericGuardrailAPIInputs(
|
||||
texts=["Sunny, 22C"],
|
||||
structured_messages=[
|
||||
{"role": "user", "content": "What is the weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Sunny, 22C"},
|
||||
],
|
||||
)
|
||||
|
||||
bodies: Final = await _execute_policy_bodies(inputs, "request", scan_only_tool_results=True)
|
||||
|
||||
assert bodies == ({"direction": "IN", "content": "Sunny, 22C", "policyId": 1},)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(
|
||||
"litellm.proxy.guardrails.guardrail_hooks.zscaler_ai_guard.ZscalerAIGuard.make_zscaler_ai_guard_api_call",
|
||||
|
|
|
|||
|
|
@ -1939,6 +1939,7 @@ async def test_callback_handles_every_status_the_interactions_api_can_return():
|
|||
assert released == {
|
||||
"completed",
|
||||
"requires_action",
|
||||
"requires_continuation",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"incomplete",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from litellm.proxy.deps.repositories import get_plugin_upload_repository
|
||||
from litellm.repositories.plugin_upload_repository import PluginUploadRepository
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
|
|
@ -1804,7 +1806,6 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference:
|
|||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import register_plugin
|
||||
from litellm.repositories.plugin_upload_repository import PluginUploadRepository
|
||||
from litellm.types.proxy.claude_code_endpoints import RegisterPluginRequest
|
||||
|
||||
def create(data: Mapping[str, object]) -> LiteLLM_ClaudeCodePluginTable:
|
||||
|
|
@ -1826,6 +1827,7 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference:
|
|||
prisma.db.litellm_claudecodepluginversion.create = AsyncMock()
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
app.dependency_overrides[get_plugin_upload_repository] = lambda: PluginUploadRepository(prisma)
|
||||
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", prisma):
|
||||
asyncio.run(
|
||||
|
|
@ -1843,8 +1845,10 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference:
|
|||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"),
|
||||
)
|
||||
)
|
||||
prisma.db.query_raw.return_value = [{"id": "plugin-id", "download_count": 12}]
|
||||
response: Final = TestClient(app).get("/public/skill_hub")
|
||||
|
||||
assert response.json()["plugins"][0]["download_count"] == 12
|
||||
assert response.status_code == 200
|
||||
assert [(p["name"], p["installation_preference"]) for p in response.json()["plugins"]] == [
|
||||
("security-review", preference)
|
||||
|
|
@ -1854,6 +1858,7 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference:
|
|||
_INTERNAL_PLUGIN: Final = SimpleNamespace(
|
||||
id="plugin-id",
|
||||
name="deploy-helper",
|
||||
created_by=None,
|
||||
enabled=True,
|
||||
created_at=None,
|
||||
updated_at=None,
|
||||
|
|
@ -1874,8 +1879,11 @@ def _skill_hub_response(find_many: AsyncMock, general_settings: Mapping[str, obj
|
|||
table.find_many = find_many
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_claudecodeplugintable = table
|
||||
prisma.writer_db = prisma.db
|
||||
prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
app.dependency_overrides[get_plugin_upload_repository] = lambda: PluginUploadRepository(prisma)
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", prisma),
|
||||
|
|
@ -1909,6 +1917,7 @@ def test_public_skill_hub_500_body_carries_no_exception_text() -> None:
|
|||
def test_public_skill_hub_without_a_database_answers_db_not_connected() -> None:
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
app.dependency_overrides[get_plugin_upload_repository] = lambda: None
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", None),
|
||||
|
|
|
|||
|
|
@ -1522,6 +1522,37 @@ def test_raise_if_required_body_param_missing_rejects_missing_param(route_type,
|
|||
assert exc_info.value.message == f"{route}: Missing required parameter: '{param}'."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data",
|
||||
[
|
||||
{"model": "gemini-4-argon", "input": "prove the prime number theorem"},
|
||||
{"model": "gemini-4-argon", "previous_interaction_id": "v1_abc", "continuation_token": "tok-1"},
|
||||
{"model": "gemini-4-argon", "input": "prove it again", "continuation_token": "tok-1"},
|
||||
],
|
||||
)
|
||||
def test_create_interaction_accepts_input_or_a_continuation_token(data: dict[str, object]) -> None:
|
||||
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
|
||||
|
||||
assert raise_if_required_body_param_missing(route_type="acreate_interaction", data=data, llm_router=None) is None
|
||||
|
||||
|
||||
def test_create_interaction_without_input_or_continuation_token_names_input() -> None:
|
||||
from litellm.proxy.route_llm_request import (
|
||||
ProxyMissingRequiredParamError,
|
||||
raise_if_required_body_param_missing,
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyMissingRequiredParamError) as exc_info:
|
||||
raise_if_required_body_param_missing(
|
||||
route_type="acreate_interaction",
|
||||
data={"model": "gemini-4-argon", "previous_interaction_id": "v1_abc"},
|
||||
llm_router=None,
|
||||
)
|
||||
|
||||
assert exc_info.value.param == "input"
|
||||
assert exc_info.value.message == "/interactions: Missing required parameter: 'input'."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data, param",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.models.plugin_upload import (
|
|||
PluginPublication,
|
||||
PluginPublicationBase,
|
||||
PluginPublicationError,
|
||||
PluginReview,
|
||||
PublishedPlugin,
|
||||
)
|
||||
from litellm.repositories.plugin_publication import (
|
||||
|
|
@ -144,6 +145,44 @@ async def test_first_legacy_update_preserves_its_original_snapshot(
|
|||
}
|
||||
assert replacement.kwargs["data"] == {
|
||||
"plugin_id": current.id, "version": result.version, "description": "replacement",
|
||||
"manifest_json": json.dumps({"source": source, "version": result.version}),
|
||||
"manifest_json": json.dumps({"source": source, "version": result.version, "_litellm_published_by": "admin"}),
|
||||
"published_at": result.updated_at, "published_by": "admin", "content": None, "archive_available": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("same_source", (True, False))
|
||||
@pytest.mark.parametrize("review", [None, PluginReview("reviewer@example.test", False)])
|
||||
@pytest.mark.asyncio
|
||||
async def test_publication_preserves_only_trusted_review_for_unchanged_source(
|
||||
publication_tx: MagicMock, review: PluginReview | None, same_source: bool,
|
||||
) -> None:
|
||||
original = publication_tx.litellm_claudecodeplugintable.find_unique.return_value
|
||||
stored_review = {"approved_by": "original-reviewer", "security_scan_passed": True}
|
||||
stored_manifest = {**json.loads(original.manifest_json), "_litellm_review": stored_review}
|
||||
current = original.model_copy(update={"manifest_json": json.dumps(stored_manifest)})
|
||||
publication_tx.litellm_claudecodeplugintable.find_unique.return_value = current
|
||||
|
||||
async def update(where: Mapping[str, str], data: Mapping[str, object]) -> LiteLLM_ClaudeCodePluginTable:
|
||||
return current.model_copy(update=dict(data))
|
||||
|
||||
publication_tx.litellm_claudecodeplugintable.update.side_effect = update
|
||||
result = await publish_plugin(
|
||||
cast(PluginPublicationTransaction, publication_tx),
|
||||
PluginPublication("skill", None, "updated", {
|
||||
"_litellm_review": {"approved_by": "forged", "security_scan_passed": True},
|
||||
"source": stored_manifest["source"] if same_source else {"source": "github", "repo": "example/skills"},
|
||||
}, "author", review=review),
|
||||
PluginPublicationBase(current.id, current.updated_at),
|
||||
)
|
||||
assert isinstance(result, PublishedPlugin)
|
||||
manifest = json.loads(publication_tx.litellm_claudecodeplugintable.update.call_args.kwargs["data"]["manifest_json"])
|
||||
assert manifest["_litellm_published_by"] == (review.approved_by if review else "author")
|
||||
expected = (
|
||||
{"approved_by": review.approved_by, "security_scan_passed": review.security_scan_passed} if review
|
||||
else stored_review if same_source else None
|
||||
)
|
||||
assert manifest.get("_litellm_review") == expected
|
||||
assert (None if result.review is None else {
|
||||
"approved_by": result.review.approved_by, "security_scan_passed": result.review.security_scan_passed,
|
||||
}) == expected
|
||||
assert json.loads(publication_tx.litellm_claudecodepluginversion.create.call_args.kwargs["data"]["manifest_json"]).get("_litellm_review") == expected
|
||||
|
|
|
|||
31
tests/unit/repositories/test_skill_submission_repository.py
Normal file
31
tests/unit/repositories/test_skill_submission_repository.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.repositories.skill_submission_repository import scan_passed_for_archive
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,digest,report,expected", [
|
||||
("completed", "current", {"complete": True, "risk_score": 0, "findings": []}, True),
|
||||
("running", "current", {"complete": True, "risk_score": 0, "findings": []}, False),
|
||||
("failed", "current", {"complete": True, "risk_score": 0, "findings": []}, False),
|
||||
("completed", "previous", {"complete": True, "risk_score": 0, "findings": []}, False),
|
||||
("completed", "current", {"complete": False, "risk_score": 0, "findings": []}, False),
|
||||
("completed", "current", None, False),
|
||||
("completed", "current", {"complete": True, "risk_score": 90, "findings": []}, False),
|
||||
("completed", "current", {"complete": True, "risk_score": 0, "findings": [{"rule_id": "rule", "severity": "LOW", "message": "Issue", "file": "SKILL.md"}]}, False),
|
||||
("completed", "current", {"complete": True, "risk_score": 0}, False),
|
||||
])
|
||||
def test_scan_pass_requires_clean_complete_report_for_published_archive(
|
||||
status: str, digest: str, report: dict[str, object] | None, expected: bool,
|
||||
) -> None:
|
||||
assert scan_passed_for_archive(json.dumps({
|
||||
"id": "scan", "requested_by": "admin", "started_at": "2026-01-01T00:00:00Z",
|
||||
"expires_at": "2026-01-01T00:03:00Z", "status": status, "archive_sha256": digest,
|
||||
"report": None if report is None else {"scanner_version": "test", "scan_mode": "static-only", **report},
|
||||
}), "current") is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("record", [None, "", "{}", "null", "not-json"])
|
||||
def test_missing_or_invalid_scan_is_not_reported_as_passed(record: str | None) -> None:
|
||||
assert scan_passed_for_archive(record, "current") is False
|
||||
|
|
@ -2722,7 +2722,7 @@ async def test_stream_auto_execute_bills_a_follow_up_cut_after_streamed_text_wit
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_the_success_row(
|
||||
async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_one_failure_row(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
|
|
@ -2743,10 +2743,22 @@ async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_r
|
|||
if not ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked):
|
||||
ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj})
|
||||
await logging_obj.async_failure_handler(blocked, "blocked by output-policy")
|
||||
await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0)
|
||||
await _settle(lambda: len(recorder.failed) > 0)
|
||||
|
||||
_assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events))
|
||||
assert [row["response_cost"] for row in recorder.failed] == [0.0]
|
||||
assert recorder.logged == ()
|
||||
assert len(recorder.failed) == 1
|
||||
row: Final = recorder.failed[0]
|
||||
charged_once: Final = _round_cost(_ROUND_ONE_BODY) + _round_cost(_ROUND_TWO_BODY) - _FIXED_MARGIN
|
||||
assert row["status"] == "failure"
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5)
|
||||
assert row["response_cost"] == pytest.approx(charged_once)
|
||||
assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once)
|
||||
client: Final = _final_response(events)
|
||||
assert "message" in _output_types(client.output)
|
||||
assert "function_call" not in _output_types(client.output)
|
||||
assert client.usage is not None
|
||||
assert (client.usage.input_tokens, client.usage.output_tokens) == (71 + 149, 19 + 5)
|
||||
assert client.usage.cost == pytest.approx(charged_once)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
49
tests/unit/router_rust/test_walk.py
Normal file
49
tests/unit/router_rust/test_walk.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import pytest
|
||||
|
||||
from litellm.router_rust.invoke.walk import WalkMetadata
|
||||
|
||||
|
||||
def _no_base_model(_model_id: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["metadata", "litellm_metadata"])
|
||||
def test_hop_kwargs_keep_the_requests_key_labels_under_the_walk_stamps(field: str) -> None:
|
||||
"""The fallback loggers (prometheus' fallback counters) read the caller's key and team off
|
||||
the hop's metadata, so a hop keeps the request's bucket and lays its own stamps over it."""
|
||||
request = {
|
||||
"model": "primary",
|
||||
field: {
|
||||
"user_api_key_hash": "hashed-key",
|
||||
"user_api_key_alias": "key-alias",
|
||||
"user_api_key_team_id": "team-id",
|
||||
"user_api_key_team_alias": "team-alias",
|
||||
"attempted_fallbacks": "stale",
|
||||
},
|
||||
}
|
||||
walk = WalkMetadata(request, field, _no_base_model)
|
||||
walk.hop({"parent": 0, "scope": 1, "from": "primary", "to": "fallback", "depth": 1, "max_fallbacks": 5})
|
||||
|
||||
kwargs = walk.hop_kwargs({"scope": 1, "to": "fallback"})
|
||||
|
||||
assert kwargs["model"] == "fallback"
|
||||
metadata = kwargs[field]
|
||||
assert isinstance(metadata, dict)
|
||||
assert metadata["user_api_key_hash"] == "hashed-key"
|
||||
assert metadata["user_api_key_alias"] == "key-alias"
|
||||
assert metadata["user_api_key_team_id"] == "team-id"
|
||||
assert metadata["user_api_key_team_alias"] == "team-alias"
|
||||
assert metadata["model_group"] == "fallback"
|
||||
assert metadata["original_model_group"] == "primary"
|
||||
assert metadata["attempted_fallbacks"] == 1
|
||||
assert request[field]["attempted_fallbacks"] == "stale"
|
||||
assert "model_group" not in request[field]
|
||||
|
||||
|
||||
def test_hop_kwargs_without_request_metadata_hold_the_walk_stamps() -> None:
|
||||
walk = WalkMetadata({"model": "primary"}, "metadata", _no_base_model)
|
||||
walk.hop({"parent": 0, "scope": 1, "from": "primary", "to": "fallback", "depth": 1, "max_fallbacks": 5})
|
||||
|
||||
metadata = walk.hop_kwargs({"scope": 1, "to": "fallback"})["metadata"]
|
||||
|
||||
assert metadata == {"original_model_group": "primary", "model_group": "fallback", "attempted_fallbacks": 1}
|
||||
|
|
@ -1480,3 +1480,58 @@ async def test_a_pin_written_without_tools_is_read_when_tools_is_an_empty_list()
|
|||
)
|
||||
|
||||
assert filtered == [deployments[0]]
|
||||
|
||||
|
||||
class _LoggedMessagesProbe(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
self.logged_messages: list[object] = []
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.logged_messages.append((kwargs["standard_logging_object"]["messages"], kwargs["messages"]))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pin_is_written_with_message_logging_off_while_loggers_still_see_redacted_messages(
|
||||
monkeypatch, local_model_cost_map
|
||||
):
|
||||
"""
|
||||
`turn_off_message_logging` redacted the shared success-event payload before the check read it, so the
|
||||
pin was keyed off `redacted-by-litellm`, never written, and a cacheable conversation bounced across the
|
||||
group. The pin must land whatever the flag says, while a real logger alongside still sees only the
|
||||
redacted messages.
|
||||
"""
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False)
|
||||
probe = _LoggedMessagesProbe()
|
||||
monkeypatch.setattr(litellm, "callbacks", [probe])
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": MODEL_GROUP_ALIAS,
|
||||
"litellm_params": {"model": "anthropic/claude-opus-4-6", "api_key": "sk-fake"},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
for model_id in ("dep-1", "dep-2")
|
||||
],
|
||||
optional_pre_call_checks=["prompt_caching"],
|
||||
)
|
||||
messages = _messages(word_count=5000)
|
||||
|
||||
first = await router.acompletion(model=MODEL_GROUP_ALIAS, messages=messages, mock_response="ok")
|
||||
served_by = first._hidden_params["model_id"]
|
||||
|
||||
affinity_key = PromptCachingCache.get_prompt_caching_cache_key(messages, None)
|
||||
pin = await _eventually(lambda: router.prompt_caching_pin_cache.get_cache(key=affinity_key))
|
||||
assert pin == {"model_id": served_by}, pin
|
||||
|
||||
subsequent = [
|
||||
(await router.acompletion(model=MODEL_GROUP_ALIAS, messages=messages, mock_response="ok"))._hidden_params[
|
||||
"model_id"
|
||||
]
|
||||
for _ in range(4)
|
||||
]
|
||||
assert subsequent == [served_by] * 4
|
||||
|
||||
await _eventually(lambda: len(probe.logged_messages) == 5)
|
||||
redacted = [{"role": "user", "content": "redacted-by-litellm"}]
|
||||
assert probe.logged_messages == [(redacted, redacted)] * 5, probe.logged_messages
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from litellm.router_utils.cooldown_handlers import (
|
|||
async_get_cooldown_deployments,
|
||||
cast_exception_status_to_int,
|
||||
get_cooldown_deployments,
|
||||
is_response_lookup_not_found,
|
||||
mark_advisor_orchestration_failure,
|
||||
set_cooldown_deployments,
|
||||
should_cooldown_based_on_allowed_fails_policy,
|
||||
|
|
@ -3565,3 +3566,25 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the
|
|||
with pytest.raises(RouterRateLimitError) as refusal:
|
||||
_select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs))
|
||||
assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "call_type", "expected"),
|
||||
[
|
||||
(404, "aget_responses", True),
|
||||
(404, "adelete_responses", True),
|
||||
(404, "acancel_responses", True),
|
||||
(404, "alist_input_items", True),
|
||||
(404, "get_responses", True),
|
||||
(404, "aresponses", False),
|
||||
(404, "acompact_responses", False),
|
||||
(404, "acompletion", False),
|
||||
(500, "aget_responses", False),
|
||||
(429, "alist_input_items", False),
|
||||
],
|
||||
)
|
||||
def test_is_response_lookup_not_found(status_code: int, call_type: str, expected: bool) -> None:
|
||||
error: Final = litellm.NotFoundError(message="gone", llm_provider="openai", model="gpt-4o-mini")
|
||||
error.status_code = status_code
|
||||
|
||||
assert is_response_lookup_not_found(error, {"call_type": call_type}) is expected
|
||||
|
|
|
|||
|
|
@ -13,6 +13,11 @@ from collections.abc import Callable, Generator, Mapping
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import redis
|
||||
import redis.asyncio
|
||||
from fakeredis import FakeRedis, FakeServer
|
||||
from fakeredis.aioredis import FakeConnection
|
||||
from redis.asyncio import BlockingConnectionPool
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
|
@ -23,6 +28,7 @@ from litellm.router_api import ChangeSet, router_init_args
|
|||
from litellm.router_core import RouterCore
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck
|
||||
from litellm.scheduler import FlowItem
|
||||
from litellm.types.router import (
|
||||
AlertingConfig,
|
||||
AllowedFailsPolicy,
|
||||
|
|
@ -620,6 +626,41 @@ def test_applying_a_redis_cache_attaches_it_to_the_routing_state_and_the_pins(ma
|
|||
assert core.registry().plugins("prompt_caching_pins").redis_cache is redis_cache
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def shared_redis(monkeypatch: pytest.MonkeyPatch) -> FakeServer:
|
||||
server: Final = FakeServer()
|
||||
|
||||
class Pool(BlockingConnectionPool):
|
||||
def __init__(self, **kwargs: object) -> None: # kwargs-ok: forwarded to the redis pool
|
||||
super().__init__(connection_class=FakeConnection, server=server, **kwargs)
|
||||
|
||||
monkeypatch.setattr(redis, "Redis", FakeRedis)
|
||||
monkeypatch.setattr(redis.asyncio, "BlockingConnectionPool", Pool)
|
||||
return server
|
||||
|
||||
|
||||
async def test_two_workers_on_one_redis_share_the_priority_queue(make_core: MakeCore, shared_redis: FakeServer) -> None:
|
||||
first: Final = _one(make_core, redis_host="scheduler-shared", redis_port=6399)
|
||||
second: Final = _one(make_core, redis_host="scheduler-shared", redis_port=6399)
|
||||
await first.scheduler.add_request(FlowItem(priority=0, request_id="urgent", model_name="g")) # pyright: ignore[reportAttributeAccessIssue]
|
||||
waiter: Final = FlowItem(priority=5, request_id="waiter", model_name="g")
|
||||
await second.scheduler.add_request(waiter) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
assert not await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue]
|
||||
await first.scheduler.remove_request("urgent", "g") # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
async def test_without_redis_each_worker_keeps_its_own_queue(make_core: MakeCore) -> None:
|
||||
first: Final = _one(make_core)
|
||||
second: Final = _one(make_core)
|
||||
await first.scheduler.add_request(FlowItem(priority=0, request_id="urgent", model_name="g")) # pyright: ignore[reportAttributeAccessIssue]
|
||||
waiter: Final = FlowItem(priority=5, request_id="waiter", model_name="g")
|
||||
await second.scheduler.add_request(waiter) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
assert await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
def test_cache_responses_installs_a_local_response_cache_on_the_host(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
core: Final = _one(make_core, cache_responses=True)
|
||||
assert isinstance(host.cache, litellm.Cache)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.caching.in_memory_cache import InMemoryCache
|
|||
from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
get_deployment_successes_for_current_minute,
|
||||
)
|
||||
|
|
@ -921,6 +922,43 @@ async def test_video_lookup_404_never_cools_down_the_deployment_that_answered(re
|
|||
assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "method", "path"),
|
||||
[
|
||||
("aget_responses", "GET", ""),
|
||||
("adelete_responses", "DELETE", ""),
|
||||
("acancel_responses", "POST", "/cancel"),
|
||||
("alist_input_items", "GET", "/input_items"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_lookup_404_never_cools_down_the_deployment_that_answered(
|
||||
respx_mock: respx.MockRouter, endpoint: str, method: str, path: str
|
||||
) -> None:
|
||||
respx_mock.route(method=method, url=f"https://api.openai.com/v1/responses/resp_gone{path}").respond(
|
||||
404, json={"error": {"code": "NotFound", "message": "Response not found"}}
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": api_key},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for api_key, deployment_id in (("key-a", "dep-a"), ("key-b", "dep-b"))
|
||||
],
|
||||
num_retries=0,
|
||||
allowed_fails=1,
|
||||
)
|
||||
response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id("openai", "dep-b", "resp_gone")
|
||||
|
||||
for _ in range(3):
|
||||
with pytest.raises(litellm.NotFoundError):
|
||||
await getattr(router, endpoint)(response_id=response_id)
|
||||
|
||||
assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_404_still_cools_down_the_deployment(respx_mock: respx.MockRouter) -> None:
|
||||
respx_mock.post("https://api.openai.com/v1/chat/completions").respond(
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ from litellm.utils import (
|
|||
_snapshot_exception_for_hook,
|
||||
async_post_call_failure_deployment_hook,
|
||||
async_post_call_success_deployment_hook,
|
||||
async_pre_call_deployment_hook,
|
||||
calculate_max_parallel_requests,
|
||||
client,
|
||||
get_non_default_completion_params,
|
||||
|
|
@ -11626,3 +11627,25 @@ async def test_success_enqueue_is_tracked_without_starting_callbacks_synchronous
|
|||
assert GLOBAL_LOGGING_WORKER.pending_count > 0, "Tracked enqueue must finish before worker draining starts"
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
assert logging_obj.model_call_details["has_logged_async_success"] is True
|
||||
|
||||
|
||||
class _RecordingPreCallDeploymentHook(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.seen_call_types: tuple[CallTypes | None, ...] = ()
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: dict[str, object], call_type: CallTypes | None
|
||||
) -> dict[str, object] | None:
|
||||
self.seen_call_types = (*self.seen_call_types, call_type)
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_deployment_hook_hands_responses_compact_its_call_type(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
hook: Final = _RecordingPreCallDeploymentHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [hook])
|
||||
|
||||
await async_pre_call_deployment_hook({"model": "gpt-5.4-mini"}, "acompact_responses")
|
||||
|
||||
assert hook.seen_call_types == (CallTypes.acompact_responses,)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_excluded_services_value
|
||||
from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content
|
||||
from litellm.types.utils import (
|
||||
CAPTURE_MESSAGE_CONTENT_VALUES,
|
||||
CallTypes,
|
||||
CaptureMessageContent,
|
||||
captures_span_content,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", " ", "auth", "none,redis", "redis,", ",postgres"])
|
||||
|
|
@ -45,3 +50,8 @@ def test_capture_message_content_constructor_accepts_any_string() -> None:
|
|||
)
|
||||
def test_captures_span_content(mode: str | None, captures: bool) -> None:
|
||||
assert captures_span_content(mode) is captures
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call_type", ["compact_responses", "acompact_responses"])
|
||||
def test_responses_compact_call_types_round_trip_their_public_names(call_type: str) -> None:
|
||||
assert CallTypes(call_type).value == call_type
|
||||
|
|
|
|||
|
|
@ -15,11 +15,12 @@ import {
|
|||
|
||||
export interface FieldConfig {
|
||||
name: string;
|
||||
label: string;
|
||||
labelKey: ParseKeys;
|
||||
type: "text" | "textarea" | "url" | "switch" | "list" | "select";
|
||||
required?: boolean;
|
||||
tooltip?: string;
|
||||
tooltipKey?: ParseKeys;
|
||||
placeholder?: string;
|
||||
placeholderKey?: ParseKeys;
|
||||
defaultValue?: any;
|
||||
rows?: number;
|
||||
validation?: any[];
|
||||
|
|
@ -50,42 +51,41 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "name",
|
||||
label: "Display Name",
|
||||
labelKey: "agents.formConfig.fields.name.label",
|
||||
type: "text",
|
||||
required: true,
|
||||
placeholder: "e.g., Customer Support Agent",
|
||||
placeholderKey: "agents.formConfig.fields.name.placeholder",
|
||||
},
|
||||
{
|
||||
name: "description",
|
||||
label: "Description",
|
||||
labelKey: "agents.formConfig.fields.description.label",
|
||||
type: "textarea",
|
||||
required: false,
|
||||
placeholder: "Describe what this agent does...",
|
||||
placeholderKey: "agents.formConfig.fields.description.placeholder",
|
||||
rows: 3,
|
||||
},
|
||||
{
|
||||
name: "url",
|
||||
label: "URL",
|
||||
labelKey: "agents.formConfig.fields.url.label",
|
||||
type: "url",
|
||||
required: false,
|
||||
placeholder: "http://localhost:9999/",
|
||||
tooltip: "Base URL where the agent is hosted (optional)",
|
||||
tooltipKey: "agents.formConfig.fields.url.tooltip",
|
||||
},
|
||||
{
|
||||
name: "version",
|
||||
label: "Version",
|
||||
labelKey: "agents.formConfig.fields.version.label",
|
||||
type: "text",
|
||||
placeholder: "1.0.0",
|
||||
defaultValue: "1.0.0",
|
||||
},
|
||||
{
|
||||
name: "protocolVersion",
|
||||
label: "Protocol Version",
|
||||
labelKey: "agents.formConfig.fields.protocolVersion.label",
|
||||
type: "select",
|
||||
options: ["1.0", "0.3"],
|
||||
defaultValue: "1.0",
|
||||
tooltip:
|
||||
"The A2A protocol version LiteLLM serves to clients for this agent. LiteLLM converts the upstream agent's responses to this version, so clients always see the version you pick here regardless of the original agent's version.",
|
||||
tooltipKey: "agents.formConfig.fields.protocolVersion.tooltip",
|
||||
helpTextKey: "agents.formConfig.basic.protocolVersionHelp",
|
||||
},
|
||||
],
|
||||
|
|
@ -96,7 +96,7 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "skills",
|
||||
label: "Skills",
|
||||
labelKey: "agents.formConfig.fields.skills.label",
|
||||
type: "list",
|
||||
defaultValue: [],
|
||||
},
|
||||
|
|
@ -108,18 +108,18 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "streaming",
|
||||
label: "Streaming",
|
||||
labelKey: "agents.formConfig.fields.streaming.label",
|
||||
type: "switch",
|
||||
defaultValue: false,
|
||||
},
|
||||
{
|
||||
name: "pushNotifications",
|
||||
label: "Push Notifications",
|
||||
labelKey: "agents.formConfig.fields.pushNotifications.label",
|
||||
type: "switch",
|
||||
},
|
||||
{
|
||||
name: "stateTransitionHistory",
|
||||
label: "State Transition History",
|
||||
labelKey: "agents.formConfig.fields.stateTransitionHistory.label",
|
||||
type: "switch",
|
||||
},
|
||||
],
|
||||
|
|
@ -130,19 +130,19 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "iconUrl",
|
||||
label: "Icon URL",
|
||||
labelKey: "agents.formConfig.fields.iconUrl.label",
|
||||
type: "url",
|
||||
placeholder: "https://example.com/icon.png",
|
||||
},
|
||||
{
|
||||
name: "documentationUrl",
|
||||
label: "Documentation URL",
|
||||
labelKey: "agents.formConfig.fields.documentationUrl.label",
|
||||
type: "url",
|
||||
placeholder: "https://docs.example.com",
|
||||
},
|
||||
{
|
||||
name: "supportsAuthenticatedExtendedCard",
|
||||
label: "Supports Authenticated Extended Card",
|
||||
labelKey: "agents.formConfig.fields.supportsAuthenticatedExtendedCard.label",
|
||||
type: "switch",
|
||||
},
|
||||
],
|
||||
|
|
@ -153,12 +153,12 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "model",
|
||||
label: "Model (Optional)",
|
||||
labelKey: "agents.formConfig.fields.model.label",
|
||||
type: "text",
|
||||
},
|
||||
{
|
||||
name: "make_public",
|
||||
label: "Make Public",
|
||||
labelKey: "agents.formConfig.fields.makePublic.label",
|
||||
type: "switch",
|
||||
},
|
||||
],
|
||||
|
|
@ -169,24 +169,24 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "cost_per_query",
|
||||
label: "Cost Per Query ($)",
|
||||
labelKey: "agents.formConfig.fields.costPerQuery.label",
|
||||
type: "text",
|
||||
placeholder: "0.0",
|
||||
tooltip: "Fixed cost per query",
|
||||
tooltipKey: "agents.formConfig.fields.costPerQuery.tooltip",
|
||||
},
|
||||
{
|
||||
name: "input_cost_per_token",
|
||||
label: "Input Cost Per Token ($)",
|
||||
labelKey: "agents.formConfig.fields.inputCostPerToken.label",
|
||||
type: "text",
|
||||
placeholder: "0.000001",
|
||||
tooltip: "Cost per input token",
|
||||
tooltipKey: "agents.formConfig.fields.inputCostPerToken.tooltip",
|
||||
},
|
||||
{
|
||||
name: "output_cost_per_token",
|
||||
label: "Output Cost Per Token ($)",
|
||||
labelKey: "agents.formConfig.fields.outputCostPerToken.label",
|
||||
type: "text",
|
||||
placeholder: "0.000002",
|
||||
tooltip: "Cost per output token",
|
||||
tooltipKey: "agents.formConfig.fields.outputCostPerToken.tooltip",
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
@ -196,10 +196,10 @@ export const AGENT_FORM_CONFIG: {
|
|||
fields: [
|
||||
{
|
||||
name: "enable_tracing",
|
||||
label: "Enable Tracing",
|
||||
labelKey: "agents.formConfig.fields.enableTracing.label",
|
||||
type: "switch",
|
||||
defaultValue: false,
|
||||
tooltip: "Enable request tracing for this agent",
|
||||
tooltipKey: "agents.formConfig.fields.enableTracing.tooltip",
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -256,23 +256,24 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
|
|||
<AgentFormField
|
||||
key={field.name}
|
||||
name={field.name}
|
||||
label={field.tooltip ? labelWithHint(field.label, field.tooltip) : field.label}
|
||||
label={field.tooltipKey ? labelWithHint(t(field.labelKey), t(field.tooltipKey)) : t(field.labelKey)}
|
||||
description={field.helpTextKey ? t(field.helpTextKey) : undefined}
|
||||
rules={
|
||||
field.required
|
||||
? { required: t("agents.formFields.errors.enterField", { field: field.label.toLowerCase() }) }
|
||||
? { required: t("agents.formFields.errors.enterField", { field: t(field.labelKey).toLowerCase() }) }
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{({ value, onChange, ref, ...control }) => {
|
||||
const text = typeof value === "string" ? value : "";
|
||||
const placeholder = field.placeholderKey ? t(field.placeholderKey) : field.placeholder;
|
||||
if (field.type === "textarea") {
|
||||
return (
|
||||
<Textarea
|
||||
{...control}
|
||||
ref={ref}
|
||||
rows={field.rows}
|
||||
placeholder={field.placeholder}
|
||||
placeholder={placeholder}
|
||||
value={text}
|
||||
onChange={onChange}
|
||||
/>
|
||||
|
|
@ -282,7 +283,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
|
|||
return (
|
||||
<Select value={text || null} onValueChange={onChange}>
|
||||
<SelectTrigger {...control} className="w-full">
|
||||
<SelectValue placeholder={field.placeholder} />
|
||||
<SelectValue placeholder={placeholder} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{(field.options ?? []).map((option) => (
|
||||
|
|
@ -294,9 +295,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
|
|||
</Select>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<Input {...control} ref={ref} placeholder={field.placeholder} value={text} onChange={onChange} />
|
||||
);
|
||||
return <Input {...control} ref={ref} placeholder={placeholder} value={text} onChange={onChange} />;
|
||||
}}
|
||||
</AgentFormField>
|
||||
))}
|
||||
|
|
@ -320,7 +319,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
|
|||
panels={panels}
|
||||
>
|
||||
{AGENT_FORM_CONFIG.capabilities.fields.map((field) => (
|
||||
<AgentFormField key={field.name} name={field.name} label={field.label}>
|
||||
<AgentFormField key={field.name} name={field.name} label={t(field.labelKey)}>
|
||||
{({ value, onChange, ref, ...control }) => (
|
||||
<Switch {...control} inputRef={ref} checked={value === true} onCheckedChange={onChange} />
|
||||
)}
|
||||
|
|
@ -336,7 +335,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
|
|||
panels={panels}
|
||||
>
|
||||
{AGENT_FORM_CONFIG.optional.fields.map((field) => (
|
||||
<AgentFormField key={field.name} name={field.name} label={field.label}>
|
||||
<AgentFormField key={field.name} name={field.name} label={t(field.labelKey)}>
|
||||
{({ value, onChange, ref, ...control }) =>
|
||||
field.type === "switch" ? (
|
||||
<Switch {...control} inputRef={ref} checked={value === true} onCheckedChange={onChange} />
|
||||
|
|
@ -372,7 +371,7 @@ const AgentFormFields: React.FC<AgentFormFieldsProps> = ({ panels, showAgentName
|
|||
panels={panels}
|
||||
>
|
||||
{AGENT_FORM_CONFIG.litellm.fields.map((field) => (
|
||||
<AgentFormField key={field.name} name={field.name} label={field.label}>
|
||||
<AgentFormField key={field.name} name={field.name} label={t(field.labelKey)}>
|
||||
{({ value, onChange, ref, ...control }) =>
|
||||
field.type === "switch" ? (
|
||||
<Switch {...control} inputRef={ref} checked={value === true} onCheckedChange={onChange} />
|
||||
|
|
|
|||
|
|
@ -1,32 +1,36 @@
|
|||
import React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { AGENT_FORM_CONFIG } from "./agent_config";
|
||||
import { AgentFormField, labelWithHint } from "./AgentFormKit";
|
||||
|
||||
export const COST_FIELD_NAMES: readonly string[] = AGENT_FORM_CONFIG.cost.fields.map((field) => field.name);
|
||||
|
||||
const CostConfigFields: React.FC = () => (
|
||||
<>
|
||||
{AGENT_FORM_CONFIG.cost.fields.map((field) => (
|
||||
<AgentFormField
|
||||
key={field.name}
|
||||
name={field.name}
|
||||
label={field.tooltip ? labelWithHint(field.label, field.tooltip) : field.label}
|
||||
>
|
||||
{({ value, onChange, ref, ...control }) => (
|
||||
<Input
|
||||
{...control}
|
||||
ref={ref}
|
||||
type="number"
|
||||
step="0.000001"
|
||||
placeholder={field.placeholder}
|
||||
value={typeof value === "string" || typeof value === "number" ? value : ""}
|
||||
onChange={onChange}
|
||||
/>
|
||||
)}
|
||||
</AgentFormField>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
const CostConfigFields: React.FC = () => {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<>
|
||||
{AGENT_FORM_CONFIG.cost.fields.map((field) => (
|
||||
<AgentFormField
|
||||
key={field.name}
|
||||
name={field.name}
|
||||
label={field.tooltipKey ? labelWithHint(t(field.labelKey), t(field.tooltipKey)) : t(field.labelKey)}
|
||||
>
|
||||
{({ value, onChange, ref, ...control }) => (
|
||||
<Input
|
||||
{...control}
|
||||
ref={ref}
|
||||
type="number"
|
||||
step="0.000001"
|
||||
placeholder={field.placeholder}
|
||||
value={typeof value === "string" || typeof value === "number" ? value : ""}
|
||||
onChange={onChange}
|
||||
/>
|
||||
)}
|
||||
</AgentFormField>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default CostConfigFields;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
"use client";
|
||||
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page";
|
||||
import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel";
|
||||
|
||||
export default function AlertsPage() {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<Page className="h-full gap-4">
|
||||
<PageTabs defaultValue="model-deprecations" className="gap-4">
|
||||
<PageTabsList>
|
||||
<PageTabsTrigger value="model-deprecations">Model deprecations</PageTabsTrigger>
|
||||
<PageTabsTrigger value="model-deprecations">{t("alerts.page.tabs.modelDeprecations")}</PageTabsTrigger>
|
||||
</PageTabsList>
|
||||
<PageTabsContent value="model-deprecations">
|
||||
<ModelDeprecationsPanel />
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import React, { useId } from "react";
|
||||
import type { ParseKeys } from "i18next";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Controller } from "react-hook-form";
|
||||
import { z } from "zod";
|
||||
|
|
@ -41,11 +42,11 @@ type NumberFieldName = "tpm_limit" | "rpm_limit" | "tpd_limit" | "max_budget";
|
|||
|
||||
const NO_RESET = "never";
|
||||
|
||||
const RESET_OPTIONS = [
|
||||
{ value: NO_RESET, label: "in total" },
|
||||
{ value: "24h", label: "per day" },
|
||||
{ value: "7d", label: "per week" },
|
||||
{ value: "30d", label: "per month" },
|
||||
const RESET_OPTIONS: readonly { value: string; labelKey: ParseKeys }[] = [
|
||||
{ value: NO_RESET, labelKey: "budgets.form.resetOptions.inTotal" },
|
||||
{ value: "24h", labelKey: "budgets.form.resetOptions.perDay" },
|
||||
{ value: "7d", labelKey: "budgets.form.resetOptions.perWeek" },
|
||||
{ value: "30d", labelKey: "budgets.form.resetOptions.perMonth" },
|
||||
];
|
||||
|
||||
const numberProps = <T extends { value?: number | null; onChange: (value: number | null) => void }>({
|
||||
|
|
@ -69,6 +70,7 @@ const OptionalBadge = () => {
|
|||
function SpendLimitField({ form }: { form: BudgetForm }) {
|
||||
const { t } = useTranslation();
|
||||
const amountId = useId();
|
||||
const resetItems = RESET_OPTIONS.map(({ value, labelKey }) => ({ value, label: t(labelKey) }));
|
||||
return (
|
||||
<Field className="gap-2">
|
||||
<FieldLabel htmlFor={amountId}>
|
||||
|
|
@ -93,7 +95,7 @@ function SpendLimitField({ form }: { form: BudgetForm }) {
|
|||
name="budget_duration"
|
||||
render={({ field }) => (
|
||||
<Select
|
||||
items={RESET_OPTIONS}
|
||||
items={resetItems}
|
||||
value={field.value ?? NO_RESET}
|
||||
onValueChange={(next) => field.onChange(next === NO_RESET ? null : next)}
|
||||
>
|
||||
|
|
@ -101,7 +103,7 @@ function SpendLimitField({ form }: { form: BudgetForm }) {
|
|||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{RESET_OPTIONS.map((option) => (
|
||||
{resetItems.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue