mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
546402c98c
* feat(decisions): add the OpenAI Decisions spec types and the System One translation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): share one DecisionsModel config and require model and usage on responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): rename the shared pydantic parent to DecisionsObjectBase Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): map OpenRouter connection failures to APIConnectionError and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(decisions): switch /v1/decisions to the OpenAI Decisions schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(decisions): build the usage the OpenAI response schema now requires Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lens): ask signal questions through the OpenAI Decisions schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(decisions): refuse safety_identifier for System One providers unless drop_params * feat(decisions): refuse safety_identifier on providers that cannot take it unless drop_params drops it * test(decisions): cover hosted_vllm in the safety_identifier and per-provider wire tests * fix(decisions): check a System One request's safety_identifier against the provider too * fix(decisions): drop a non-string safety_identifier under drop_params A malformed safety_identifier now follows the drop_params convention in both body shapes: it answers 400 without drop_params and is dropped before the provider call with it. A string identifier on OpenAI stays on the wire either way. * fix(decisions): let /v1/decisions drop a non-string safety_identifier under drop_params The route checked the whole body before routing, so a non-string safety_identifier answered 400 even when the deployment or the body set drop_params. The route now leaves that field to the Decisions call, which knows every drop_params source * refactor(decisions): return the unsupported safety_identifier as a value and raise it in _prepare_call --------- Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
181 lines
6.9 KiB
Python
181 lines
6.9 KiB
Python
from typing import Final
|
|
|
|
from integration.translation.case import TranslationTestCase
|
|
|
|
"""Provider request and reply shape from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06.
|
|
"""
|
|
CLEF_TEST_CASE: Final = TranslationTestCase(
|
|
scenario="basic",
|
|
litellm_endpoint="/v1/decisions",
|
|
litellm_request={
|
|
"model": "cloudflare/@cf/cloudflare/clef",
|
|
"input": "Ticket (billing): The export job hangs at 99% and never finishes",
|
|
"questions": [
|
|
{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"},
|
|
{
|
|
"type": "choice",
|
|
"name": "severity",
|
|
"instructions": "How severe is it?",
|
|
"choices": [
|
|
{"value": "low", "description": "cosmetic"},
|
|
{"value": "high", "description": "blocks users"},
|
|
],
|
|
},
|
|
{
|
|
"type": "score",
|
|
"name": "confidence",
|
|
"instructions": "How sure are you?",
|
|
"levels": [{"label": "unsure"}, {"label": "sure"}],
|
|
},
|
|
],
|
|
"cache": {"no-cache": True},
|
|
},
|
|
expected_provider_endpoint="/ai/run/@cf/cloudflare/clef",
|
|
expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"},
|
|
expected_provider_request={
|
|
"model": "clef",
|
|
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
|
"questions": {
|
|
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
|
"severity": {
|
|
"type": "choice",
|
|
"instructions": "How severe is it?",
|
|
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
|
},
|
|
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
|
},
|
|
},
|
|
mock_provider_response={
|
|
"result": {
|
|
"model": "clef",
|
|
"answers": {
|
|
"defect": {"type": "noul", "noul": 0.9345},
|
|
"severity": {
|
|
"type": "choice",
|
|
"choice": "high",
|
|
"confidence": 0.8067,
|
|
"probabilities": {"low": 0.0509, "high": 0.9491},
|
|
},
|
|
"confidence": {
|
|
"type": "score",
|
|
"score": 0.9036,
|
|
"confidence": 0.6515,
|
|
"legend": {"0": "unsure", "1": "sure"},
|
|
"probabilities": {"0": 0.0964, "1": 0.9036},
|
|
},
|
|
},
|
|
"usage": {"input_tokens": 290, "output_tokens": 0},
|
|
},
|
|
"success": True,
|
|
"errors": [],
|
|
"messages": [],
|
|
},
|
|
expected_litellm_response={
|
|
"model": "clef",
|
|
"answers": [
|
|
{"type": "predicate", "name": "defect", "probability": 0.9345},
|
|
{
|
|
"type": "choice",
|
|
"name": "severity",
|
|
"choice": "high",
|
|
"probabilities": [{"value": "low", "probability": 0.0509}, {"value": "high", "probability": 0.9491}],
|
|
"confidence": 0.8067,
|
|
},
|
|
{
|
|
"type": "score",
|
|
"name": "confidence",
|
|
"score": 0.9036,
|
|
"probabilities": [
|
|
{"value": 0, "label": "unsure", "probability": 0.0964},
|
|
{"value": 1, "label": "sure", "probability": 0.9036},
|
|
],
|
|
"confidence": 0.6515,
|
|
},
|
|
],
|
|
"usage": {
|
|
"input_tokens": 290,
|
|
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
|
|
"output_tokens": 0,
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
"total_tokens": 290,
|
|
},
|
|
},
|
|
)
|
|
CLEF_SYSTEMONE_TEST_CASE: Final = TranslationTestCase(
|
|
scenario="systemone",
|
|
litellm_endpoint="/v1/systemone",
|
|
litellm_request={
|
|
"model": "cloudflare/@cf/cloudflare/clef",
|
|
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
|
"questions": {
|
|
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
|
"severity": {
|
|
"type": "choice",
|
|
"instructions": "How severe is it?",
|
|
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
|
},
|
|
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
|
},
|
|
"cache": {"no-cache": True},
|
|
},
|
|
expected_provider_endpoint="/ai/run/@cf/cloudflare/clef",
|
|
expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"},
|
|
expected_provider_request={
|
|
"model": "clef",
|
|
"state": "Ticket (billing): The export job hangs at 99% and never finishes",
|
|
"questions": {
|
|
"defect": {"type": "noul", "instructions": "Is this a defect?"},
|
|
"severity": {
|
|
"type": "choice",
|
|
"instructions": "How severe is it?",
|
|
"criteria": {"low": "cosmetic", "high": "blocks users"},
|
|
},
|
|
"confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]},
|
|
},
|
|
},
|
|
mock_provider_response={
|
|
"result": {
|
|
"model": "clef",
|
|
"answers": {
|
|
"defect": {"type": "noul", "noul": 0.9345},
|
|
"severity": {
|
|
"type": "choice",
|
|
"choice": "high",
|
|
"confidence": 0.8067,
|
|
"probabilities": {"low": 0.0509, "high": 0.9491},
|
|
},
|
|
"confidence": {
|
|
"type": "score",
|
|
"score": 0.9036,
|
|
"confidence": 0.6515,
|
|
"legend": {"0": "unsure", "1": "sure"},
|
|
"probabilities": {"0": 0.0964, "1": 0.9036},
|
|
},
|
|
},
|
|
"usage": {"input_tokens": 290, "output_tokens": 0},
|
|
},
|
|
"success": True,
|
|
"errors": [],
|
|
"messages": [],
|
|
},
|
|
expected_litellm_response={
|
|
"model": "clef",
|
|
"answers": {
|
|
"defect": {"type": "noul", "noul": 0.9345},
|
|
"severity": {
|
|
"type": "choice",
|
|
"choice": "high",
|
|
"confidence": 0.8067,
|
|
"probabilities": {"low": 0.0509, "high": 0.9491},
|
|
},
|
|
"confidence": {
|
|
"type": "score",
|
|
"score": 0.9036,
|
|
"confidence": 0.6515,
|
|
"legend": {"0": "unsure", "1": "sure"},
|
|
"probabilities": {"0": 0.0964, "1": 0.9036},
|
|
},
|
|
},
|
|
"usage": {"input_tokens": 290, "output_tokens": 0},
|
|
},
|
|
)
|