mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(rust): price built-in tools from every websocket tier
Python recurses completion_cost per service-tier part, so each part bills its own usage-reported web search at the per-query rate; Rust had priced only the first group. Keep the explicit per-response tool stamp and additional costs on the first group so a caller's flat charge and the router fee are not multiplied by the tier count. Pin the batch invalid-rate error while here
This commit is contained in:
parent
b8220044b6
commit
b401bc9be2
3 changed files with 100 additions and 19 deletions
|
|
@ -525,7 +525,7 @@ fn add_completion_costs(first: CompletionCost, second: CompletionCost) -> Comple
|
|||
}
|
||||
}
|
||||
|
||||
fn built_in_tool_cost(
|
||||
fn config_driven_tool_cost(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: CompletionResponseCostRequest<'_>,
|
||||
model: &str,
|
||||
|
|
@ -534,25 +534,24 @@ fn built_in_tool_cost(
|
|||
usage: Option<&ChatUsage>,
|
||||
) -> f64 {
|
||||
let Some(config) = request.built_in_tool_config else {
|
||||
return request.built_in_tool_cost;
|
||||
return 0.0;
|
||||
};
|
||||
let Some(response) = request.input.model_selection.response else {
|
||||
return request.built_in_tool_cost;
|
||||
return 0.0;
|
||||
};
|
||||
request.built_in_tool_cost
|
||||
+ catalog.built_in_tool_cost(
|
||||
model,
|
||||
catalog.built_in_tool_cost(
|
||||
model,
|
||||
provider,
|
||||
region,
|
||||
BuiltInToolCostRequest {
|
||||
response,
|
||||
response_kind: config.response_kind,
|
||||
usage,
|
||||
provider,
|
||||
region,
|
||||
BuiltInToolCostRequest {
|
||||
response,
|
||||
response_kind: config.response_kind,
|
||||
usage,
|
||||
provider,
|
||||
params: config.params,
|
||||
defaults: config.defaults,
|
||||
},
|
||||
)
|
||||
params: config.params,
|
||||
defaults: config.defaults,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn price_responses_websocket(
|
||||
|
|
@ -608,10 +607,17 @@ fn price_responses_websocket(
|
|||
},
|
||||
)?;
|
||||
let built_in = if index == 0 {
|
||||
built_in_tool_cost(catalog, request, &model, provider, region, Some(usage))
|
||||
request.built_in_tool_cost
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
} + config_driven_tool_cost(
|
||||
catalog,
|
||||
request,
|
||||
&model,
|
||||
provider,
|
||||
region,
|
||||
Some(usage),
|
||||
);
|
||||
let additional = if index == 0 {
|
||||
request.additional_costs
|
||||
} else {
|
||||
|
|
@ -868,7 +874,8 @@ pub fn completion_cost_from_response(
|
|||
if is_search {
|
||||
0.0
|
||||
} else {
|
||||
built_in_tool_cost(catalog, request, &model, provider, region, Some(usage))
|
||||
request.built_in_tool_cost
|
||||
+ config_driven_tool_cost(catalog, request, &model, provider, region, Some(usage))
|
||||
},
|
||||
&additional_costs,
|
||||
provider,
|
||||
|
|
|
|||
|
|
@ -1551,3 +1551,61 @@ fn rateless_custom_pricing_strips_provider_reported_cost_like_python() {
|
|||
let reported = completion_cost_from_response(&catalog, base).unwrap();
|
||||
assert!((reported.cost.total - 99.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn responses_websocket_prices_built_in_tools_from_each_tiers_usage() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"openai/model".to_owned(),
|
||||
json!({
|
||||
"input_cost_per_token": 0.002,
|
||||
"output_cost_per_token": 0.003,
|
||||
"search_context_cost_per_query": {"search_context_size_medium": 0.02}
|
||||
}),
|
||||
)]));
|
||||
let response = json!({"results": [
|
||||
{"type": "response.completed", "response": {"service_tier": "default", "usage": {
|
||||
"input_tokens": 100, "output_tokens": 40,
|
||||
"input_token_details": {"web_search_requests": 2}
|
||||
}}},
|
||||
{"type": "response.incomplete", "response": {"service_tier": "priority", "usage": {
|
||||
"input_tokens": 60, "output_tokens": 10,
|
||||
"input_token_details": {"web_search_requests": 3}
|
||||
}}}
|
||||
]});
|
||||
let empty = json!({});
|
||||
let base = request(
|
||||
Some(&response),
|
||||
Some("model"),
|
||||
Some("openai"),
|
||||
&empty,
|
||||
&empty,
|
||||
);
|
||||
let result = completion_cost_from_response(
|
||||
&catalog,
|
||||
CompletionResponseCostRequest {
|
||||
input: CompletionInputRequest {
|
||||
call_type: Some("_aresponses_websocket"),
|
||||
..base.input
|
||||
},
|
||||
built_in_tool_config: Some(BuiltInToolCostConfig {
|
||||
response_kind: ToolResponseKind::Other,
|
||||
params: &empty,
|
||||
defaults: DefaultToolRates {
|
||||
file_search_per_call: 0.0,
|
||||
azure_file_search_per_gb_day: 0.0,
|
||||
azure_vector_store_per_gb_day: 0.0,
|
||||
azure_computer_input_per_1k_tokens: 0.0,
|
||||
azure_computer_output_per_1k_tokens: 0.0,
|
||||
code_interpreter_per_session: None,
|
||||
xai_web_search_per_call: 0.0,
|
||||
groq_browser_open_per_call: 0.0,
|
||||
},
|
||||
}),
|
||||
..base
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let tokens = 100.0 * 0.002 + 40.0 * 0.003 + 60.0 * 0.002 + 10.0 * 0.003;
|
||||
assert!((result.cost.built_in_tools - 2.0 * 0.02).abs() < 1e-12);
|
||||
assert!((result.cost.total - tokens - 2.0 * 0.02).abs() < 1e-12);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -649,3 +649,19 @@ fn batch_model_info_honors_an_explicitly_zero_batch_rate() {
|
|||
let cost = batch_cost_from_model_info(&model_info, &usage, Some("anthropic"), None).unwrap();
|
||||
assert_eq!((cost.prompt, cost.completion), (0.0, 0.0));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn batch_model_info_rejects_invalid_rates_with_an_error_not_a_negative_cost() {
|
||||
let model_info = json!({
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_batches": -1e-6
|
||||
});
|
||||
let usage = ChatUsage {
|
||||
prompt_tokens: 1000,
|
||||
completion_tokens: 500,
|
||||
total_tokens: 1500,
|
||||
..ChatUsage::default()
|
||||
};
|
||||
assert!(batch_cost_from_model_info(&model_info, &usage, Some("anthropic"), None).is_err());
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue