litellm/litellm-rust/crates/inference-responses/tests/responses.rs

use litellm_host::{
    interceptors::{ExecutionFacts, ResultSource},
    lifecycle::ExecutionEvent,
};
use std::sync::Arc;

use futures_util::TryStreamExt;
use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
use litellm_inference_responses::{
    Error,
    route::Responses,
    types::{ResponsesCall, ResponsesOutput},
};
use litellm_inference_testing::{RecordingSecrets, no_secrets};
use rstest::{fixture, rstest};
use serde_json::json;
use wiremock::ResponseTemplate;

mod support;
use support::*;

#[fixture]
fn call() -> ResponsesCall {
    ResponsesCall {
        model: "openai/test-model".into(),
        input: json!("hello"),
        optional_params: Default::default(),
        api_key: Some("test-key".into()),
        api_base: None,
        custom_llm_provider: None,
        extra_headers: None,
        timeout: None,
    }
}

#[rstest]
#[case::bare_openai("gpt-5", None, "gpt-5")]
#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
#[case::explicit_openrouter_prefix(
    "openrouter/anthropic/claude",
    Some("openrouter"),
    "anthropic/claude"
)]
#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
#[tokio::test]
async fn provider_resolution_preserves_the_wire_model(
    #[case] model: &str,
    #[case] custom_llm_provider: Option<&str>,
    #[case] expected_wire_model: &str,
) {
    let response = json!({
        "id": "response-1",
        "model": expected_wire_model,
        "output": [{"type": "message", "content": []}],
        "usage": {"total_tokens": 7}
    });
    let upstream = upstream([json_response(response)]).await;
    responses_route(no_secrets())
        .execute(
            ResponsesCall {
                model: model.into(),
                api_key: Some("test-key".into()),
                api_base: Some(upstream.uri()),
                custom_llm_provider: custom_llm_provider.map(str::to_owned),
                ..call()
            },
            &(),
            None,
        )
        .await
        .unwrap();

    assert_eq!(
        only_request(&upstream).await.json()["model"],
        expected_wire_model
    );
}

#[rstest]
#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
#[tokio::test]
async fn unsupported_providers_fail_before_transport(
    #[case] model: &str,
    #[case] custom_llm_provider: Option<&str>,
) {
    let upstream = upstream([]).await;
    let result = responses_route(no_secrets())
        .execute(
            ResponsesCall {
                model: model.into(),
                api_base: Some(upstream.uri()),
                custom_llm_provider: custom_llm_provider.map(str::to_owned),
                ..call()
            },
            &(),
            None,
        )
        .await;

    assert!(matches!(result, Err(ref error) if error.declined().is_some()));
    assert!(received(&upstream).await.is_empty());
}

#[rstest]
#[case::openai_with_vendor_segment("openai/other/model", None)]
#[case::empty_model("openai/", None)]
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
#[tokio::test]
async fn invalid_provider_models_fail_before_transport(
    #[case] model: &str,
    #[case] custom_llm_provider: Option<&str>,
) {
    let upstream = upstream([]).await;
    let result = responses_route(no_secrets())
        .execute(
            ResponsesCall {
                model: model.into(),
                api_base: Some(upstream.uri()),
                custom_llm_provider: custom_llm_provider.map(str::to_owned),
                ..call()
            },
            &(),
            None,
        )
        .await;

    assert!(matches!(result, Err(Error::InvalidProvider(_))));
    assert!(received(&upstream).await.is_empty());
}

#[rstest]
#[case::direct(false)]
#[case::hosted(true)]
#[tokio::test]
async fn http_responses_share_execution_and_hooks(call: ResponsesCall, #[case] hosted: bool) {
    let body = json!({"id": "response-1", "model": "test-model", "output": [{"type":"message", "content":[]}], "usage":{"total_tokens":7}, "provider_extra": true});
    let upstream = upstream([json_response(body.clone())]).await;
    let host = RecordingCall::<Responses>::new(ResponsesCall {
        api_base: Some(upstream.uri()),
        ..call
    });
    let response = if hosted {
        let HostedCompletion::Complete(response) = litellm_host_native::in_process::run_hosted(
            responses_route(no_secrets())
                .machine(host.request().unwrap(), Some(host.events.0.sender.clone())),
            host.runtime(),
        )
        .await
        .unwrap() else {
            panic!()
        };
        response
    } else {
        let call = host.request.lock().unwrap().take().unwrap();
        let ResponsesOutput::Complete(response) = responses_route(no_secrets())
            .execute(call, &host, Some(host.events.0.sender.clone()))
            .await
            .unwrap()
        else {
            panic!()
        };
        response
    };
    assert_eq!(serde_json::to_value(response).unwrap(), body);
    let sent = only_request(&upstream).await;
    assert_eq!(sent.url.path(), "/responses");
    assert_eq!(sent.header("authorization"), Some("Bearer test-key"));
    assert_eq!(sent.header("x-hook"), Some("called"));
    assert_eq!(sent.json(), json!({"model":"test-model", "input":"hello"}));
    assert!(matches!(
        &host.events.0.lock().unwrap()[..],
        [
            CallEvent::Started { .. },
            CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { .. }),
            CallEvent::Execution(ExecutionEvent::ResultReady {
                facts: ExecutionFacts {
                    source: ResultSource::Provider,
                    ..
                }
            }),
            CallEvent::Succeeded { .. }
        ]
    ));
}

#[rstest]
#[case::direct(false)]
#[case::hosted(true)]
#[tokio::test]
async fn streaming_keeps_headers_and_bytes_and_finishes_after_consumption(
    call: ResponsesCall,
    #[case] hosted: bool,
) {
    let body = "event: response.completed\ndata: {\"type\":\"response.completed\"}\n\n";
    let upstream = upstream([ResponseTemplate::new(200)
        .insert_header("x-request-id", "response-stream")
        .set_body_raw(body, "text/event-stream")])
    .await;
    let host = RecordingCall::<Responses>::new(ResponsesCall {
        api_base: Some(upstream.uri()),
        optional_params: json!({"stream":true}).as_object().unwrap().clone(),
        ..call
    });
    let (headers, bytes) = if hosted {
        assert_eq!(
            litellm_host_native::in_process::run_hosted(
                responses_route(no_secrets())
                    .machine(host.request().unwrap(), Some(host.events.0.sender.clone())),
                host.runtime(),
            )
            .await
            .unwrap(),
            HostedCompletion::StreamEnded
        );
        (
            host.head.lock().unwrap().take().unwrap().headers,
            host.chunks.lock().unwrap().concat(),
        )
    } else {
        let call = host.request.lock().unwrap().take().unwrap();
        let ResponsesOutput::Stream { head, chunks } = responses_route(no_secrets())
            .execute(call, &host, Some(host.events.0.sender.clone()))
            .await
            .unwrap()
        else {
            panic!()
        };
        assert!(matches!(
            &host.events.0.lock().unwrap()[..],
            [
                CallEvent::Started { .. },
                CallEvent::Execution(ExecutionEvent::ResultReady {
                    facts: ExecutionFacts {
                        source: ResultSource::Provider,
                        ..
                    }
                }),
            ]
        ));
        (
            head.headers,
            chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
        )
    };
    assert!(headers.contains(&("x-request-id".into(), "response-stream".into())));
    assert_eq!(bytes, body.as_bytes());
    assert!(matches!(
        &host.events.0.lock().unwrap()[..],
        [
            CallEvent::Started { .. },
            CallEvent::Execution(ExecutionEvent::ResultReady {
                facts: ExecutionFacts {
                    source: ResultSource::Provider,
                    ..
                }
            }),
            CallEvent::Succeeded { .. }
        ]
    ));
}

#[rstest]
#[case::http(429, json!({"error":"limited"}))]
#[case::invalid_response(200, json!({"unexpected":true}))]
#[tokio::test]
async fn provider_failures_emit_failure_once(
    call: ResponsesCall,
    #[case] status: u16,
    #[case] body: serde_json::Value,
) {
    let upstream = upstream([ResponseTemplate::new(status).set_body_json(body)]).await;
    let host = RecordingCall::<Responses>::new(ResponsesCall {
        api_base: Some(upstream.uri()),
        ..call
    });
    let call = host.request.lock().unwrap().take().unwrap();
    assert!(
        responses_route(no_secrets())
            .execute(call, &host, Some(host.events.0.sender.clone()))
            .await
            .is_err()
    );
    assert_eq!(received(&upstream).await.len(), 1);
    let events = host.events.0.lock().unwrap();
    assert!(matches!(events.last(), Some(CallEvent::Failed { .. })));
    assert_eq!(
        events
            .iter()
            .filter(|event| matches!(
                event,
                CallEvent::Failed { .. } | CallEvent::Succeeded { .. }
            ))
            .count(),
        1
    );
}

#[rstest]
#[tokio::test]
async fn a_transport_error_does_not_echo_the_provider_url(call: ResponsesCall) {
    let host = RecordingCall::<Responses>::new(ResponsesCall {
        api_base: Some(format!("{UNREACHABLE_BASE}/tenant-token-1234")),
        ..call
    });
    let call = host.request.lock().unwrap().take().unwrap();

    let Err(error) = responses_route(no_secrets())
        .execute(call, &host, None)
        .await
    else {
        panic!("nothing is listening");
    };

    assert!(!error.to_string().contains("tenant-token-1234"), "{error}");
}

#[rstest]
#[case::explicit(true)]
#[case::from_secrets(false)]
#[tokio::test]
async fn credentials_and_endpoint_are_resolved_only_when_needed(
    call: ResponsesCall,
    #[case] explicit: bool,
) {
    let upstream = upstream([json_response(
        json!({"id":"response", "model":"test-model", "output":[]}),
    )])
    .await;
    let base = upstream.uri();
    let key = "resolved-test-key";
    let secrets = Arc::new(RecordingSecrets::new([
        ("OPENAI_API_KEY", key),
        ("OPENAI_BASE_URL", base.as_str()),
    ]));
    let call = ResponsesCall {
        api_key: explicit.then(|| key.into()),
        api_base: explicit.then(|| base.clone()),
        ..call
    };
    responses_route(secrets.clone())
        .execute(call, &(), None)
        .await
        .unwrap();
    assert_eq!(
        only_request(&upstream).await.header("authorization"),
        Some(format!("Bearer {key}").as_str())
    );
    if explicit {
        assert!(secrets.requested().is_empty());
    } else {
        assert!(secrets.requested().contains(&"OPENAI_API_KEY".into()));
        assert!(secrets.requested().contains(&"OPENAI_BASE_URL".into()));
    }
}

#[rstest]
#[case::provider("other/model", "other")]
#[case::conflicting_prefix("other/model", "openai")]
#[tokio::test]
async fn unsupported_providers_fail_before_secrets_or_transport(
    call: ResponsesCall,
    #[case] model: &str,
    #[case] provider: &str,
) {
    let upstream = upstream([]).await;
    let secrets = Arc::new(RecordingSecrets::failing());
    let call = ResponsesCall {
        model: model.into(),
        custom_llm_provider: Some(provider.into()),
        api_base: Some(upstream.uri()),
        ..call
    };
    assert!(
        responses_route(secrets.clone())
            .execute(call, &(), None)
            .await
            .is_err()
    );
    assert!(secrets.requested().is_empty());
    assert!(received(&upstream).await.is_empty());
}

#[rstest]
#[case::success(200, json!({"id":"response-1", "model":"test-model", "output":[]}), "success")]
#[case::invalid_response(200, json!("private-response-sentinel"), "failure")]
#[case::upstream_error(429, json!({"error":"private-response-sentinel"}), "failure")]
#[tokio::test]
async fn route_tracing_covers_native_and_hosted_outcomes(
    call: ResponsesCall,
    traces: TraceCapture,
    #[case] status: u16,
    #[case] body: serde_json::Value,
    #[case] outcome: &str,
    #[values(false, true)] hosted: bool,
) {
    let upstream = upstream([status_response(status, body)]).await;
    let host = RecordingCall::<Responses>::new(ResponsesCall {
        api_base: Some(upstream.uri()),
        input: json!("private-prompt-sentinel"),
        api_key: Some("private-key-sentinel".into()),
        ..call
    });
    let route = responses_route(no_secrets());
    let result = traces
        .logger()
        .instrument(async {
            if hosted {
                litellm_host_native::in_process::run_hosted(
                    route
                        .clone()
                        .machine(host.request().unwrap(), Some(host.events.0.sender.clone())),
                    host.runtime(),
                )
                .await
                .map(|_| ())
            } else {
                route
                    .execute(host.request().unwrap(), &(), None)
                    .await
                    .map(|_| ())
            }
        })
        .await;
    assert_eq!(result.is_ok(), outcome == "success");
    let summaries = traces.summaries("litellm.route");
    let [summary] = summaries.as_slice() else {
        panic!("expected one route summary: {summaries:?}")
    };
    assert_eq!(summary["route"], "responses");
    assert_eq!(summary["model"], "openai/test-model");
    assert_eq!(summary["resolved_model"], "test-model");
    assert_eq!(summary["provider"], "openai");
    assert_eq!(summary["stream"], false);
    assert_eq!(summary["outcome"], outcome);
    assert!(summary["duration_ms"].as_f64().unwrap() >= 0.0);
    let sends = traces.summaries("litellm.provider.send");
    assert_eq!(sends.len(), 1);
    assert_eq!(sends[0]["status"], status);
    assert!(!format!("{:?}", traces.records()).contains("private-"));
}

#[rstest]
#[case::exhausted(true, "success")]
#[case::dropped(false, "cancelled")]
#[tokio::test]
async fn stream_trace_survives_handoff_and_closes_before_the_stream_object_is_dropped(
    call: ResponsesCall,
    traces: TraceCapture,
    #[case] exhaust: bool,
    #[case] outcome: &str,
) {
    let upstream = upstream([ResponseTemplate::new(200).set_body_string("stream-bytes")]).await;
    let output = traces
        .logger()
        .instrument(async {
            responses_route(no_secrets())
                .execute(
                    ResponsesCall {
                        api_base: Some(upstream.uri()),
                        optional_params: json!({"stream":true}).as_object().unwrap().clone(),
                        ..call
                    },
                    &(),
                    None,
                )
                .await
        })
        .await
        .unwrap();
    assert!(traces.summaries("litellm.route").is_empty());
    let ResponsesOutput::Stream { mut chunks, .. } = output else {
        panic!("expected stream")
    };
    let captured = traces.clone();
    tokio::spawn(async move {
        if exhaust {
            while chunks.try_next().await.unwrap().is_some() {}
            assert_eq!(captured.summaries("litellm.route").len(), 1);
            assert!(chunks.try_next().await.unwrap().is_none());
        }
        drop(chunks);
    })
    .await
    .unwrap();
    let summaries = traces.summaries("litellm.route");
    assert_eq!(summaries.len(), 1);
    assert_eq!(summaries[0]["stream"], true);
    assert_eq!(summaries[0]["outcome"], outcome);
}

#[rstest]
#[tokio::test]
async fn preparation_failure_is_traced_but_unpolled_builders_are_not(
    call: ResponsesCall,
    traces: TraceCapture,
) {
    traces.logger().scope(|| {
        drop(responses_route(no_secrets()).execute(
            ResponsesCall {
                model: "unknown/model".into(),
                ..call
            },
            &(),
            None,
        ))
    });
    assert!(traces.records().is_empty());
    let result = traces
        .logger()
        .instrument(async {
            responses_route(no_secrets())
                .execute(
                    ResponsesCall {
                        model: "unknown/model".into(),
                        ..self::call()
                    },
                    &(),
                    None,
                )
                .await
        })
        .await;
    assert!(result.is_err());
    let summaries = traces.summaries("litellm.route");
    assert_eq!(summaries.len(), 1);
    assert_eq!(summaries[0]["outcome"], "failure");
    assert!(traces.summaries("litellm.provider.send").is_empty());
}

#[rstest]
#[tokio::test]
async fn websocket_operations_trace_outcomes_without_capturing_frames_or_credentials(
    traces: TraceCapture,
) {
    use futures_util::{SinkExt, StreamExt};
    use litellm_inference_responses::websocket::ResponsesWebSocketConnection;

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
    let address = listener.local_addr().unwrap();
    let server = tokio::spawn(async move {
        let (socket, _) = listener.accept().await.unwrap();
        let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
        let message = socket.next().await.unwrap().unwrap();
        socket.send(message).await.unwrap();
        let _ = socket.next().await;
    });
    traces
        .logger()
        .instrument(async {
            let connection = ResponsesWebSocketConnection::connect_url(
                &litellm_http::Client::for_test(),
                &format!("ws://{address}/responses"),
                &std::collections::HashMap::from([(
                    "authorization".into(),
                    "private-key-sentinel".into(),
                )]),
                None,
            )
            .await
            .unwrap();
            connection
                .send_text("private-frame-sentinel".into())
                .await
                .unwrap();
            assert_eq!(
                connection.recv_text().await.unwrap().as_deref(),
                Some("private-frame-sentinel")
            );
            connection.close().await.unwrap();
            let error = connection
                .send_text("private-frame-sentinel".into())
                .await
                .expect_err("a closed connection sends nothing");
            assert!(!error.dispatched(), "{error:?}");
        })
        .await;
    server.await.unwrap();
    let sends = traces.summaries("litellm.websocket.send_text");
    assert_eq!(sends.len(), 2);
    assert_eq!(sends[0]["outcome"], "success");
    assert_eq!(sends[1]["outcome"], "failure");
    assert_eq!(
        traces.summaries("litellm.websocket.connect_url")[0]["outcome"],
        "success"
    );
    assert_eq!(
        traces.summaries("litellm.websocket.recv_text")[0]["outcome"],
        "success"
    );
    assert_eq!(
        traces.summaries("litellm.websocket.close")[0]["outcome"],
        "success"
    );
    assert!(!format!("{:?}", traces.records()).contains("private-"));
}