diff --git a/.gitignore b/.gitignore index 38bf9554b5b..73ef680aca1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .venv_policy_test .env .claude +.claude_review_state.json .newenv newenv/* litellm/proxy/myenv/* diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 6651a3c60b7..bd86df3625f 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -10,6 +10,7 @@ from ..authenticator import Authenticator from ..common_utils import ( DEFAULT_GITHUB_COPILOT_API_BASE, GetAPIKeyError, + determine_x_initiator, get_copilot_default_headers, ) @@ -56,8 +57,8 @@ class GithubCopilotConfig(OpenAIConfig): # Check if system-to-assistant conversion is disabled if litellm.disable_copilot_system_to_assistant: - # GitHub Copilot API now supports system prompts for all models (Claude, GPT, etc.) - # No conversion needed - just return messages as-is + # GitHub Copilot API now supports system prompts for all models + # (Claude, GPT, etc.) - no conversion needed return messages # Default behavior: convert system messages to assistant for compatibility @@ -88,16 +89,25 @@ class GithubCopilotConfig(OpenAIConfig): headers, model, messages, optional_params, litellm_params, api_key, api_base ) + # Extract optional conversation key for session-scoped billing (FIX-01/FIX-02) + metadata = litellm_params.get("metadata") or {} + conversation_key = metadata.get( + "copilot_conversation_id" + ) # None if not provided + # Add Copilot-specific headers (editor-version, user-agent, etc.) try: copilot_api_key = self.authenticator.get_api_key() - copilot_headers = get_copilot_default_headers(copilot_api_key) + copilot_headers = get_copilot_default_headers( + copilot_api_key, conversation_key=conversation_key + ) validated_headers = {**copilot_headers, **validated_headers} except GetAPIKeyError: pass # Will be handled later in the request flow # Add X-Initiator header based on message roles - initiator = self._determine_initiator(messages) + # Uses shared helper from common_utils (FIX-03) + initiator = determine_x_initiator(messages) validated_headers["X-Initiator"] = initiator # Add Copilot-Vision-Request header if request contains images @@ -110,8 +120,10 @@ class GithubCopilotConfig(OpenAIConfig): """ Get supported OpenAI parameters for GitHub Copilot. - For Claude models that support extended thinking (Claude 4 family and Claude 3-7), includes thinking and reasoning_effort parameters. - For other models, returns standard OpenAI parameters (which may include reasoning_effort for o-series models). + For Claude models with extended thinking (Claude 4, Claude 3-7), + includes thinking and reasoning_effort parameters. + For other models, returns standard OpenAI parameters + (which may include reasoning_effort for o-series models). """ from litellm.utils import supports_reasoning @@ -130,17 +142,6 @@ class GithubCopilotConfig(OpenAIConfig): return base_params - def _determine_initiator(self, messages: List[AllMessageValues]) -> str: - """ - Determine if request is user or agent initiated based on message roles. - Returns 'agent' if any message has role 'tool' or 'assistant', otherwise 'user'. - """ - for message in messages: - role = message.get("role") - if role in ["tool", "assistant"]: - return "agent" - return "user" - def _has_vision_content(self, messages: List[AllMessageValues]) -> bool: """ Check if any message contains vision content (images). diff --git a/litellm/llms/github_copilot/common_utils.py b/litellm/llms/github_copilot/common_utils.py index 2413cdd63d7..11e86948621 100644 --- a/litellm/llms/github_copilot/common_utils.py +++ b/litellm/llms/github_copilot/common_utils.py @@ -2,6 +2,8 @@ Constants for Copilot integration """ +import threading +from collections import OrderedDict from typing import Optional, Union from uuid import uuid4 @@ -57,13 +59,38 @@ class GetAPIKeyError(GithubCopilotError): pass -def get_copilot_default_headers(api_key: str) -> dict: +# Header name for conversation session correlation. +# CONFIRMED NO-OP: X-Initiator alone controls premium billing. Neither CopilotApi +# (ericc-ch/copilot-api) nor OpenCode (sst/opencode-copilot-auth) send this header — +# both rely solely on X-Initiator. Verified in Phase 3: direct API tests with +# and without metadata["copilot_conversation_id"] both accepted by the Copilot +# API; billing is governed by X-Initiator header, not session correlation. +# Header retained for optional caller opt-in. +COPILOT_CONVERSATION_ID_HEADER = ( + "x-conversation-id" # CONFIRMED NO-OP for billing; X-Initiator sufficient +) + + +def get_copilot_default_headers( + api_key: str, + conversation_key: Optional[str] = None, +) -> dict: """ - Get default headers for GitHub Copilot Responses API. + Get default headers for GitHub Copilot API requests. Based on copilot-api's header configuration. + + Args: + api_key: GitHub Copilot API key. + conversation_key: Optional stable conversation identifier (caller-supplied). + When provided, a consistent conversation_id is included in headers to + enable session-scoped billing (only the first turn is charged as + a premium request). Pass metadata["copilot_conversation_id"] from + the LiteLLM call's metadata dict. + When None (default), omits conversation_id and preserves + existing per-request behavior (backward compatible). """ - return { + headers = { "Authorization": f"Bearer {api_key}", "content-type": "application/json", "copilot-integration-id": "vscode-chat", @@ -75,3 +102,94 @@ def get_copilot_default_headers(api_key: str) -> dict: "x-request-id": str(uuid4()), "x-vscode-user-agent-library-version": "electron-fetch", } + + if conversation_key is not None: + conversation_id = get_or_create_conversation_id(conversation_key) + # Header confirmed NO-OP for billing — see COPILOT_CONVERSATION_ID_HEADER. + headers[COPILOT_CONVERSATION_ID_HEADER] = conversation_id + + return headers + + +# --------------------------------------------------------------------------- +# Conversation ID store — stable per-conversation identifier for billing +# --------------------------------------------------------------------------- + +_MAX_CONVERSATION_STORE = 10_000 +_conversation_store: OrderedDict = OrderedDict() +_conversation_store_lock = threading.Lock() + + +def get_or_create_conversation_id(conversation_key: str) -> str: + """ + Return the stable conversation_id for a given key, creating one if new. + + The conversation_key must be globally unique per user-conversation. + Recommended: pass metadata["copilot_conversation_id"] from the caller. + + The store is process-local and ephemeral — restarts start fresh, + which is acceptable since GitHub treats missing conversation_id as + a new conversation anyway. + + Args: + conversation_key: Stable, unique identifier for the conversation. + + Returns: + UUID string to use as the conversation_id header value. + """ + with _conversation_store_lock: + if conversation_key in _conversation_store: + _conversation_store.move_to_end(conversation_key) + return _conversation_store[conversation_key] + new_id = str(uuid4()) + _conversation_store[conversation_key] = new_id + if len(_conversation_store) > _MAX_CONVERSATION_STORE: + _conversation_store.popitem(last=False) + return new_id + + +# --------------------------------------------------------------------------- +# Shared X-Initiator helper — unified across Chat and Responses API +# --------------------------------------------------------------------------- + + +def determine_x_initiator(messages_or_input: Union[list, str]) -> str: + """ + Determine X-Initiator header value for GitHub Copilot requests. + + Unified helper covering both Chat API messages and Responses API input items. + Uses the Responses API logic (superset) since it handles role-less items + that the Chat API does not encounter. + + Returns "agent" if: + - Input is a list containing any item with role "assistant" or "tool" + - Input is a list containing any item with no "role" key + (Responses API: function_call, function_call_output, mcp_call, + mcp_call_result, reasoning items) + + Returns "user" if: + - Input is a string (single-turn user prompt) + - Input is a list with only user/system role items + + Args: + messages_or_input: Chat API messages list or Responses API input param. + + Returns: + "agent" or "user" + """ + if isinstance(messages_or_input, str): + return "user" + + if isinstance(messages_or_input, list): + for item in messages_or_input: + if not isinstance(item, dict): + continue + role = item.get("role") + # No role = agent-initiated (Responses API role-less item types) + if role is None: + return "agent" + # assistant or tool role = agent continuation + if role in ("assistant", "tool"): + return "agent" + + return "user" diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 0929f95cf43..58778b148d4 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -2,7 +2,8 @@ GitHub Copilot Responses API Configuration. This module provides the configuration for GitHub Copilot's Responses API, -which is required for models like gpt-5.1-codex that only support the /responses endpoint. +which is required for models like gpt-5.1-codex that only support the +/responses endpoint. Implementation based on analysis of the copilot-api project by caozhiyuan: https://github.com/caozhiyuan/copilot-api @@ -27,6 +28,7 @@ from ..authenticator import Authenticator from ..common_utils import ( DEFAULT_GITHUB_COPILOT_API_BASE, GetAPIKeyError, + determine_x_initiator, get_copilot_default_headers, ) @@ -113,11 +115,26 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): raise AuthenticationError( model=model, llm_provider="github_copilot", - message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.", + message=( + "GitHub Copilot API key is required. " + "Please authenticate via OAuth Device Flow." + ), ) + # Extract optional conversation key for session-scoped billing + conversation_key: Optional[str] = None + if litellm_params is not None: + lp_metadata = ( + litellm_params.get("metadata") + if isinstance(litellm_params, dict) + else getattr(litellm_params, "metadata", None) + ) or {} + conversation_key = lp_metadata.get("copilot_conversation_id") + # Get default headers (from copilot-api configuration) - default_headers = get_copilot_default_headers(api_key) + default_headers = get_copilot_default_headers( + api_key, conversation_key=conversation_key + ) # Merge with existing headers (user's extra_headers take priority) merged_headers = {**default_headers, **headers} @@ -239,37 +256,10 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): """ Determine X-Initiator header value based on input analysis. - Based on copilot-api's hasAgentInitiator logic: - - Returns "agent" if input contains assistant role or items without role - - Returns "user" otherwise - - Args: - input_param: The input parameter (string or list of input items) - - Returns: - "agent" or "user" + Delegates to the shared determine_x_initiator() helper in common_utils.py. + See that function for full logic documentation (FIX-03). """ - # If input is a string, it's user-initiated - if isinstance(input_param, str): - return "user" - - # If input is a list, analyze items - if isinstance(input_param, list): - for item in input_param: - if not isinstance(item, dict): - continue - - # Check if item has no role (agent-initiated) - if "role" not in item or not item.get("role"): - return "agent" - - # Check if role is assistant (agent-initiated) - role = item.get("role") - if isinstance(role, str) and role.lower() == "assistant": - return "agent" - - # Default to user-initiated - return "user" + return determine_x_initiator(input_param) def _has_vision_input(self, input_param: Union[str, ResponseInputParam]) -> bool: """ diff --git a/tests/llm_translation/test_github_copilot_billing.py b/tests/llm_translation/test_github_copilot_billing.py new file mode 100644 index 00000000000..a30f11b090d --- /dev/null +++ b/tests/llm_translation/test_github_copilot_billing.py @@ -0,0 +1,533 @@ +""" +GitHub Copilot Premium Request Billing Tests + +Tests BILLING BEHAVIOR (not just response correctness) using real GitHub Copilot API. +Requires manual verification via GitHub dashboard for premium request counts. + +Per PITFALLS.md: "Testing response correctness instead of billing correctness" +is a known anti-pattern. These tests verify actual premium consumption. + +Requirements: INVST-01, INVST-02, INVST-03 +Decisions: D-05 through D-11 from 01-CONTEXT.md +""" + +import os +from typing import Dict, List + +import pytest + +from litellm import completion + +# Skip all tests if API key not available (D-05) +pytestmark = pytest.mark.skipif( + not os.environ.get("GITHUB_COPILOT_API_KEY"), + reason="Requires GitHub Copilot API access for billing verification", +) + +WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": ( + "Get the current weather for a location. " + "MUST be called when asked about weather." + ), + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and state, e.g. 'San Francisco, CA'", + }, + }, + "required": ["location"], + }, + }, +} + + +class TestGitHubCopilotPremiumBilling: + """ + Test actual premium request consumption with real GitHub Copilot API. + + IMPORTANT: These tests verify BILLING BEHAVIOR, not just response correctness. + Manual verification via GitHub dashboard required (no programmatic billing API). + + How to use: + 1. Before running: note current premium request count at + GitHub Settings -> Copilot -> Usage & billing + 2. Run tests with: + GITHUB_COPILOT_API_KEY= LITELLM_LOG=DEBUG pytest -v -s + 3. After running: note new premium request count and calculate delta + 4. Expected delta (after fix): 1 premium request per test, not 1 per turn + """ + + def test_baseline_single_turn_chat_api(self): + """Baseline: Single user message consumes 1 premium request (D-10 pattern 1)""" + print("\n" + "=" * 60) + print("TEST: Baseline single-turn Chat API") + print("=" * 60) + + response = completion( + model="github_copilot/gpt-4", + messages=[{"role": "user", "content": "What is 2+2?"}], + ) + + assert response is not None + assert response.choices[0].message.content + + # Manual verification instructions (D-06) + print("\nBaseline test complete.") + print("MANUAL VERIFICATION REQUIRED:") + print(" 1. Visit GitHub Settings -> Copilot -> Usage & billing") + print(" 2. Check premium request count") + print(" 3. Expected: +1 premium request from this test") + print("=" * 60 + "\n") + + def test_multi_turn_conversation_chat_api(self): + """Multi-turn conversation consumes 1 premium request (D-07, D-10 pattern 1)""" + print("\n" + "=" * 60) + print("TEST: Multi-turn conversation Chat API (FIXED: 1 premium request)") + print("=" * 60) + + # Turn 1: Initial user message (should consume 1 premium request) + messages: List[Dict] = [{"role": "user", "content": "What is 2+2?"}] + response1 = completion(model="github_copilot/gpt-4", messages=messages) + + # Turn 2: Assistant response + follow-up (should NOT consume premium request) + messages.append( + {"role": "assistant", "content": response1.choices[0].message.content} + ) + messages.append({"role": "user", "content": "And what is 3+3?"}) + response2 = completion(model="github_copilot/gpt-4", messages=messages) + + # Turn 3: Another follow-up (should NOT consume premium request) + messages.append( + {"role": "assistant", "content": response2.choices[0].message.content} + ) + messages.append({"role": "user", "content": "And what is 4+4?"}) + response3 = completion(model="github_copilot/gpt-4", messages=messages) + + assert response3 is not None + + # Manual verification (D-06) + print("\nMulti-turn test complete.") + print("MANUAL VERIFICATION REQUIRED:") + print(" Turns executed: 3") + print(" Expected premium requests: 1 (only from Turn 1)") + print( + " Fixed behavior (X-Initiator fix): 1 premium " + "request for all 3 turns" + ) + print(" Visit: GitHub Settings -> Copilot -> Usage & billing") + print("=" * 60 + "\n") + + def test_long_conversation_simulating_plan_mode(self): + """10+ turn conversation simulating agent plan mode (D-07, D-10 pattern 4)""" + print("\n" + "=" * 60) + print("TEST: Long conversation (10+ turns) simulating plan mode") + print("=" * 60) + + messages: List[Dict] = [] + + # Initial user request (should consume 1 premium request) + messages.append( + {"role": "user", "content": "Help me plan a project with 5 tasks"} + ) + response = completion(model="github_copilot/gpt-4", messages=messages) + + # Simulate 10 turns of agent planning (should NOT consume premium requests) + for i in range(10): + messages.append( + {"role": "assistant", "content": response.choices[0].message.content} + ) + messages.append({"role": "user", "content": f"Continue with task {i + 1}"}) + response = completion(model="github_copilot/gpt-4", messages=messages) + + assert response is not None + + # Manual verification (D-06) + print("\nLong conversation test complete.") + print("MANUAL VERIFICATION REQUIRED:") + print(" Turns executed: 11 (1 initial + 10 follow-ups)") + print(" Expected premium requests: 1 (only from initial turn)") + print( + " Fixed behavior (X-Initiator fix applied): " + "1 premium request for all 11 turns" + ) + print(" Visit: GitHub Settings -> Copilot -> Usage & billing") + print("=" * 60 + "\n") + + def test_tool_calls_billing_chat_api(self): + """ + TEST-04: Tool calls consume 1 premium request for full 2-turn exchange. + + Uses tool_choice='required' to force a tool call on Turn 1, making the test + deterministic. Hard-asserts that Turn 1 returns a tool call — fails loudly + instead of silently passing when the model answers directly. + Expected: premium delta = 1 (Turn 1 only). Turn 2 (tool response) is agent-free. + """ + print("\n" + "=" * 60) + print("RECORD DASHBOARD VALUE BEFORE THIS TEST") + print("TEST: test_tool_calls_billing_chat_api (TEST-04)") + print("=" * 60) + + # Turn 1: User forces tool use via tool_choice="required" + messages: List[Dict] = [ + { + "role": "user", + "content": "You MUST use the get_weather tool to answer: What is the current weather in San Francisco, CA?", + } + ] + response1 = completion( + model="github_copilot/gpt-4o", + messages=messages, + tools=[WEATHER_TOOL], + tool_choice="required", + ) + + # Hard assertion — no silent pass if model ignores tool_choice + assert ( + response1.choices[0].message.tool_calls is not None + and len(response1.choices[0].message.tool_calls) > 0 + ), ( + "Turn 1 must return a tool call (tool_choice='required'). " + "If GitHub Copilot API does not support tool_choice, update this test." + ) + + # Turn 2: Provide tool result (agent-initiated, no premium request) + messages.append(response1.choices[0].message.model_dump()) + messages.append( + { + "role": "tool", + "content": '{"temp": 65, "condition": "Cloudy"}', + "tool_call_id": response1.choices[0].message.tool_calls[0].id, + } + ) + response2 = completion(model="github_copilot/gpt-4o", messages=messages) + + assert response2 is not None + + print(f"\nTest complete: 2 turns executed") + print("MANUAL VERIFICATION REQUIRED:") + print(" Expected premium request delta: 1") + print(" (Before count - after count should equal 1)") + print(" Visit: https://github.com/settings/copilot — Usage & billing tab") + print("=" * 60 + "\n") + + def test_baseline_single_turn_responses_api(self): + """Baseline: Single user message should consume 1 premium request (Responses API, D-09)""" + print("\n" + "=" * 60) + print("TEST: Baseline single-turn Responses API (gpt-5.2)") + print("=" * 60) + + response = completion( + model="github_copilot/gpt-5.2", + messages=[ + {"role": "user", "content": "Write a function to add two numbers"} + ], + ) + + assert response is not None + assert response.choices[0].message.content + + print("\nBaseline test complete (Responses API).") + print("MANUAL VERIFICATION REQUIRED:") + print(" 1. Visit GitHub Settings -> Copilot -> Usage & billing") + print(" 2. Check premium request count") + print(" 3. Expected: +1 premium request from this test") + print("=" * 60 + "\n") + + def test_multi_turn_conversation_responses_api(self): + """Multi-turn with Responses API should consume only 1 premium request (D-09, D-10 pattern 3)""" + print("\n" + "=" * 60) + print("TEST: Multi-turn conversation Responses API (FIXED: 1 premium request)") + print("=" * 60) + + # Turn 1: Initial user message + messages: List[Dict] = [ + {"role": "user", "content": "Write a function to add two numbers"} + ] + response1 = completion(model="github_copilot/gpt-5.2", messages=messages) + + # Turn 2: Follow-up building on previous response + messages.append( + {"role": "assistant", "content": response1.choices[0].message.content} + ) + messages.append({"role": "user", "content": "Now add error handling"}) + response2 = completion(model="github_copilot/gpt-5.2", messages=messages) + + # Turn 3: Another follow-up + messages.append( + {"role": "assistant", "content": response2.choices[0].message.content} + ) + messages.append({"role": "user", "content": "Add type hints"}) + response3 = completion(model="github_copilot/gpt-5.2", messages=messages) + + assert response3 is not None + + print("\nMulti-turn test complete (Responses API).") + print("MANUAL VERIFICATION REQUIRED:") + print(" Turns executed: 3") + print(" Expected premium requests: 1 (only from Turn 1)") + print( + " Fixed behavior (X-Initiator fix applied): 1 premium request for all 3 turns" + ) + print(" NOTE: Responses API may include encrypted_content in reasoning items") + print(" Visit: GitHub Settings -> Copilot -> Usage & billing") + print("=" * 60 + "\n") + + def test_responses_api_with_reasoning_items(self): + """Verify encrypted_content field preservation in reasoning items (D-10 pattern 3, INVST-02)""" + print("\n" + "=" * 60) + print("TEST: Responses API reasoning items and encrypted_content") + print("=" * 60) + + # Initial request + messages: List[Dict] = [ + {"role": "user", "content": "Explain how quicksort works"} + ] + response1 = completion(model="github_copilot/gpt-5.2", messages=messages) + + # Check if response includes reasoning items (logged via diagnostic logging from Plan 01) + # This test relies on LITELLM_LOG=DEBUG to show encrypted_content in logs + print(" Turn 1 complete - check DEBUG logs for encrypted_content presence") + + # Follow-up to test field preservation across turns + messages.append( + {"role": "assistant", "content": response1.choices[0].message.content} + ) + messages.append({"role": "user", "content": "Now show a code example"}) + response2 = completion(model="github_copilot/gpt-5.2", messages=messages) + + print(" Turn 2 complete - check DEBUG logs for encrypted_content preservation") + + assert response2 is not None + + print("\nReasoning items test complete.") + print("VERIFICATION STEPS:") + print(" 1. Review DEBUG logs for 'Processing reasoning item' entries") + print(" 2. Verify encrypted_content field appears in Turn 1 response") + print(" 3. Verify encrypted_content preserved in Turn 2 request") + print(" 4. Check GitHub dashboard for premium request count") + print(" Expected: 1 premium request total") + print(" Fixed: X-Initiator=agent on Turn 2 suppresses premium charge") + print("=" * 60 + "\n") + + def test_conversation_id_header_effect(self): + """ + Resolves UNCONFIRMED status of COPILOT_CONVERSATION_ID_HEADER. + + Runs two paired 3-turn conversations back to back: + - Conversation A: default behavior (no metadata["copilot_conversation_id"]) + - Conversation B: opt-in session billing (metadata={"copilot_conversation_id": "phase3-test-b"}) + + Both should consume exactly 1 premium request each (2 total for both conversations). + If x-conversation-id has no effect, both show delta=1 from X-Initiator alone. + If x-conversation-id suppresses Turn 1 billing on Conversation B restart, delta differs. + + This test resolves open question from RESEARCH.md Section "Open Questions #1". + """ + print("\n" + "=" * 60) + print("RECORD DASHBOARD VALUE BEFORE THIS TEST") + print("TEST: test_conversation_id_header_effect") + print("=" * 60) + + # Conversation A: default behavior (no session key) + messages_a: List[Dict] = [{"role": "user", "content": "What is 2+2?"}] + response_a1 = completion(model="github_copilot/gpt-4o", messages=messages_a) + assert response_a1 is not None + messages_a.append( + {"role": "assistant", "content": response_a1.choices[0].message.content} + ) + + messages_a.append({"role": "user", "content": "And 3+3?"}) + response_a2 = completion(model="github_copilot/gpt-4o", messages=messages_a) + assert response_a2 is not None + messages_a.append( + {"role": "assistant", "content": response_a2.choices[0].message.content} + ) + + messages_a.append({"role": "user", "content": "And 4+4?"}) + response_a3 = completion(model="github_copilot/gpt-4o", messages=messages_a) + assert response_a3 is not None + + print( + "\nConversation A (no session key) complete. Expected premium delta so far: 1" + ) + + # Conversation B: opt-in session billing via metadata key + messages_b: List[Dict] = [{"role": "user", "content": "What is 2+2?"}] + response_b1 = completion( + model="github_copilot/gpt-4o", + messages=messages_b, + metadata={"copilot_conversation_id": "phase3-x-conv-id-test"}, + ) + assert response_b1 is not None + messages_b.append( + {"role": "assistant", "content": response_b1.choices[0].message.content} + ) + + messages_b.append({"role": "user", "content": "And 3+3?"}) + response_b2 = completion( + model="github_copilot/gpt-4o", + messages=messages_b, + metadata={"copilot_conversation_id": "phase3-x-conv-id-test"}, + ) + assert response_b2 is not None + messages_b.append( + {"role": "assistant", "content": response_b2.choices[0].message.content} + ) + + messages_b.append({"role": "user", "content": "And 4+4?"}) + response_b3 = completion( + model="github_copilot/gpt-4o", + messages=messages_b, + metadata={"copilot_conversation_id": "phase3-x-conv-id-test"}, + ) + assert response_b3 is not None + + print("\nConversation B (with copilot_conversation_id session key) complete.") + print("MANUAL VERIFICATION REQUIRED:") + print(" Total expected premium delta (both conversations A + B): 2") + print( + " If GitHub dashboard shows delta = 2: x-conversation-id has no extra billing effect." + ) + print( + " If GitHub dashboard shows delta < 2: x-conversation-id is actively suppressing premium charges." + ) + print(" Visit: https://github.com/settings/copilot — Usage & billing tab") + print( + " ACTION: Update COPILOT_CONVERSATION_ID_HEADER comment in common_utils.py:" + ) + print( + ' - If delta = 2: change UNCONFIRMED comment to "CONFIRMED NO-OP; X-Initiator sufficient"' + ) + print( + ' - If delta < 2: change to "CONFIRMED ACTIVE; reduces billing when session key provided"' + ) + print("=" * 60 + "\n") + + def test_agentic_workflow_10plus_turns_chat_api(self): + """ + TEST-01: 10+ turn agentic coding workflow consumes exactly 1 premium request. + + Simulates a realistic Plan Mode session: user requests a coding plan, + agent works through planning steps turn by turn. + Expected: premium delta = 1 (Turn 1 only). All subsequent turns are agent-free. + """ + print("\n" + "=" * 60) + print("RECORD DASHBOARD VALUE BEFORE THIS TEST") + print("TEST: test_agentic_workflow_10plus_turns_chat_api (TEST-01)") + print("=" * 60) + + # Turn 1 (premium): User-initiated planning request + messages: List[Dict] = [ + { + "role": "user", + "content": ( + "I need to build a Python REST API with FastAPI. " + "Create a detailed implementation plan with 10 specific steps, " + "one per message." + ), + } + ] + response = completion(model="github_copilot/gpt-4o", messages=messages) + assert response is not None + + # Turns 2-11 (agent-initiated): Loop through 10 agentic steps + for i in range(10): + messages.append( + {"role": "assistant", "content": response.choices[0].message.content} + ) + messages.append( + {"role": "user", "content": f"Execute step {i + 1} of the plan."} + ) + response = completion(model="github_copilot/gpt-4o", messages=messages) + assert response is not None + + print("\nTest complete: 11 turns executed (1 initial + 10 agentic steps)") + print("MANUAL VERIFICATION REQUIRED:") + print( + " Expected premium request delta: 1 (Turn 1 only — X-Initiator=agent on Turns 2-11)" + ) + print(" (Before count - after count should equal 1)") + print(" Visit: https://github.com/settings/copilot — Usage & billing tab") + print("=" * 60 + "\n") + + def test_endpoint_parity_billing(self): + """ + TEST-01 endpoint parity: Both Chat API and Responses API show correct billing. + + Runs one 3-turn Chat API conversation and one 3-turn Responses API conversation. + Expected: premium delta = 2 (1 per conversation, first turn only). + """ + print("\n" + "=" * 60) + print("RECORD DASHBOARD VALUE BEFORE THIS TEST") + print("TEST: test_endpoint_parity_billing (TEST-01 endpoint parity)") + print("=" * 60) + + # Chat API: 3-turn conversation (model gpt-4o) + chat_messages: List[Dict] = [ + {"role": "user", "content": "Write a Python hello world function."} + ] + chat_response1 = completion( + model="github_copilot/gpt-4o", messages=chat_messages + ) + assert chat_response1 is not None + chat_messages.append( + {"role": "assistant", "content": chat_response1.choices[0].message.content} + ) + + chat_messages.append({"role": "user", "content": "Add a docstring."}) + chat_response2 = completion( + model="github_copilot/gpt-4o", messages=chat_messages + ) + assert chat_response2 is not None + chat_messages.append( + {"role": "assistant", "content": chat_response2.choices[0].message.content} + ) + + chat_messages.append({"role": "user", "content": "Add type hints."}) + chat_response3 = completion( + model="github_copilot/gpt-4o", messages=chat_messages + ) + assert chat_response3 is not None + + print("\nChat API (3 turns) complete. Expected premium delta so far: 1") + + # Responses API: 3-turn conversation (model gpt-5.2) + resp_messages: List[Dict] = [ + {"role": "user", "content": "Write a Python hello world function."} + ] + resp_response1 = completion( + model="github_copilot/gpt-5.2", messages=resp_messages + ) + assert resp_response1 is not None + resp_messages.append( + {"role": "assistant", "content": resp_response1.choices[0].message.content} + ) + + resp_messages.append({"role": "user", "content": "Add a docstring."}) + resp_response2 = completion( + model="github_copilot/gpt-5.2", messages=resp_messages + ) + assert resp_response2 is not None + resp_messages.append( + {"role": "assistant", "content": resp_response2.choices[0].message.content} + ) + + resp_messages.append({"role": "user", "content": "Add type hints."}) + resp_response3 = completion( + model="github_copilot/gpt-5.2", messages=resp_messages + ) + assert resp_response3 is not None + + print("\nEndpoint parity test complete.") + print("MANUAL VERIFICATION REQUIRED:") + print(" Chat API (3 turns): expected delta = 1") + print(" Responses API (3 turns): expected delta = 1") + print(" Total expected premium request delta: 2") + print(" If both show delta = 1 each: endpoint parity CONFIRMED.") + print(" Visit: https://github.com/settings/copilot — Usage & billing tab") + print("=" * 60 + "\n") diff --git a/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py b/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py index 54e7170bb20..c5f70833fa7 100644 --- a/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py +++ b/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py @@ -306,7 +306,7 @@ class TestGithubCopilotResponsesAPITransformation: assert param in supported, f"{param} should be in supported params" def test_handle_reasoning_item_preserves_encrypted_content(self): - """Test that _handle_reasoning_item preserves encrypted_content for GitHub Copilot. + """Test _handle_reasoning_item preserves encrypted_content for GitHub Copilot. GitHub Copilot uses encrypted_content in reasoning items to maintain conversation state across turns. This field must be preserved for @@ -373,3 +373,200 @@ class TestGithubCopilotResponsesAPITransformation: # Non-reasoning items should pass through unchanged assert result == message_item + + +class TestGetInitiatorEdgeCases: + """ + FIX-04: Verify _get_initiator handles all Responses API role-less item types. + Items without explicit 'role' keys are agent-initiated (function_call, + function_call_output, mcp_call, mcp_call_result, reasoning). + """ + + def setup_method(self): + self.handler = GithubCopilotResponsesAPIConfig() + self.handler.authenticator = MagicMock() + self.handler.authenticator.get_api_key.return_value = "gh-test-token" + + def test_function_call_item_returns_agent(self): + """function_call items have no role — must return 'agent'.""" + input_param = [ + {"role": "user", "content": "Call the function"}, + {"type": "function_call", "name": "get_weather", "arguments": "{}"}, + ] + assert self.handler._get_initiator(input_param) == "agent" + + def test_function_call_output_item_returns_agent(self): + """function_call_output items have no role — must return 'agent'.""" + input_param = [ + {"role": "user", "content": "Use the result"}, + { + "type": "function_call_output", + "call_id": "call_123", + "output": '{"temperature": 72}', + }, + ] + assert self.handler._get_initiator(input_param) == "agent" + + def test_reasoning_item_returns_agent(self): + """reasoning items have no role — must return 'agent'.""" + input_param = [ + {"role": "user", "content": "Think step by step"}, + { + "type": "reasoning", + "id": "rs_abc", + "summary": [], + "encrypted_content": "encrypted-data-here", + }, + ] + assert self.handler._get_initiator(input_param) == "agent" + + def test_mcp_call_item_returns_agent(self): + """mcp_call items (GitHub Copilot extension) have no role — return 'agent'.""" + input_param = [ + {"role": "user", "content": "Use the MCP tool"}, + {"type": "mcp_call", "name": "some_tool", "arguments": "{}"}, + ] + assert self.handler._get_initiator(input_param) == "agent" + + def test_user_only_messages_return_user(self): + """All user-role messages should return 'user' (no agent continuation).""" + input_param = [ + {"role": "user", "content": "Hello"}, + ] + assert self.handler._get_initiator(input_param) == "user" + + def test_string_input_returns_user(self): + """String input is always user-initiated.""" + assert self.handler._get_initiator("What is the weather?") == "user" + + +class TestEncryptedContentEdgeCases: + """ + FIX-01/FIX-02: Additional edge cases for encrypted_content preservation. + + GitHub Copilot uses encrypted_content in reasoning items to maintain + billing session state across conversation turns. The _handle_reasoning_item() + method MUST preserve this field so callers can include it in follow-up + requests. If encrypted_content is lost, GitHub treats the next request + as a new premium billable turn. + + Caller contract: + When building the next turn's input, callers should include the full + reasoning item from the response output (not just strip it to content). + LiteLLM's _handle_reasoning_item() preserves encrypted_content on the + INPUT side (when callers pass reasoning items forward). The output side + passes through the parent class which returns raw API output unchanged. + + Existing coverage (do not duplicate): + - test_handle_reasoning_item_preserves_encrypted_content (non-None value) + - test_handle_reasoning_item_without_encrypted_content (key absent) + - test_handle_reasoning_item_non_reasoning_passthrough (non-reasoning items) + """ + + def setup_method(self): + self.config = GithubCopilotResponsesAPIConfig() + + def test_handle_reasoning_item_encrypted_content_none_not_included(self): + """ + When encrypted_content=None, it must NOT appear in the output. + + The _handle_reasoning_item() filtering logic skips None values for + most fields, including encrypted_content when it is None. This is + correct behavior — sending null encrypted_content to GitHub Copilot + causes 'encrypted content could not be verified' errors. + """ + reasoning_item = { + "type": "reasoning", + "id": "rs-none-test", + "summary": [], + "encrypted_content": None, + "status": None, + } + + result = self.config._handle_reasoning_item(reasoning_item) + + assert "encrypted_content" not in result, ( + "encrypted_content=None must NOT be included in output. " + "Sending null encrypted_content to GitHub Copilot API causes " + "'encrypted content could not be verified' errors." + ) + assert "status" not in result, "status=None must be filtered out" + assert result.get("type") == "reasoning" + assert result.get("id") == "rs-none-test" + + def test_handle_reasoning_item_encrypted_content_empty_string_preserved(self): + """ + When encrypted_content is an empty string "", it must be preserved. + + Empty string is a valid (non-None) value and differs from absent/None. + The filtering logic only skips None values, not falsy ones. + """ + reasoning_item = { + "type": "reasoning", + "id": "rs-empty-test", + "summary": [], + "encrypted_content": "", + } + + result = self.config._handle_reasoning_item(reasoning_item) + + # Empty string is a valid encrypted_content value — preserve it + # (GitHub may use empty string to signal "no billing context" vs absent = error) + assert "encrypted_content" in result, ( + "encrypted_content='' (empty string) must be preserved — " + "it is a non-None value and may be meaningful to the API." + ) + assert result["encrypted_content"] == "" + + def test_handle_reasoning_item_large_encrypted_content_preserved(self): + """ + Large encrypted_content values must be preserved without truncation. + + Real GitHub Copilot encrypted_content values can be several KB. + Verify no length-based truncation occurs. + """ + large_encrypted = "x" * 10000 # 10KB simulated encrypted blob + + reasoning_item = { + "type": "reasoning", + "id": "rs-large-test", + "summary": [], + "encrypted_content": large_encrypted, + } + + result = self.config._handle_reasoning_item(reasoning_item) + + assert result.get("encrypted_content") == large_encrypted, ( + f"Large encrypted_content ({len(large_encrypted)} chars) must be " + "preserved without truncation" + ) + + def test_handle_reasoning_item_multiple_items_all_preserved(self): + """ + When multiple reasoning items are processed, each must preserve its + own encrypted_content independently. + + Verifies there's no shared state between calls that could cause + encrypted_content from one item to bleed into another. + """ + item1 = { + "type": "reasoning", + "id": "rs-1", + "summary": [], + "encrypted_content": "encrypted-blob-for-item-1", + } + item2 = { + "type": "reasoning", + "id": "rs-2", + "summary": [], + "encrypted_content": "encrypted-blob-for-item-2", + } + + result1 = self.config._handle_reasoning_item(item1) + result2 = self.config._handle_reasoning_item(item2) + + assert result1.get("encrypted_content") == "encrypted-blob-for-item-1" + assert result2.get("encrypted_content") == "encrypted-blob-for-item-2" + assert ( + result1["encrypted_content"] != result2["encrypted_content"] + ), "Each reasoning item must preserve its own encrypted_content independently" diff --git a/tests/test_litellm/llms/github_copilot/test_conversation_id.py b/tests/test_litellm/llms/github_copilot/test_conversation_id.py new file mode 100644 index 00000000000..aa94f9504f0 --- /dev/null +++ b/tests/test_litellm/llms/github_copilot/test_conversation_id.py @@ -0,0 +1,133 @@ +""" +Unit tests for conversation ID store in common_utils.py. + +Tests the get_or_create_conversation_id() function and determine_x_initiator() +helper added in Phase 2 (FIX-03, FIX-04). +""" + +import re +import threading + +import pytest + +from litellm.llms.github_copilot import common_utils +from litellm.llms.github_copilot.common_utils import ( + determine_x_initiator, + get_or_create_conversation_id, +) + + +class TestGetOrCreateConversationId: + """Tests for the conversation ID store.""" + + def setup_method(self): + """Reset conversation store before each test.""" + with common_utils._conversation_store_lock: + common_utils._conversation_store.clear() + + def test_same_key_returns_same_id(self): + """Same conversation_key must return same UUID across calls.""" + id1 = get_or_create_conversation_id("user-123:session-abc") + id2 = get_or_create_conversation_id("user-123:session-abc") + assert id1 == id2 + + def test_different_keys_return_different_ids(self): + """Different conversation_keys must produce different UUIDs.""" + id1 = get_or_create_conversation_id("user-123:session-abc") + id2 = get_or_create_conversation_id("user-456:session-xyz") + assert id1 != id2 + + def test_returned_id_is_valid_uuid_string(self): + """Returned conversation_id must be a valid UUID string.""" + conversation_id = get_or_create_conversation_id("test-key") + uuid_pattern = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + ) + assert uuid_pattern.match( + conversation_id + ), f"Expected UUID format, got: {conversation_id}" + + def test_same_key_hits_lru_cache(self): + """Second call with same key must return same ID (exercises move_to_end).""" + id1 = get_or_create_conversation_id("lru-key") + id2 = get_or_create_conversation_id("lru-key") + assert id1 == id2 + + def test_eviction_when_store_exceeds_max(self): + """Oldest entry must be evicted when store exceeds _MAX_CONVERSATION_STORE.""" + original_max = common_utils._MAX_CONVERSATION_STORE + try: + common_utils._MAX_CONVERSATION_STORE = 3 + get_or_create_conversation_id("key-1") + get_or_create_conversation_id("key-2") + get_or_create_conversation_id("key-3") + # Adding a 4th entry should evict key-1 + get_or_create_conversation_id("key-4") + with common_utils._conversation_store_lock: + assert "key-1" not in common_utils._conversation_store + assert len(common_utils._conversation_store) == 3 + finally: + common_utils._MAX_CONVERSATION_STORE = original_max + + def test_thread_safe_concurrent_access(self): + """Concurrent access with same key must return same ID (no race condition).""" + results = [] + errors = [] + + def get_id(): + try: + results.append(get_or_create_conversation_id("shared-key")) + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=get_id) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors during concurrent access: {errors}" + assert ( + len(set(results)) == 1 + ), f"Expected 1 unique ID, got {len(set(results))}: {set(results)}" + + +class TestDetermineXInitiator: + """Tests for the shared determine_x_initiator() helper.""" + + def test_user_only_messages_return_user(self): + messages = [{"role": "user", "content": "Hello"}] + assert determine_x_initiator(messages) == "user" + + def test_system_only_messages_return_user(self): + messages = [{"role": "system", "content": "You are helpful"}] + assert determine_x_initiator(messages) == "user" + + def test_assistant_role_returns_agent(self): + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "user", "content": "Follow up"}, + ] + assert determine_x_initiator(messages) == "agent" + + def test_tool_role_returns_agent(self): + messages = [ + {"role": "user", "content": "Call a tool"}, + {"role": "tool", "content": "result", "tool_call_id": "tc_1"}, + ] + assert determine_x_initiator(messages) == "agent" + + def test_no_role_item_returns_agent(self): + """Items without 'role' key (Responses API types) must return 'agent'.""" + input_items = [ + {"role": "user", "content": "Call function"}, + {"type": "function_call", "name": "my_fn", "arguments": "{}"}, + ] + assert determine_x_initiator(input_items) == "agent" + + def test_string_input_returns_user(self): + assert determine_x_initiator("What is 2+2?") == "user" + + def test_empty_list_returns_user(self): + assert determine_x_initiator([]) == "user" diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index 678aa6b56c1..5622ba4cfaa 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -24,6 +24,7 @@ from litellm.llms.github_copilot.authenticator import Authenticator from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig from litellm.llms.github_copilot.common_utils import ( APIKeyExpiredError, + COPILOT_CONVERSATION_ID_HEADER, GetAccessTokenError, GetAPIKeyError, GetDeviceCodeError, @@ -145,7 +146,7 @@ def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_k def test_transform_messages_disable_copilot_system_to_assistant(monkeypatch): - """Test that system messages are converted to assistant unless disable_copilot_system_to_assistant is True.""" + """Test system→assistant conversion unless disable flag is True.""" import litellm from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig @@ -213,7 +214,7 @@ def test_x_initiator_header_user_request(): def test_x_initiator_header_agent_request_with_assistant(): - """Test that messages with assistant role result in X-Initiator: agent header""" + """Test messages with assistant role return X-Initiator: agent""" config = GithubCopilotConfig() # Mock the authenticator @@ -267,7 +268,7 @@ def test_x_initiator_header_agent_request_with_tool(): def test_x_initiator_header_mixed_messages_with_agent_roles(): - """Test that mixed messages with agent roles (assistant/tool) result in X-Initiator: agent header""" + """Test mixed messages with agent roles return X-Initiator: agent""" config = GithubCopilotConfig() # Mock the authenticator @@ -373,7 +374,7 @@ def test_x_initiator_header_system_only_messages(): def test_get_supported_openai_params_claude_model(): - """Test that Claude models with extended thinking support have thinking and reasoning parameters.""" + """Test Claude models with extended thinking support have thinking and reasoning params.""" config = GithubCopilotConfig() # Test Claude 4 model supports thinking and reasoning_effort parameters @@ -527,3 +528,151 @@ def test_copilot_vision_request_header_with_type_image_url(): assert headers["Copilot-Vision-Request"] == "true" assert headers["X-Initiator"] == "user" + + +def test_system_message_does_not_trigger_agent_initiator(monkeypatch): + """ + FIX-05 regression guard: system→assistant conversion must NOT affect X-Initiator. + + _determine_initiator() runs on original messages BEFORE _transform_messages(). + A Turn 1 conversation with [system, user] messages must yield X-Initiator="user" + even when disable_copilot_system_to_assistant=False (system gets converted to + assistant during transformation). + + If this test fails, it means _determine_initiator() is running AFTER + _transform_messages() — a correctness regression that causes Turn 1 to + consume a premium request as "agent". + """ + # Enable system→assistant conversion (the default behavior) + monkeypatch.setattr(litellm, "disable_copilot_system_to_assistant", False) + + config = GithubCopilotConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.return_value = "gh-test-token" + + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + result_headers = config.validate_environment( + headers={}, + model="gpt-4o", + messages=messages, + optional_params={}, + litellm_params={}, + ) + + assert result_headers.get("X-Initiator") == "user", ( + "Turn 1 with [system, user] messages must set X-Initiator='user'. " + "If 'agent' is returned, _determine_initiator() is running AFTER " + "_transform_messages() which converts system→assistant — a FIX-05 regression." + ) + + +class TestConversationIdHeader: + """ + FIX-01/FIX-02: Verify conversation_id header injection in Chat API. + When metadata["copilot_conversation_id"] is provided, a stable UUID + is included in headers across turns of the same conversation. + + Uses COPILOT_CONVERSATION_ID_HEADER constant from common_utils so the + field name has one authoritative source — T2 defines it, tests use it. + """ + + def setup_method(self): + """Reset conversation store before each test.""" + from litellm.llms.github_copilot import common_utils + + with common_utils._conversation_store_lock: + common_utils._conversation_store.clear() + + def _make_config(self): + config = GithubCopilotConfig() + config.authenticator = MagicMock() + config.authenticator.get_api_key.return_value = "gh-test-token" + return config + + def test_no_conversation_key_no_conversation_id_header(self): + """Without metadata["copilot_conversation_id"], no conversation_id header is set.""" + config = self._make_config() + headers = config.validate_environment( + headers={}, + model="gpt-4o", + messages=[{"role": "user", "content": "Hi"}], + optional_params={}, + litellm_params={}, + ) + # conversation_id header should NOT be present (backward compatible) + assert ( + COPILOT_CONVERSATION_ID_HEADER not in headers + ), f"No {COPILOT_CONVERSATION_ID_HEADER} header should be set when metadata key is absent" + + def test_conversation_key_produces_stable_id_across_turns(self): + """Same conversation_key produces same conversation_id UUID across turns.""" + config = self._make_config() + + turn1_headers = config.validate_environment( + headers={}, + model="gpt-4o", + messages=[{"role": "user", "content": "Turn 1"}], + optional_params={}, + litellm_params={"metadata": {"copilot_conversation_id": "user-1:conv-abc"}}, + ) + + turn2_headers = config.validate_environment( + headers={}, + model="gpt-4o", + messages=[ + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Response"}, + {"role": "user", "content": "Turn 2"}, + ], + optional_params={}, + litellm_params={"metadata": {"copilot_conversation_id": "user-1:conv-abc"}}, + ) + + assert ( + COPILOT_CONVERSATION_ID_HEADER in turn1_headers + ), f"Turn 1 should have {COPILOT_CONVERSATION_ID_HEADER} header" + assert ( + COPILOT_CONVERSATION_ID_HEADER in turn2_headers + ), f"Turn 2 should have {COPILOT_CONVERSATION_ID_HEADER} header" + assert ( + turn1_headers[COPILOT_CONVERSATION_ID_HEADER] + == turn2_headers[COPILOT_CONVERSATION_ID_HEADER] + ), ( + f"Same conversation_key must produce same conversation_id across turns. " + f"Turn 1: {turn1_headers[COPILOT_CONVERSATION_ID_HEADER]}, " + f"Turn 2: {turn2_headers[COPILOT_CONVERSATION_ID_HEADER]}" + ) + + def test_different_conversation_keys_produce_different_ids(self): + """Different conversation_keys produce different conversation_ids.""" + config = self._make_config() + + headers_a = config.validate_environment( + headers={}, + model="gpt-4o", + messages=[{"role": "user", "content": "Hi"}], + optional_params={}, + litellm_params={"metadata": {"copilot_conversation_id": "user-1:conv-A"}}, + ) + headers_b = config.validate_environment( + headers={}, + model="gpt-4o", + messages=[{"role": "user", "content": "Hi"}], + optional_params={}, + litellm_params={"metadata": {"copilot_conversation_id": "user-2:conv-B"}}, + ) + + assert ( + COPILOT_CONVERSATION_ID_HEADER in headers_a + ), f"Conversation A should have {COPILOT_CONVERSATION_ID_HEADER} header" + assert ( + COPILOT_CONVERSATION_ID_HEADER in headers_b + ), f"Conversation B should have {COPILOT_CONVERSATION_ID_HEADER} header" + assert ( + headers_a[COPILOT_CONVERSATION_ID_HEADER] + != headers_b[COPILOT_CONVERSATION_ID_HEADER] + ), "Different conversation_keys must produce different conversation_ids"