From 09463bc5840808d326895bf32a5813659f0787a9 Mon Sep 17 00:00:00 2001 From: Caixiaopig Date: Fri, 5 Apr 2024 14:44:40 +0800 Subject: [PATCH 1/2] Updating the default Anthropic Claude 3 max_tokens to 4096 The default value of max_tokens used to be 256. If the client does not set a larger value, the model's output may be truncated, so the default value has been changed to 4096. This value is also the maximum output value described in the official interface. see: https://docs.anthropic.com/claude/reference/messages_post --- litellm/llms/anthropic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 864ad658fbb..080dbd259b7 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -37,12 +37,12 @@ class AnthropicError(Exception): class AnthropicConfig: """ - Reference: https://docs.anthropic.com/claude/reference/complete_post + Reference: https://docs.anthropic.com/claude/reference/messages_post to pass metadata to anthropic, it's {"user_id": "any-relevant-information"} """ - max_tokens: Optional[int] = litellm.max_tokens # anthropic requires a default + max_tokens: Optional[int] = 4096 # anthropic requires a default value (Opus, Sonnet, and Haiku have the same default) stop_sequences: Optional[list] = None temperature: Optional[int] = None top_p: Optional[int] = None @@ -52,7 +52,7 @@ class AnthropicConfig: def __init__( self, - max_tokens: Optional[int] = 256, # anthropic requires a default + max_tokens: Optional[int] = None, # You can pass in a value yourself or use the default value 4096 stop_sequences: Optional[list] = None, temperature: Optional[int] = None, top_p: Optional[int] = None, From 3d96e810b0f2dcb54c0f5c94df2d7779be93c602 Mon Sep 17 00:00:00 2001 From: Caixiaopig Date: Fri, 5 Apr 2024 09:45:57 -0500 Subject: [PATCH 2/2] Updating the default Anthropic Officlal Claude 3 max_tokens to 4096 fix bug --- litellm/llms/anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 080dbd259b7..70cd9c3af8f 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -52,7 +52,7 @@ class AnthropicConfig: def __init__( self, - max_tokens: Optional[int] = None, # You can pass in a value yourself or use the default value 4096 + max_tokens: Optional[int] = 4096, # You can pass in a value yourself or use the default value 4096 stop_sequences: Optional[list] = None, temperature: Optional[int] = None, top_p: Optional[int] = None,