From b5e0ec2d8dcb58be5c6c48cd68f4b28ae3157534 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:05:08 +0800 Subject: [PATCH] Modify budget calculation for text limit safety margin Adjust budget calculation to use 92% margin for token estimation. --- reme/components/embedding_store/base_embedding_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reme/components/embedding_store/base_embedding_store.py b/reme/components/embedding_store/base_embedding_store.py index 9075b1de..cf557304 100644 --- a/reme/components/embedding_store/base_embedding_store.py +++ b/reme/components/embedding_store/base_embedding_store.py @@ -46,7 +46,7 @@ class BaseEmbeddingStore(BaseComponent): return text[:limit] # Reserve a 5% safety margin for token estimation. - budget = limit * 2 * 95 // 100 + budget = limit * 2 * 92 // 100 used = 0 for index, char in enumerate(text): used += 3 if unicodedata.east_asian_width(char) in {"W", "F"} else 2