scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-09-06 08:16:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix bugs for large batch size of embeddings

Browse source
This commit is contained in:
鸣山 2025-06-16 15:16:44 +08:00
parent c339ed108d
commit b1cf4b7dd3
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
experiencemaker/model/base_embedding_model.py
View file

@ -76,7 +76,9 @@ class BaseEmbeddingModel(BaseModel, ABC):
return nodes
elif isinstance(nodes, list):
embeddings = self.get_embeddings(input_text=[node.content for node in nodes])
max_batch_size = 10 # text-embedding-v4 batch size should not be larger than 10
embeddings = [emb for i in range(0, len(nodes), max_batch_size) for emb in
self.get_embeddings(input_text=[node.content for node in nodes[i:i + max_batch_size]])]
if len(embeddings) != len(nodes):
logger.warning(f"embeddings.size={len(embeddings)} <> nodes.size={len(nodes)}")
else: