mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-06 02:48:04 +00:00
feat: integrate grounding model for computer use desktop with bbox coordinate scaling
- Add grounding model config (model selection, coordinate width/height) to admin Interface settings for calibrating VL model bbox output - Implement _execute_grounding() in middleware: fetches screenshot from open-terminal, calls grounding VL model, parses bounding boxes, and scales coordinates from model space to actual screen resolution - Inject desktop_locate tool for harness model, hide non-essential desktop tools to steer model toward locate-then-act workflow - Display desktop tool images in chat UI only (not sent to harness model context), with lazy loading and base64 stripping for performance - Add terminal_id to chat payload for routing to correct terminal server
This commit is contained in:
volker
parent
683f126dab
commit
d16bd616d6
8 changed files with 418 additions and 12 deletions
|
|
@ -1701,6 +1701,24 @@ TASK_MODEL_EXTERNAL = PersistentConfig(
|
|||
os.environ.get('TASK_MODEL_EXTERNAL', ''),
|
||||
)
|
||||
|
||||
GROUNDING_MODEL = PersistentConfig(
|
||||
'GROUNDING_MODEL',
|
||||
'task.model.grounding',
|
||||
os.environ.get('GROUNDING_MODEL', ''),
|
||||
)
|
||||
|
||||
GROUNDING_MODEL_WIDTH = PersistentConfig(
|
||||
'GROUNDING_MODEL_WIDTH',
|
||||
'task.model.grounding_width',
|
||||
int(os.environ.get('GROUNDING_MODEL_WIDTH', '1000')),
|
||||
)
|
||||
|
||||
GROUNDING_MODEL_HEIGHT = PersistentConfig(
|
||||
'GROUNDING_MODEL_HEIGHT',
|
||||
'task.model.grounding_height',
|
||||
int(os.environ.get('GROUNDING_MODEL_HEIGHT', '1000')),
|
||||
)
|
||||
|
||||
TITLE_GENERATION_PROMPT_TEMPLATE = PersistentConfig(
|
||||
'TITLE_GENERATION_PROMPT_TEMPLATE',
|
||||
'task.title.prompt_template',
|
||||
|
|
|
|||
|
|
@ -447,6 +447,9 @@ from open_webui.config import (
|
|||
# Tasks
|
||||
TASK_MODEL,
|
||||
TASK_MODEL_EXTERNAL,
|
||||
GROUNDING_MODEL,
|
||||
GROUNDING_MODEL_WIDTH,
|
||||
GROUNDING_MODEL_HEIGHT,
|
||||
ENABLE_TAGS_GENERATION,
|
||||
ENABLE_TITLE_GENERATION,
|
||||
ENABLE_FOLLOW_UP_GENERATION,
|
||||
|
|
@ -1292,6 +1295,9 @@ app.state.speech_speaker_embeddings_dataset = None
|
|||
|
||||
app.state.config.TASK_MODEL = TASK_MODEL
|
||||
app.state.config.TASK_MODEL_EXTERNAL = TASK_MODEL_EXTERNAL
|
||||
app.state.config.GROUNDING_MODEL = GROUNDING_MODEL
|
||||
app.state.config.GROUNDING_MODEL_WIDTH = GROUNDING_MODEL_WIDTH
|
||||
app.state.config.GROUNDING_MODEL_HEIGHT = GROUNDING_MODEL_HEIGHT
|
||||
|
||||
|
||||
app.state.config.ENABLE_SEARCH_QUERY_GENERATION = ENABLE_SEARCH_QUERY_GENERATION
|
||||
|
|
|
|||
|
|
@ -66,6 +66,9 @@ async def get_task_config(request: Request, user=Depends(get_verified_user)):
|
|||
return {
|
||||
'TASK_MODEL': request.app.state.config.TASK_MODEL,
|
||||
'TASK_MODEL_EXTERNAL': request.app.state.config.TASK_MODEL_EXTERNAL,
|
||||
'GROUNDING_MODEL': request.app.state.config.GROUNDING_MODEL,
|
||||
'GROUNDING_MODEL_WIDTH': request.app.state.config.GROUNDING_MODEL_WIDTH,
|
||||
'GROUNDING_MODEL_HEIGHT': request.app.state.config.GROUNDING_MODEL_HEIGHT,
|
||||
'TITLE_GENERATION_PROMPT_TEMPLATE': request.app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE,
|
||||
'IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE': request.app.state.config.IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE,
|
||||
'ENABLE_AUTOCOMPLETE_GENERATION': request.app.state.config.ENABLE_AUTOCOMPLETE_GENERATION,
|
||||
|
|
@ -86,6 +89,9 @@ async def get_task_config(request: Request, user=Depends(get_verified_user)):
|
|||
class TaskConfigForm(BaseModel):
|
||||
TASK_MODEL: Optional[str]
|
||||
TASK_MODEL_EXTERNAL: Optional[str]
|
||||
GROUNDING_MODEL: Optional[str]
|
||||
GROUNDING_MODEL_WIDTH: int = 1000
|
||||
GROUNDING_MODEL_HEIGHT: int = 1000
|
||||
ENABLE_TITLE_GENERATION: bool
|
||||
TITLE_GENERATION_PROMPT_TEMPLATE: str
|
||||
IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE: str
|
||||
|
|
@ -106,6 +112,9 @@ class TaskConfigForm(BaseModel):
|
|||
async def update_task_config(request: Request, form_data: TaskConfigForm, user=Depends(get_admin_user)):
|
||||
request.app.state.config.TASK_MODEL = form_data.TASK_MODEL
|
||||
request.app.state.config.TASK_MODEL_EXTERNAL = form_data.TASK_MODEL_EXTERNAL
|
||||
request.app.state.config.GROUNDING_MODEL = form_data.GROUNDING_MODEL
|
||||
request.app.state.config.GROUNDING_MODEL_WIDTH = form_data.GROUNDING_MODEL_WIDTH
|
||||
request.app.state.config.GROUNDING_MODEL_HEIGHT = form_data.GROUNDING_MODEL_HEIGHT
|
||||
request.app.state.config.ENABLE_TITLE_GENERATION = form_data.ENABLE_TITLE_GENERATION
|
||||
request.app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE = form_data.TITLE_GENERATION_PROMPT_TEMPLATE
|
||||
|
||||
|
|
@ -132,6 +141,9 @@ async def update_task_config(request: Request, form_data: TaskConfigForm, user=D
|
|||
return {
|
||||
'TASK_MODEL': request.app.state.config.TASK_MODEL,
|
||||
'TASK_MODEL_EXTERNAL': request.app.state.config.TASK_MODEL_EXTERNAL,
|
||||
'GROUNDING_MODEL': request.app.state.config.GROUNDING_MODEL,
|
||||
'GROUNDING_MODEL_WIDTH': request.app.state.config.GROUNDING_MODEL_WIDTH,
|
||||
'GROUNDING_MODEL_HEIGHT': request.app.state.config.GROUNDING_MODEL_HEIGHT,
|
||||
'ENABLE_TITLE_GENERATION': request.app.state.config.ENABLE_TITLE_GENERATION,
|
||||
'TITLE_GENERATION_PROMPT_TEMPLATE': request.app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE,
|
||||
'IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE': request.app.state.config.IMAGE_PROMPT_GENERATION_PROMPT_TEMPLATE,
|
||||
|
|
|
|||
|
|
@ -936,6 +936,156 @@ def apply_source_context_to_messages(
|
|||
)
|
||||
|
||||
|
||||
async def _execute_grounding(request, metadata, params):
|
||||
import httpx
|
||||
|
||||
grounding_model = metadata.get('grounding_model', '')
|
||||
terminal_id = metadata.get('grounding_terminal_id', '')
|
||||
description = params.get('description', '')
|
||||
|
||||
if not grounding_model:
|
||||
return {'error': 'No grounding model configured'}
|
||||
|
||||
connections = request.app.state.config.TERMINAL_SERVER_CONNECTIONS or []
|
||||
connection = next((c for c in connections if c.get('id') == terminal_id), None)
|
||||
if not connection:
|
||||
return {'error': f'Terminal server {terminal_id} not found'}
|
||||
|
||||
base_url = connection.get('url', '').rstrip('/')
|
||||
api_key = connection.get('key', '')
|
||||
headers = {'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json'}
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
resp = await client.post(f'{base_url}/desktop/screenshot', json={}, headers=headers)
|
||||
resp.raise_for_status()
|
||||
screenshot_data = resp.json()
|
||||
except Exception as e:
|
||||
return {'error': f'Failed to get screenshot: {e}'}
|
||||
|
||||
image_url = screenshot_data.get('image', '')
|
||||
img_width = screenshot_data.get('width', 0)
|
||||
img_height = screenshot_data.get('height', 0)
|
||||
|
||||
if not image_url:
|
||||
return {'error': 'No screenshot image available'}
|
||||
|
||||
models = request.app.state.MODELS
|
||||
model_info = models.get(grounding_model)
|
||||
if not model_info:
|
||||
return {'error': f'Grounding model {grounding_model} not available'}
|
||||
|
||||
url_idx = model_info.get('urlIdx', 0)
|
||||
api_base_urls = request.app.state.config.OPENAI_API_BASE_URLS
|
||||
api_keys = request.app.state.config.OPENAI_API_KEYS
|
||||
api_base = api_base_urls[url_idx] if url_idx < len(api_base_urls) else api_base_urls[0]
|
||||
api_key_val = api_keys[url_idx] if url_idx < len(api_keys) else api_keys[0]
|
||||
|
||||
system_prompt = (
|
||||
'You are a UI element detection model. Analyze the screenshot and identify ALL visible UI elements. '
|
||||
'For each element, provide a short description and its bounding box as [x1, y1, x2, y2] in pixel coordinates. '
|
||||
'Reply with ONLY a JSON array: [{"description": "...", "bounding_box": [x1, y1, x2, y2]}, ...]. '
|
||||
'Include buttons, links, input fields, text areas, menus, icons, images, and any other interactive or visible elements. '
|
||||
'Be thorough — list every distinct UI element you can see.'
|
||||
)
|
||||
|
||||
user_text = (
|
||||
f'List all UI elements in this screenshot with their bounding boxes.'
|
||||
if not description
|
||||
else f'List all UI elements in this screenshot with their bounding boxes. The user is looking for: {description}'
|
||||
)
|
||||
|
||||
image_content = {'type': 'image_url', 'image_url': {'url': image_url}}
|
||||
if img_width and img_height:
|
||||
image_content['resized_width'] = img_width
|
||||
image_content['resized_height'] = img_height
|
||||
|
||||
messages = [
|
||||
{'role': 'system', 'content': system_prompt},
|
||||
{
|
||||
'role': 'user',
|
||||
'content': [
|
||||
image_content,
|
||||
{'type': 'text', 'text': user_text},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
resp = await client.post(
|
||||
f'{api_base}/chat/completions',
|
||||
headers={'Authorization': f'Bearer {api_key_val}', 'Content-Type': 'application/json'},
|
||||
json={'model': grounding_model, 'messages': messages, 'max_tokens': 4096, 'temperature': 0.1},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
return {'error': f'Grounding model call failed: {e}'}
|
||||
|
||||
content = ''
|
||||
choices = data.get('choices', [])
|
||||
if choices:
|
||||
content = choices[0].get('message', {}).get('content', '')
|
||||
|
||||
try:
|
||||
start = content.index('[')
|
||||
end = content.rindex(']') + 1
|
||||
elements = json.loads(content[start:end])
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
try:
|
||||
start = content.index('{')
|
||||
end = content.rindex('}') + 1
|
||||
elements = [json.loads(content[start:end])]
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
return {'raw_response': content, 'error': 'Could not parse grounding response'}
|
||||
|
||||
gm_width = request.app.state.config.GROUNDING_MODEL_WIDTH or 1000
|
||||
gm_height = request.app.state.config.GROUNDING_MODEL_HEIGHT or 1000
|
||||
scale_x = img_width / gm_width if gm_width and img_width else 1.0
|
||||
scale_y = img_height / gm_height if gm_height and img_height else 1.0
|
||||
|
||||
for elem in elements:
|
||||
bbox = elem.get('bounding_box', [])
|
||||
if len(bbox) == 4:
|
||||
elem['bounding_box'] = [
|
||||
int(bbox[0] * scale_x),
|
||||
int(bbox[1] * scale_y),
|
||||
int(bbox[2] * scale_x),
|
||||
int(bbox[3] * scale_y),
|
||||
]
|
||||
|
||||
if description and elements:
|
||||
desc_lower = description.lower()
|
||||
for elem in elements:
|
||||
elem_desc = elem.get('description', '').lower()
|
||||
if desc_lower in elem_desc or any(w in elem_desc for w in desc_lower.split()):
|
||||
bbox = elem.get('bounding_box', [])
|
||||
if len(bbox) == 4:
|
||||
x = (bbox[0] + bbox[2]) // 2
|
||||
y = (bbox[1] + bbox[3]) // 2
|
||||
elem['best_match'] = True
|
||||
return {
|
||||
'elements': elements,
|
||||
'match': {'description': elem.get('description', ''), 'bounding_box': bbox, 'x': x, 'y': y},
|
||||
'description': description,
|
||||
}
|
||||
|
||||
if description and elements:
|
||||
bbox = elements[0].get('bounding_box', [])
|
||||
if len(bbox) == 4:
|
||||
x = (bbox[0] + bbox[2]) // 2
|
||||
y = (bbox[1] + bbox[3]) // 2
|
||||
elements[0]['best_match'] = True
|
||||
return {
|
||||
'elements': elements,
|
||||
'match': {'description': elements[0].get('description', ''), 'bounding_box': bbox, 'x': x, 'y': y},
|
||||
'description': description,
|
||||
}
|
||||
|
||||
return {'elements': elements, 'description': description}
|
||||
|
||||
|
||||
def process_tool_result(
|
||||
request,
|
||||
tool_function_name,
|
||||
|
|
@ -1108,7 +1258,26 @@ def process_tool_result(
|
|||
if isinstance(tool_result, list):
|
||||
tool_result = {'results': tool_result}
|
||||
|
||||
if isinstance(tool_result, dict) or isinstance(tool_result, list):
|
||||
if isinstance(tool_result, dict):
|
||||
images = tool_result.pop('images', None)
|
||||
if images and isinstance(images, list):
|
||||
for img_item in images:
|
||||
if isinstance(img_item, dict):
|
||||
url = img_item.get('image', '')
|
||||
label = img_item.get('label', '')
|
||||
else:
|
||||
url = str(img_item)
|
||||
label = ''
|
||||
if url and url.startswith('data:image/'):
|
||||
file_entry = {'type': 'image', 'url': url}
|
||||
if label:
|
||||
file_entry['label'] = label
|
||||
tool_result_files.append(file_entry)
|
||||
image_url = tool_result.pop('image', None)
|
||||
if image_url and isinstance(image_url, str) and image_url.startswith('data:image/'):
|
||||
tool_result_files.append({'type': 'image', 'url': image_url})
|
||||
tool_result = json.dumps(tool_result, indent=2, ensure_ascii=False)
|
||||
elif isinstance(tool_result, list):
|
||||
tool_result = json.dumps(tool_result, indent=2, ensure_ascii=False)
|
||||
|
||||
# Safety: ensure tool_result is always a string (or None) to prevent
|
||||
|
|
@ -2381,6 +2550,8 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||
terminal_id = form_data.pop('terminal_id', None)
|
||||
files = form_data.pop('files', None)
|
||||
|
||||
log.info(f'[DEBUG] terminal_id={terminal_id!r}, tool_ids={tool_ids!r}')
|
||||
|
||||
# Caller-provided OpenAI-style tools take precedence over server-side
|
||||
# tool resolution (tool_ids, MCP servers, builtin tools).
|
||||
payload_tools = form_data.get('tools', None)
|
||||
|
|
@ -2620,6 +2791,9 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||
system_prompt = None
|
||||
if terminal_tools:
|
||||
tools_dict = {**tools_dict, **terminal_tools}
|
||||
log.info(f'[DEBUG] terminal_tools loaded: {list(terminal_tools.keys())}')
|
||||
else:
|
||||
log.info(f'[DEBUG] terminal_tools EMPTY for terminal_id={terminal_id}')
|
||||
if system_prompt:
|
||||
form_data['messages'] = add_or_update_system_message(
|
||||
system_prompt,
|
||||
|
|
@ -2674,15 +2848,51 @@ async def process_chat_payload(request, form_data, user, metadata, model):
|
|||
if name not in tools_dict:
|
||||
tools_dict[name] = tool_dict
|
||||
|
||||
grounding_model = getattr(request.app.state.config, 'GROUNDING_MODEL', '')
|
||||
if grounding_model and any(
|
||||
k.startswith('desktop_') for k in tools_dict
|
||||
):
|
||||
tools_dict['desktop_locate'] = {
|
||||
'tool_id': 'builtin:grounding',
|
||||
'type': 'builtin',
|
||||
'spec': {
|
||||
'name': 'desktop_locate',
|
||||
'description': (
|
||||
'The ONLY way to see the desktop screen. '
|
||||
'REQUIRED before AND after every desktop_click, desktop_type, desktop_key, desktop_drag, desktop_mouse_move, or desktop_scroll. '
|
||||
'Takes a fresh screenshot, uses a vision model to detect all visible UI elements, '
|
||||
'and returns a list of elements with bounding boxes [x1, y1, x2, y2] in pixel coordinates. '
|
||||
'When a description is provided, the best matching element is highlighted with its center coordinates. '
|
||||
'Workflow: desktop_locate(description="...") → get x,y → desktop_click(x,y) → desktop_locate(description="...") to verify.'
|
||||
),
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'description': {
|
||||
'type': 'string',
|
||||
'description': 'What to find, e.g. "Submit button", "search input field", "Chrome address bar". '
|
||||
'All visible elements are returned; the best match for this description is highlighted.',
|
||||
},
|
||||
},
|
||||
'required': ['description'],
|
||||
},
|
||||
},
|
||||
}
|
||||
for _hidden in ('desktop_screenshot', 'desktop_status', 'desktop_start', 'desktop_stop', 'desktop_mouse_location'):
|
||||
tools_dict.pop(_hidden, None)
|
||||
metadata['grounding_model'] = grounding_model
|
||||
if terminal_id:
|
||||
metadata['grounding_terminal_id'] = terminal_id
|
||||
|
||||
if tools_dict:
|
||||
log.info(f'[DEBUG] tools_dict keys: {list(tools_dict.keys())}')
|
||||
log.info(f'[DEBUG] function_calling={metadata.get("params", {}).get("function_calling")!r}')
|
||||
if metadata.get('params', {}).get('function_calling') == 'native':
|
||||
# If the function calling is native, then call the tools function calling handler
|
||||
metadata['tools'] = tools_dict
|
||||
form_data['tools'] = [
|
||||
{'type': 'function', 'function': tool.get('spec', {})} for tool in tools_dict.values()
|
||||
]
|
||||
else:
|
||||
# If the function calling is not native, then call the tools function calling handler
|
||||
try:
|
||||
form_data, flags = await chat_completion_tools_handler(
|
||||
request, form_data, extra_params, user, models, tools_dict
|
||||
|
|
@ -4157,7 +4367,14 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
k: v for k, v in tool_function_params.items() if k in allowed_params
|
||||
}
|
||||
|
||||
if direct_tool:
|
||||
if tool_function_name == 'desktop_locate' and tool.get('tool_id') == 'builtin:grounding':
|
||||
tool_result = await _execute_grounding(
|
||||
request,
|
||||
metadata,
|
||||
tool_function_params,
|
||||
)
|
||||
|
||||
elif direct_tool:
|
||||
tool_result = await event_caller(
|
||||
{
|
||||
'type': 'execute:tool',
|
||||
|
|
@ -4249,15 +4466,19 @@ async def streaming_chat_response_handler(response, ctx):
|
|||
for result in results:
|
||||
output_parts = [{'type': 'input_text', 'text': result.get('content', '')}]
|
||||
|
||||
# Separate image data URIs (for LLM via input_image) from
|
||||
# other files (for frontend display via files attribute).
|
||||
# Separate image data URIs from other files.
|
||||
# Desktop tool images go to display_files only (UI) —
|
||||
# the grounding model fetches its own screenshots.
|
||||
# Non-desktop images are sent to the harness model via
|
||||
# input_image AND displayed in the UI.
|
||||
display_files = []
|
||||
is_desktop_tool = tool_function_name.startswith('desktop_')
|
||||
for file_item in result.get('files', []):
|
||||
if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'):
|
||||
# LLM-only: add as input_image part (invisible to serialize_output)
|
||||
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
|
||||
if not is_desktop_tool:
|
||||
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
|
||||
display_files.append(file_item)
|
||||
else:
|
||||
# Frontend display (MCP images, audio, etc.)
|
||||
display_files.append(file_item)
|
||||
|
||||
output.append(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@
|
|||
let taskConfig = {
|
||||
TASK_MODEL: '',
|
||||
TASK_MODEL_EXTERNAL: '',
|
||||
GROUNDING_MODEL: '',
|
||||
GROUNDING_MODEL_WIDTH: 1000,
|
||||
GROUNDING_MODEL_HEIGHT: 1000,
|
||||
ENABLE_TITLE_GENERATION: true,
|
||||
TITLE_GENERATION_PROMPT_TEMPLATE: '',
|
||||
ENABLE_FOLLOW_UP_GENERATION: true,
|
||||
|
|
@ -208,6 +211,81 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class=" mb-2 font-medium flex items-center">
|
||||
<div class=" text-xs mr-1">{$i18n.t('Grounding Model')}</div>
|
||||
<Tooltip
|
||||
content={$i18n.t(
|
||||
'A vision model used to locate UI elements on the desktop. Called by the harness model to find precise click targets.'
|
||||
)}
|
||||
>
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke-width="1.5"
|
||||
stroke="currentColor"
|
||||
class="size-3.5"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="m11.25 11.25.041-.02a.75.75 0 0 1 1.063.852l-.708 2.836a.75.75 0 0 0 1.063.853l.041-.021M21 12a9 9 0 1 1-18 0 9 9 0 0 1 18 0Zm-9-3.75h.008v.008H12V8.25Z"
|
||||
/>
|
||||
</svg>
|
||||
</Tooltip>
|
||||
</div>
|
||||
|
||||
<div class=" mb-2.5 flex w-full gap-2">
|
||||
<div class="flex-1">
|
||||
<select
|
||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||
bind:value={taskConfig.GROUNDING_MODEL}
|
||||
placeholder={$i18n.t('Select a model')}
|
||||
on:change={() => {
|
||||
if (taskConfig.GROUNDING_MODEL) {
|
||||
const model = models.find((m) => m.id === taskConfig.GROUNDING_MODEL);
|
||||
if (model) {
|
||||
taskConfig.GROUNDING_MODEL = model.id;
|
||||
} else {
|
||||
taskConfig.GROUNDING_MODEL = '';
|
||||
}
|
||||
}
|
||||
}}
|
||||
>
|
||||
<option value="" selected>{$i18n.t('Disabled')}</option>
|
||||
{#each models as model}
|
||||
<option value={model.id} class="bg-gray-100 dark:bg-gray-700">
|
||||
{model.name}
|
||||
{model?.connection_type === 'local' ? `(${$i18n.t('Local')})` : ''}
|
||||
</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{#if taskConfig.GROUNDING_MODEL}
|
||||
<div class="mb-2.5 flex w-full gap-2">
|
||||
<div class="flex-1">
|
||||
<div class=" mb-1 text-xs font-medium">{$i18n.t('Grounding Coordinate Width')}</div>
|
||||
<input
|
||||
type="number"
|
||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||
bind:value={taskConfig.GROUNDING_MODEL_WIDTH}
|
||||
min="1"
|
||||
/>
|
||||
</div>
|
||||
<div class="flex-1">
|
||||
<div class=" mb-1 text-xs font-medium">{$i18n.t('Grounding Coordinate Height')}</div>
|
||||
<input
|
||||
type="number"
|
||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||
bind:value={taskConfig.GROUNDING_MODEL_HEIGHT}
|
||||
min="1"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="mb-2.5 flex w-full items-center justify-between">
|
||||
<div class=" self-center text-xs font-medium">
|
||||
{$i18n.t('Title Generation')}
|
||||
|
|
|
|||
|
|
@ -415,6 +415,36 @@
|
|||
}
|
||||
};
|
||||
|
||||
const MAX_TOOL_IMAGES = 3;
|
||||
const STRIP_THRESHOLD_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
const stripToolImageBase64 = (content, keepCount = MAX_TOOL_IMAGES) => {
|
||||
const matches = [];
|
||||
let searchPos = 0;
|
||||
while (true) {
|
||||
const pos = content.indexOf('files="', searchPos);
|
||||
if (pos === -1) break;
|
||||
const valueStart = pos + 7;
|
||||
const valueEnd = content.indexOf('"', valueStart);
|
||||
if (valueEnd === -1) break;
|
||||
const value = content.substring(valueStart, valueEnd);
|
||||
if (value.includes('data:image/')) {
|
||||
matches.push({ valueStart, valueEnd });
|
||||
}
|
||||
searchPos = valueEnd + 1;
|
||||
}
|
||||
|
||||
if (matches.length <= keepCount) return content;
|
||||
|
||||
const cutoff = matches.length - keepCount;
|
||||
let result = content;
|
||||
for (let i = cutoff - 1; i >= 0; i--) {
|
||||
const m = matches[i];
|
||||
result = result.substring(0, m.valueStart) + result.substring(m.valueEnd);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const chatEventHandler = async (event, cb) => {
|
||||
console.log(event);
|
||||
|
||||
|
|
@ -447,6 +477,9 @@
|
|||
}
|
||||
} else if (type === 'chat:message:delta' || type === 'message') {
|
||||
message.content += data.content;
|
||||
if (message.content.length > STRIP_THRESHOLD_BYTES) {
|
||||
message.content = stripToolImageBase64(message.content);
|
||||
}
|
||||
} else if (type === 'chat:message' || type === 'replace') {
|
||||
message.content = data.content;
|
||||
} else if (type === 'chat:message:files' || type === 'files') {
|
||||
|
|
@ -1282,7 +1315,7 @@
|
|||
if (history.currentId) {
|
||||
for (const message of Object.values(history.messages)) {
|
||||
if (message && message.role === 'assistant' && message.done !== false) {
|
||||
message.done = true;
|
||||
message.done = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1621,6 +1654,10 @@
|
|||
} else {
|
||||
message.content += value;
|
||||
|
||||
if (message.content.length > STRIP_THRESHOLD_BYTES) {
|
||||
message.content = stripToolImageBase64(message.content);
|
||||
}
|
||||
|
||||
if (navigator.vibrate && ($settings?.hapticFeedback ?? false)) {
|
||||
navigator.vibrate(5);
|
||||
}
|
||||
|
|
@ -1656,6 +1693,9 @@
|
|||
if (content) {
|
||||
// REALTIME_CHAT_SAVE is disabled
|
||||
message.content = content;
|
||||
if (message.content.length > STRIP_THRESHOLD_BYTES) {
|
||||
message.content = stripToolImageBase64(message.content);
|
||||
}
|
||||
|
||||
if (navigator.vibrate && ($settings?.hapticFeedback ?? false)) {
|
||||
navigator.vibrate(5);
|
||||
|
|
@ -1700,6 +1740,7 @@
|
|||
|
||||
if (done) {
|
||||
message.done = true;
|
||||
message.content = stripToolImageBase64(message.content);
|
||||
|
||||
if ($settings.responseAutoCopy) {
|
||||
copyToClipboard(message.content);
|
||||
|
|
|
|||
|
|
@ -59,11 +59,41 @@
|
|||
]
|
||||
});
|
||||
|
||||
const MAX_TOOL_IMAGES = 5;
|
||||
|
||||
const stripOldToolImages = (str) => {
|
||||
const matches = [];
|
||||
let searchPos = 0;
|
||||
while (true) {
|
||||
const pos = str.indexOf('files="', searchPos);
|
||||
if (pos === -1) break;
|
||||
const valueStart = pos + 7;
|
||||
const valueEnd = str.indexOf('"', valueStart);
|
||||
if (valueEnd === -1) break;
|
||||
const value = str.substring(valueStart, valueEnd);
|
||||
if (value.includes('data:image/')) {
|
||||
matches.push({ valueStart, valueEnd });
|
||||
}
|
||||
searchPos = valueEnd + 1;
|
||||
}
|
||||
|
||||
if (matches.length <= MAX_TOOL_IMAGES) return str;
|
||||
|
||||
const cutoff = matches.length - MAX_TOOL_IMAGES;
|
||||
let result = str;
|
||||
for (let i = cutoff - 1; i >= 0; i--) {
|
||||
const m = matches[i];
|
||||
result = result.substring(0, m.valueStart) + result.substring(m.valueEnd);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const parseTokens = () => {
|
||||
if (content === lastContent) return;
|
||||
lastContent = content;
|
||||
|
||||
const processed = replaceTokens(processResponseContent(content), model?.name, $user?.name);
|
||||
let processed = replaceTokens(processResponseContent(content), model?.name, $user?.name);
|
||||
processed = stripOldToolImages(processed);
|
||||
if (processed === lastParsedContent) return;
|
||||
lastParsedContent = processed;
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
aria-label={$i18n.t('Show image preview')}
|
||||
type="button"
|
||||
>
|
||||
<img src={_src} {alt} class={imageClassName} draggable="false" data-cy="image" />
|
||||
<img src={_src} {alt} class={imageClassName} draggable="false" loading="lazy" decoding="async" data-cy="image" />
|
||||
</button>
|
||||
|
||||
{#if dismissible}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue