mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-09 03:18:18 +00:00
refactor: remove unused helpers from the shared misc and payload utilities
Eight helpers in the two shared utility modules have no caller: message list accessors, a system message splitter and the list filter only that splitter used, a filename sanitiser, a path extractor, an Ollama modelfile parser and an embedding payload converter that a differently named live version already replaced. The modelfile parser alone is around 85 lines of parsing for a format nothing in the tree reads. These modules ship in the published package, so a Function or Tool that imports one of these names by hand would stop loading. None of them appear in the plugin boilerplate the editor offers, and nothing in the tree uses them. This removes around 185 lines from two modules that most of the backend imports from, so there is less to scan when looking for a helper that already exists.
This commit is contained in:
Classic298
parent
8a42aa53e8
commit
dbef825eb4
2 changed files with 0 additions and 187 deletions
|
|
@ -8,7 +8,6 @@ import threading
|
|||
import time
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Sequence, Union
|
||||
|
||||
import aiohttp
|
||||
|
|
@ -546,13 +545,6 @@ def set_last_user_message_content(content: str, messages: list[dict]) -> list[di
|
|||
return messages
|
||||
|
||||
|
||||
def get_last_assistant_message_item(messages: list[dict]) -> dict | None:
|
||||
for message in reversed(messages):
|
||||
if message['role'] == 'assistant':
|
||||
return message
|
||||
return None
|
||||
|
||||
|
||||
def get_last_assistant_message(messages: list[dict]) -> str | None:
|
||||
for message in reversed(messages):
|
||||
if message['role'] == 'assistant':
|
||||
|
|
@ -567,14 +559,6 @@ def get_system_message(messages: list[dict]) -> dict | None:
|
|||
return None
|
||||
|
||||
|
||||
def remove_system_message(messages: list[dict]) -> list[dict]:
|
||||
return [message for message in messages if message['role'] != 'system']
|
||||
|
||||
|
||||
def pop_system_message(messages: list[dict]) -> tuple[dict | None, list[dict]]:
|
||||
return get_system_message(messages), remove_system_message(messages)
|
||||
|
||||
|
||||
def merge_system_messages(messages: list[dict]) -> list[dict]:
|
||||
"""
|
||||
Merge all system messages into one at position 0.
|
||||
|
|
@ -672,25 +656,6 @@ def prepend_to_first_user_message_content(content: str, messages: list[dict]) ->
|
|||
return messages
|
||||
|
||||
|
||||
def append_or_update_assistant_message(content: str, messages: list[dict]):
|
||||
"""
|
||||
Adds a new assistant message at the end of the messages list
|
||||
or updates the existing assistant message at the end.
|
||||
|
||||
:param msg: The message to be added or appended.
|
||||
:param messages: The list of message dictionaries.
|
||||
:return: The updated list of message dictionaries.
|
||||
"""
|
||||
|
||||
if messages and messages[-1].get('role') == 'assistant':
|
||||
messages[-1]['content'] = f'{messages[-1]["content"]}\n{content}'
|
||||
else:
|
||||
# Insert at the end
|
||||
messages.append({'role': 'assistant', 'content': content})
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def strip_empty_content_blocks(messages: list[dict]) -> list[dict]:
|
||||
"""
|
||||
Remove empty text content blocks from multimodal message content arrays.
|
||||
|
|
@ -820,19 +785,6 @@ def validate_email_format(email: str) -> bool:
|
|||
return bool(re.match(r'[^@]+@[^@]+\.[^@]+', email))
|
||||
|
||||
|
||||
def sanitize_filename(file_name):
|
||||
# Convert to lowercase
|
||||
lower_case_file_name = file_name.lower()
|
||||
|
||||
# Remove special characters using regular expression
|
||||
sanitized_file_name = re.sub(r'[^\w\s]', '', lower_case_file_name)
|
||||
|
||||
# Replace spaces with dashes
|
||||
final_file_name = re.sub(r'\s+', '-', sanitized_file_name)
|
||||
|
||||
return final_file_name
|
||||
|
||||
|
||||
def json_text_variants(value: str) -> list[str]:
|
||||
"""Both spellings ``value`` can take inside a serialized JSON column, unquoted.
|
||||
|
||||
|
|
@ -932,30 +884,6 @@ def sanitize_metadata(metadata: dict) -> dict:
|
|||
return _sanitize(metadata)
|
||||
|
||||
|
||||
def extract_folders_after_data_docs(path):
|
||||
# Convert the path to a Path object if it's not already
|
||||
path = Path(path)
|
||||
|
||||
# Extract parts of the path
|
||||
parts = path.parts
|
||||
|
||||
# Find the index of '/data/docs' in the path
|
||||
try:
|
||||
index_data_docs = parts.index('data') + 1
|
||||
index_docs = parts.index('docs', index_data_docs) + 1
|
||||
except ValueError:
|
||||
return []
|
||||
|
||||
# Exclude the filename and accumulate folder names
|
||||
tags = []
|
||||
|
||||
folders = parts[index_docs:-1]
|
||||
for idx, _ in enumerate(folders):
|
||||
tags.append('/'.join(folders[: idx + 1]))
|
||||
|
||||
return tags
|
||||
|
||||
|
||||
def parse_duration(duration: str) -> timedelta | None:
|
||||
if duration == '-1' or duration == '0':
|
||||
return None
|
||||
|
|
@ -987,92 +915,6 @@ def parse_duration(duration: str) -> timedelta | None:
|
|||
return total_duration
|
||||
|
||||
|
||||
def parse_ollama_modelfile(model_text):
|
||||
parameters_meta = {
|
||||
'mirostat': int,
|
||||
'mirostat_eta': float,
|
||||
'mirostat_tau': float,
|
||||
'num_ctx': int,
|
||||
'repeat_last_n': int,
|
||||
'repeat_penalty': float,
|
||||
'temperature': float,
|
||||
'seed': int,
|
||||
'tfs_z': float,
|
||||
'num_predict': int,
|
||||
'top_k': int,
|
||||
'top_p': float,
|
||||
'num_keep': int,
|
||||
'presence_penalty': float,
|
||||
'frequency_penalty': float,
|
||||
'num_batch': int,
|
||||
'num_gpu': int,
|
||||
'use_mmap': bool,
|
||||
'use_mlock': bool,
|
||||
'num_thread': int,
|
||||
}
|
||||
|
||||
data = {'base_model_id': None, 'params': {}}
|
||||
|
||||
# Parse base model
|
||||
base_model_match = re.search(r'^FROM\s+(\w+)', model_text, re.MULTILINE | re.IGNORECASE)
|
||||
if base_model_match:
|
||||
data['base_model_id'] = base_model_match.group(1)
|
||||
|
||||
# Parse template
|
||||
template_match = re.search(r'TEMPLATE\s+"""(.+?)"""', model_text, re.DOTALL | re.IGNORECASE)
|
||||
if template_match:
|
||||
data['params'] = {'template': template_match.group(1).strip()}
|
||||
|
||||
# Parse stops
|
||||
stops = re.findall(r'PARAMETER stop "(.*?)"', model_text, re.IGNORECASE)
|
||||
if stops:
|
||||
data['params']['stop'] = stops
|
||||
|
||||
# Parse other parameters from the provided list
|
||||
for param, param_type in parameters_meta.items():
|
||||
param_match = re.search(rf'PARAMETER {param} (.+)', model_text, re.IGNORECASE)
|
||||
if param_match:
|
||||
value = param_match.group(1)
|
||||
|
||||
try:
|
||||
if param_type is int:
|
||||
value = int(value)
|
||||
elif param_type is float:
|
||||
value = float(value)
|
||||
elif param_type is bool:
|
||||
value = value.lower() == 'true'
|
||||
except Exception as e:
|
||||
log.exception(f'Failed to parse parameter {param}: {e}')
|
||||
continue
|
||||
|
||||
data['params'][param] = value
|
||||
|
||||
# Parse adapter
|
||||
adapter_match = re.search(r'ADAPTER (.+)', model_text, re.IGNORECASE)
|
||||
if adapter_match:
|
||||
data['params']['adapter'] = adapter_match.group(1)
|
||||
|
||||
# Parse system description
|
||||
system_desc_match = re.search(r'SYSTEM\s+"""(.+?)"""', model_text, re.DOTALL | re.IGNORECASE)
|
||||
system_desc_match_single = re.search(r'SYSTEM\s+([^\n]+)', model_text, re.IGNORECASE)
|
||||
|
||||
if system_desc_match:
|
||||
data['params']['system'] = system_desc_match.group(1).strip()
|
||||
elif system_desc_match_single:
|
||||
data['params']['system'] = system_desc_match_single.group(1).strip()
|
||||
|
||||
# Parse messages
|
||||
messages = []
|
||||
message_matches = re.findall(r'MESSAGE (\w+) (.+)', model_text, re.IGNORECASE)
|
||||
for role, content in message_matches:
|
||||
messages.append({'role': role, 'content': content})
|
||||
|
||||
if messages:
|
||||
data['params']['messages'] = messages
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def convert_logit_bias_input_to_json(logit_bias_input) -> str | None:
|
||||
if not logit_bias_input:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -445,35 +445,6 @@ def convert_payload_openai_to_ollama(openai_payload: dict) -> dict:
|
|||
return ollama_payload
|
||||
|
||||
|
||||
def convert_embedding_payload_openai_to_ollama(openai_payload: dict) -> dict:
|
||||
"""
|
||||
Convert an embeddings request payload from OpenAI format to Ollama format.
|
||||
|
||||
Args:
|
||||
openai_payload (dict): The original payload designed for OpenAI API usage.
|
||||
|
||||
Returns:
|
||||
dict: A payload compatible with the Ollama API embeddings endpoint.
|
||||
"""
|
||||
ollama_payload = {'model': openai_payload.get('model')}
|
||||
input_value = openai_payload.get('input')
|
||||
|
||||
# Ollama expects 'input' as a list, and 'prompt' as a single string.
|
||||
if isinstance(input_value, list):
|
||||
ollama_payload['input'] = input_value
|
||||
ollama_payload['prompt'] = '\n'.join(str(x) for x in input_value)
|
||||
else:
|
||||
ollama_payload['input'] = [input_value]
|
||||
ollama_payload['prompt'] = str(input_value)
|
||||
|
||||
# Optionally forward other fields if present
|
||||
for optional_key in ('options', 'truncate', 'keep_alive'):
|
||||
if optional_key in openai_payload:
|
||||
ollama_payload[optional_key] = openai_payload[optional_key]
|
||||
|
||||
return ollama_payload
|
||||
|
||||
|
||||
def convert_embed_payload_openai_to_ollama(openai_payload: dict) -> dict:
|
||||
"""
|
||||
Convert an embeddings request payload from OpenAI format to Ollama's
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue