scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-28 05:27:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refac

Browse source
This commit is contained in:
Timothy Jaeryang Baek 2026-07-27 03:32:21 -04:00
parent 3ab2026262
commit db2d24896b
8 changed files with 103 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
backend/open_webui/config.py
View file

@ -853,6 +853,17 @@ ONEDRIVE_SHAREPOINT_TENANT_ID = os.getenv('ONEDRIVE_SHAREPOINT_TENANT_ID', '')
# RAG Content Extraction
CONTENT_EXTRACTION_ENGINE = os.getenv('CONTENT_EXTRACTION_ENGINE', '').lower()
content_extraction_supported_media_mime_types = os.getenv('CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES')
CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES = (
[
mime_type.strip()
for mime_type in content_extraction_supported_media_mime_types.split(',')
if mime_type.strip()
]
if content_extraction_supported_media_mime_types is not None
else None
)
DATALAB_MARKER_API_KEY = os.getenv('DATALAB_MARKER_API_KEY', '')
DATALAB_MARKER_API_BASE_URL = os.getenv('DATALAB_MARKER_API_BASE_URL', '')
@ -1999,6 +2010,8 @@ FOLDER_MAX_FILE_COUNT = os.getenv('FOLDER_MAX_FILE_COUNT', '')
ENABLE_CHANNELS = os.getenv('ENABLE_CHANNELS', 'False').lower() == 'true'
CHANNEL_MODEL_RESPONSE_MODE = os.getenv('CHANNEL_MODEL_RESPONSE_MODE', 'thread')
ENABLE_CALENDAR = os.getenv('ENABLE_CALENDAR', 'True').lower() == 'true'
ENABLE_AUTOMATIONS = os.getenv('ENABLE_AUTOMATIONS', 'True').lower() == 'true'
@ -2817,6 +2830,7 @@ DEFAULT_CONFIG = {
'onedrive.sharepoint_url': ONEDRIVE_SHAREPOINT_URL,
'onedrive.sharepoint_tenant_id': ONEDRIVE_SHAREPOINT_TENANT_ID,
'rag.content_extraction_engine': CONTENT_EXTRACTION_ENGINE,
'rag.content_extraction.supported_media_mime_types': CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES,
'rag.datalab_marker_api_key': DATALAB_MARKER_API_KEY,
'rag.datalab_marker_api_base_url': DATALAB_MARKER_API_BASE_URL,
'rag.datalab_marker_additional_config': DATALAB_MARKER_ADDITIONAL_CONFIG,
@ -3049,6 +3063,7 @@ DEFAULT_CONFIG = {
'folders.enable': ENABLE_FOLDERS,
'folders.max_file_count': FOLDER_MAX_FILE_COUNT,
'channels.enable': ENABLE_CHANNELS,
'channels.model_response_mode': CHANNEL_MODEL_RESPONSE_MODE,
'calendar.enable': ENABLE_CALENDAR,
'automations.enable': ENABLE_AUTOMATIONS,
'subagents.enable': ENABLE_SUBAGENTS,

5
backend/open_webui/routers/auths.py
View file

@ -117,6 +117,7 @@ ADMIN_CONFIG_KEYS = {
'AUTOMATION_MIN_INTERVAL': 'automations.min_interval',
'ENABLE_AUTOMATIONS': 'automations.enable',
'ENABLE_CHANNELS': 'channels.enable',
'CHANNEL_MODEL_RESPONSE_MODE': 'channels.model_response_mode',
'ENABLE_CALENDAR': 'calendar.enable',
'ENABLE_MEMORIES': 'memories.enable',
'ENABLE_MEMORY_SYSTEM_CONTEXT': 'memories.system_context.enable',
@ -1197,6 +1198,7 @@ class AdminConfig(BaseModel):
AUTOMATION_MIN_INTERVAL: int | str | None = None
ENABLE_AUTOMATIONS: bool
ENABLE_CHANNELS: bool
CHANNEL_MODEL_RESPONSE_MODE: str = 'thread'
ENABLE_CALENDAR: bool
ENABLE_MEMORIES: bool
ENABLE_MEMORY_SYSTEM_CONTEXT: bool
@ -1220,6 +1222,9 @@ async def update_admin_config(request: Request, form_data: AdminConfig, user=Dep
if form_data.DEFAULT_USER_ROLE not in ['pending', 'user', 'admin']:
updates.pop('ui.default_user_role', None)
if form_data.CHANNEL_MODEL_RESPONSE_MODE not in ['thread', 'channel']:
updates.pop('channels.model_response_mode', None)
pattern = r'^(-1|0|(-?\d+(\.\d+)?)(ms|s|m|h|d|w))$'
# Check if the input string matches the pattern

7
backend/open_webui/routers/channels.py
View file

@ -987,13 +987,18 @@ async def model_response_handler(request, channel, message, user, db=None):
db=db,
)
)[::-1]
response_parent_id = (
message.parent_id
if message.parent_id
else (message.id if await Config.get('channels.model_response_mode', 'thread') == 'thread' else None)
)
response_message, channel = await new_message_handler(
request,
channel.id,
MessageForm(
**{
'parent_id': (message.parent_id if message.parent_id else message.id),
'parent_id': response_parent_id,
'content': f'',
'data': {},
'meta': {

29
backend/open_webui/routers/files.py
View file

@ -108,6 +108,23 @@ def _cleanup_local_cache(file_path: str) -> None:
log.warning(f'Failed to clean up local cache for {file_path}: {e}')
def _matches_configured_mime_type(supported: list[str] | str, content_type: str) -> bool:
if isinstance(supported, str):
supported = supported.split(',')
supported = [item.strip() for item in (supported or []) if item.strip()]
if not supported:
return False
return bool(strict_match_mime_type(supported, content_type))
def _media_supported_for_extraction(
content_extraction_engine: str | None, supported: list[str] | str | None, content_type: str
) -> bool:
if supported is None:
return content_extraction_engine == 'external'
return bool(content_extraction_engine and _matches_configured_mime_type(supported, content_type))
async def process_uploaded_file(
request,
file,
@ -127,6 +144,10 @@ async def process_uploaded_file(
content_type = 'text/plain'
stt_supported = await Config.get('audio.stt.supported_content_types', [])
content_extraction_engine = await Config.get('rag.content_extraction_engine')
content_extraction_supported_media_mime_types = await Config.get(
'rag.content_extraction.supported_media_mime_types'
)
if content_type and strict_match_mime_type(stt_supported, content_type):
# Audio / STT-supported files → transcribe then index
@ -147,9 +168,10 @@ async def process_uploaded_file(
elif (
content_type
and content_type.startswith(('image/', 'video/'))
and await Config.get('rag.content_extraction_engine') != 'external'
and not _media_supported_for_extraction(
content_extraction_engine, content_extraction_supported_media_mime_types, content_type
)
):
# Media files without an external extraction engine
if content_type.startswith('video/'):
# Videos are stored as-is for downstream multimodal
# processing (Tools, vision models). Attempting text
@ -165,7 +187,8 @@ async def process_uploaded_file(
raise Exception(f'File type {content_type} is not supported for processing')
else:
# Documents, or any file when an external engine is configured
# Documents, or media files explicitly enabled for the
# configured content extraction engine.
if not content_type:
log.info(f'File type {file.content_type} is not provided, but trying to process anyway')
await process_file(

9
backend/open_webui/routers/retrieval.py
View file

@ -265,6 +265,7 @@ RETRIEVAL_CONFIG_KEYS = {
'CHUNK_MIN_SIZE_TARGET': 'rag.chunk_min_size_target',
'CHUNK_OVERLAP': 'rag.chunk_overlap',
'CHUNK_SIZE': 'rag.chunk_size',
'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': 'rag.content_extraction.supported_media_mime_types',
'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine',
'DATALAB_MARKER_ADDITIONAL_CONFIG': 'rag.datalab_marker_additional_config',
'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url',
@ -630,6 +631,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
'HYBRID_BM25_WEIGHT': config.HYBRID_BM25_WEIGHT,
# Content extraction settings
'CONTENT_EXTRACTION_ENGINE': config.CONTENT_EXTRACTION_ENGINE,
'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES,
'PDF_EXTRACT_IMAGES': config.PDF_EXTRACT_IMAGES,
'PDF_LOADER_MODE': config.PDF_LOADER_MODE,
'DATALAB_MARKER_API_KEY': config.DATALAB_MARKER_API_KEY,
@ -861,6 +863,7 @@ class ConfigForm(BaseModel):
# Content extraction settings
CONTENT_EXTRACTION_ENGINE: str | None = None
CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES: list[str] | None = None
PDF_EXTRACT_IMAGES: bool | None = None
PDF_LOADER_MODE: str | None = None
@ -973,6 +976,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
if form_data.CONTENT_EXTRACTION_ENGINE is not None
else config.CONTENT_EXTRACTION_ENGINE
)
config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES = (
form_data.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES
if form_data.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES is not None
else config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES
)
config.PDF_EXTRACT_IMAGES = (
form_data.PDF_EXTRACT_IMAGES if form_data.PDF_EXTRACT_IMAGES is not None else config.PDF_EXTRACT_IMAGES
)
@ -1333,6 +1341,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
'HYBRID_BM25_WEIGHT': config.HYBRID_BM25_WEIGHT,
# Content extraction settings
'CONTENT_EXTRACTION_ENGINE': config.CONTENT_EXTRACTION_ENGINE,
'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES,
'PDF_EXTRACT_IMAGES': config.PDF_EXTRACT_IMAGES,
'PDF_LOADER_MODE': config.PDF_LOADER_MODE,
'DATALAB_MARKER_API_KEY': config.DATALAB_MARKER_API_KEY,

1
src/lib/apis/retrieval/index.ts
View file

@ -52,6 +52,7 @@ type YoutubeConfigForm = {
type RAGConfigForm = {
PDF_EXTRACT_IMAGES?: boolean;
CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES?: string[];
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;

24
src/lib/components/admin/Settings/Documents.svelte
View file

@ -33,7 +33,7 @@
import AdminSettingRow from './AdminSettingRow.svelte';
import AdminSettingSection from './AdminSettingSection.svelte';
const i18n = getContext('i18n');
const i18n: any = getContext('i18n');
let updateEmbeddingModelLoading = false;
let updateRerankingModelLoading = false;
@ -69,7 +69,7 @@
hybrid: false
};
let RAGConfig = null;
let RAGConfig: any = null;
const inputClass =
'w-full h-7 rounded-lg border border-gray-100/50 bg-gray-50/40 px-2 text-xs text-gray-700 outline-hidden transition-colors placeholder:text-gray-300 focus:border-blue-400 dark:border-white/[0.04] dark:bg-white/[0.03] dark:text-gray-300 dark:placeholder:text-gray-700 dark:focus:border-blue-500';
const actionButtonClass =
@ -272,6 +272,12 @@
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS.trim() !== ''
? JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS)
: {},
CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES:
RAGConfig.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES.trim() === ''
? undefined
: RAGConfig.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES.split(',')
.map((mimeType: string) => mimeType.trim())
.filter((mimeType: string) => mimeType !== ''),
MINERU_PARAMS:
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
? JSON.parse(RAGConfig.MINERU_PARAMS)
@ -328,6 +334,9 @@
: config.EXTERNAL_DOCUMENT_LOADER_HEADERS;
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES = (
config?.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES ?? []
).join(', ');
config.RAG_TOKENIZER_MODEL = config?.RAG_TOKENIZER_MODEL ?? '';
RAGConfig = config;
@ -404,6 +413,17 @@
</SettingsSelect>
</AdminSettingRow>
<AdminSettingField
label={$i18n.t('Supported Media MIME Types')}
description={$i18n.t('Media upload MIME types the content extraction engine may process.')}
>
<input
class={inputClass}
placeholder={$i18n.t('image/*, video/*')}
bind:value={RAGConfig.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES}
/>
</AdminSettingField>
{#if RAGConfig.CONTENT_EXTRACTION_ENGINE === ''}
<AdminSettingRow
label={$i18n.t('PDF Extract Images (OCR)')}

19
src/lib/components/admin/Settings/General.svelte
View file

@ -5,6 +5,7 @@
import { getBackendConfig, getVersionUpdates } from '$lib/apis';
import { getAdminConfig, updateAdminConfig } from '$lib/apis/auths';
import { getBanners, setBanners } from '$lib/apis/configs';
import SettingsSelect from '$lib/components/common/SettingsSelect.svelte';
import Switch from '$lib/components/common/Switch.svelte';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import { WEBUI_BUILD_HASH, WEBUI_VERSION } from '$lib/constants';
@ -284,6 +285,24 @@
>
<Switch bind:state={adminConfig.ENABLE_CHANNELS} ariaLabelledbyId={labelId} />
</AdminSettingRow>
{#if adminConfig.ENABLE_CHANNELS}
<AdminSettingRow
label={$i18n.t('Model Response Mode')}
description={$i18n.t(
'Choose where model responses to root-level channel mentions are posted.'
)}
labelClassName="text-gray-500 dark:text-gray-500"
let:labelId
>
<SettingsSelect
bind:value={adminConfig.CHANNEL_MODEL_RESPONSE_MODE}
aria-labelledby={labelId}
>
<option value="thread">{$i18n.t('Thread')}</option>
<option value="channel">{$i18n.t('Channel')}</option>
</SettingsSelect>
</AdminSettingRow>
{/if}
<AdminSettingRow
label={$i18n.t('Calendar')}
description={$i18n.t('Allow users to access calendar features.')}