diff --git a/.env.example b/.env.example index 22b128ce8f..313024511a 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ ENABLE_RAG_CSV_SUMMARY=false # Set to true to preserve backing file records, storage blobs, and per-file vectors when files are removed from knowledge bases. ENABLE_KNOWLEDGE_FILE_RETENTION=false +# Comma-separated chunk metadata keys to expose to the model alongside retrieved content. +RAG_SOURCE_METADATA_KEYS='' + # Set to false to disable workspace Tools and Functions. ENABLE_PLUGINS=true diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 34663b5eda..510bdce1ba 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -12,6 +12,8 @@ body: Before submitting, search open and closed [Issues](https://github.com/open-webui/open-webui/issues) and [Discussions](https://github.com/open-webui/open-webui/discussions). The issue may already be reported or fixed on `dev`. + **Test on the latest release AND on `dev`, right before you submit this report, not last week.** A huge share of reports are for bugs already fixed on `dev`, sometimes weeks earlier, because the reporter only tested an old version and never rechecked. Reports that don't reproduce on latest and on current `dev` at submission time will be closed without further discussion, no exceptions. + Please do not open a code pull request for this report unless a maintainer asks for one, or the change is only i18n/localization. If you want to share code as reference, include it here as a local diff or patch. Actionable reproduction details are the most useful next step. Security vulnerabilities must not be reported publicly. Use the [GitHub security page](https://github.com/open-webui/open-webui/security) instead. @@ -23,12 +25,10 @@ body: options: - label: I searched open and closed issues and discussions for an existing report. required: true - - label: I checked whether this is already fixed on the `dev` branch or latest source. + - label: I reproduced this bug on the latest release AND on the current `dev` branch, right before submitting this report. I did not just check an old version or rely on a check from days ago. required: true - label: I understand that maintainers want a well-written issue before any code pull request. required: true - - label: I am using the latest available version of Open WebUI for my install method. - required: true - label: This is not a security vulnerability. required: true diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index b475ccbe02..5765a26ab6 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -741,6 +741,10 @@ else: except Exception: PGVECTOR_IVFFLAT_LISTS = 100 +PGVECTOR_ITERATIVE_SCAN = os.getenv('PGVECTOR_ITERATIVE_SCAN', 'relaxed_order').strip().lower() +if PGVECTOR_ITERATIVE_SCAN not in ('off', 'relaxed_order', 'strict_order'): + PGVECTOR_ITERATIVE_SCAN = 'relaxed_order' + # openGauss OPENGAUSS_DB_URL = os.getenv('OPENGAUSS_DB_URL', DATABASE_URL) @@ -1298,6 +1302,12 @@ TAVILY_API_KEY = os.getenv('TAVILY_API_KEY', '') TAVILY_EXTRACT_DEPTH = os.getenv('TAVILY_EXTRACT_DEPTH', 'basic') +STAAN_API_KEY = os.getenv('STAAN_API_KEY', '') + +STAAN_MARKET = os.getenv('STAAN_MARKET', 'en-us') + +STAAN_MAX_SNIPPETS = int(os.getenv('STAAN_MAX_SNIPPETS', '0')) + PLAYWRIGHT_WS_URL = os.getenv('PLAYWRIGHT_WS_URL', '') PLAYWRIGHT_TIMEOUT = int(os.getenv('PLAYWRIGHT_TIMEOUT', '10000')) @@ -3018,6 +3028,9 @@ DEFAULT_CONFIG = { 'web.search.sougou_api_sk': SOUGOU_API_SK, 'web.search.tavily_api_key': TAVILY_API_KEY, 'web.search.tavily_extract_depth': TAVILY_EXTRACT_DEPTH, + 'web.search.staan_api_key': STAAN_API_KEY, + 'web.search.staan_market': STAAN_MARKET, + 'web.search.staan_max_snippets': STAAN_MAX_SNIPPETS, 'web.loader.playwright_ws_url': PLAYWRIGHT_WS_URL, 'web.loader.playwright_timeout': PLAYWRIGHT_TIMEOUT, 'web.loader.firecrawl_api_key': FIRECRAWL_API_KEY, diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 9179f9a4b8..5936ad9939 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -371,6 +371,9 @@ ENABLE_QUERIES_CACHE = os.getenv('ENABLE_QUERIES_CACHE', 'False').lower() == 'tr ENABLE_ADMIN_CHAT_ACCESS = os.getenv('ENABLE_ADMIN_CHAT_ACCESS', 'True').lower() == 'true' RAG_SYSTEM_CONTEXT = os.getenv('RAG_SYSTEM_CONTEXT', 'False').lower() == 'true' +# Empty by default: chunk metadata also holds internal bookkeeping (file hashes, collection names, scores). +RAG_SOURCE_METADATA_KEYS = [key.strip() for key in os.getenv('RAG_SOURCE_METADATA_KEYS', '').split(',') if key.strip()] + #################################### # REDIS #################################### @@ -385,6 +388,14 @@ try: except ValueError: REDIS_RESPONSE_STREAM_TTL = 3600 +# Seconds a task survives without a heartbeat. 0 disables expiry. +try: + REDIS_TASK_TTL = int(os.getenv('REDIS_TASK_TTL', '300')) + if REDIS_TASK_TTL != 0 and REDIS_TASK_TTL < 60: + REDIS_TASK_TTL = 300 +except ValueError: + REDIS_TASK_TTL = 300 + REDIS_SENTINEL_HOSTS = os.getenv('REDIS_SENTINEL_HOSTS', '') REDIS_SENTINEL_PORT = os.getenv('REDIS_SENTINEL_PORT', '26379') diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 5ce9aa542c..e7c2915130 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -74,6 +74,7 @@ from open_webui.config import ( seed_registered_defaults, ) from open_webui.constants import ERROR_MESSAGES, TASKS +from open_webui.utils.recurrence import RecurrenceEvaluationTimeout from open_webui.env import ( USE_SLIM, AIOHTTP_CLIENT_SESSION_SSL, @@ -107,12 +108,14 @@ from open_webui.env import ( MAX_BODY_LOG_SIZE, # Redis REDIS_KEY_PREFIX, + REDIS_TASK_TTL, REDIS_URL, RESET_CONFIG_ON_START, SAFE_MODE, SCIM_TOKEN, VERSION, WEBSOCKET_HEARTBEAT_INTERVAL, + WEBSOCKET_MANAGER, # Admin Account Runtime Creation WEBUI_ADMIN_EMAIL, WEBUI_ADMIN_NAME, @@ -189,6 +192,7 @@ from open_webui.socket.main import ( get_user_id_from_session_pool, periodic_session_pool_cleanup, periodic_usage_pool_cleanup, + redis_event_listener, ) from open_webui.socket.main import ( app as socket_app, @@ -200,6 +204,7 @@ from open_webui.tasks import ( list_task_ids_by_item_id, list_tasks, redis_task_command_listener, + redis_task_heartbeat, stop_item_tasks, stop_task, ) # Import from tasks.py @@ -386,6 +391,11 @@ async def lifespan(app: FastAPI): if app.state.redis is not None: app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app)) + if REDIS_TASK_TTL > 0: + app.state.redis_task_heartbeat = asyncio.create_task(redis_task_heartbeat(app)) + + if WEBSOCKET_MANAGER == 'redis': + app.state.redis_event_listener = asyncio.create_task(redis_event_listener()) app.state.periodic_usage_pool_cleanup = asyncio.create_task(periodic_usage_pool_cleanup()) app.state.periodic_session_pool_cleanup = asyncio.create_task(periodic_session_pool_cleanup()) @@ -473,6 +483,12 @@ async def lifespan(app: FastAPI): if hasattr(app.state, 'redis_task_command_listener'): app.state.redis_task_command_listener.cancel() + if hasattr(app.state, 'redis_task_heartbeat'): + app.state.redis_task_heartbeat.cancel() + + if hasattr(app.state, 'redis_event_listener'): + app.state.redis_event_listener.cancel() + app.state.periodic_usage_pool_cleanup.cancel() app.state.periodic_session_pool_cleanup.cancel() app.state.scheduler_worker_loop.cancel() @@ -495,6 +511,12 @@ app = FastAPI( lifespan=lifespan, ) + +@app.exception_handler(RecurrenceEvaluationTimeout) +async def recurrence_timeout_handler(request: Request, exc: RecurrenceEvaluationTimeout): + return JSONResponse(status_code=400, content={'detail': str(exc)}) + + # Used by readiness checks to gate traffic until startup work is done. app.state.startup_complete = False @@ -1464,8 +1486,8 @@ async def chat_completion( async def run_initial_title_generation(): try: await background_tasks_handler(title_ctx) - except Exception as e: - log.debug('Error generating initial chat title: %s', e) + except Exception: + log.exception('Error generating initial chat title') asyncio.create_task(run_initial_title_generation()) else: @@ -1637,8 +1659,8 @@ async def chat_completion( # When the upstream provider returns an error (e.g. HTTP 400 # content-filter, quota exceeded), generate_chat_completion # returns a JSONResponse instead of raising. Detect this and - # raise so the except-block below emits chat:message:error + - # chat:tasks:cancel, unblocking the frontend. + # raise so the except-block below emits a terminal + # chat:message:error, unblocking the frontend. if isinstance(response, JSONResponse) and response.status_code >= 400: raise Exception(get_response_error_detail(response)) @@ -1674,6 +1696,7 @@ async def chat_completion( { 'parentId': metadata.get('user_message_id', None), 'error': {'content': error_detail}, + 'done': True, }, ) @@ -1682,12 +1705,9 @@ async def chat_completion( await event_emitter( { 'type': 'chat:message:error', - 'data': {'error': {'content': error_detail}}, + 'data': {'error': {'content': error_detail}, 'done': True}, } ) - await event_emitter( - {'type': 'chat:tasks:cancel'}, - ) except Exception: pass diff --git a/backend/open_webui/models/automations.py b/backend/open_webui/models/automations.py index 11a906670b..8c7f6089e3 100644 --- a/backend/open_webui/models/automations.py +++ b/backend/open_webui/models/automations.py @@ -312,6 +312,7 @@ class AutomationTable: rows = result.scalars().all() from open_webui.utils.automations import next_run_ns + from open_webui.utils.recurrence import RecurrenceEvaluationTimeout # Batch-fetch user timezones so rescheduling respects each # user's local timezone instead of falling back to server time. @@ -323,13 +324,20 @@ class AutomationTable: tz_result = await db.execute(select(User.id, User.timezone).where(User.id.in_(user_ids))) timezone_by_user_id = {uid: tz for uid, tz in tz_result.all()} + claimed = [] for row in rows: + try: + next_run_at = await next_run_ns(row.data.get('rrule', ''), tz=timezone_by_user_id.get(row.user_id)) + except RecurrenceEvaluationTimeout: + log.warning('Skipping automation %s: recurrence evaluation timed out', row.id) + continue row.last_run_at = now_ns - row.next_run_at = next_run_ns(row.data.get('rrule', ''), tz=timezone_by_user_id.get(row.user_id)) + row.next_run_at = next_run_at + claimed.append(row) await db.commit() - return [AutomationModel.model_validate(r) for r in rows] + return [AutomationModel.model_validate(r) for r in claimed] #################### diff --git a/backend/open_webui/models/calendar.py b/backend/open_webui/models/calendar.py index 3a3147a7ad..fd7390fcea 100644 --- a/backend/open_webui/models/calendar.py +++ b/backend/open_webui/models/calendar.py @@ -8,7 +8,7 @@ from open_webui.constants import ERROR_MESSAGES from open_webui.models.access_grants import AccessGrantModel, AccessGrants from open_webui.models.groups import Groups from open_webui.models.users import User, UserModel, UserResponse -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import ( JSON, BigInteger, @@ -179,6 +179,20 @@ class CalendarUpdateForm(BaseModel): access_grants: Optional[list[dict]] = None +async def validate_calendar_rrule(value: Optional[str]) -> None: + if value: + from open_webui.utils.recurrence import rrule_interval_seconds + + try: + interval = await rrule_interval_seconds(value) + except ValueError: + raise + except Exception as e: + raise ValueError(ERROR_MESSAGES.AUTOMATION_INVALID_RRULE(e)) from e + if interval is not None and interval < MIN_CALENDAR_RRULE_INTERVAL_SECONDS: + raise ValueError(ERROR_MESSAGES.CALENDAR_RRULE_TOO_FREQUENT) + + class CalendarEventForm(BaseModel): calendar_id: str title: str @@ -193,22 +207,6 @@ class CalendarEventForm(BaseModel): meta: Optional[dict] = None attendees: Optional[list[dict]] = None - @field_validator('rrule') - @classmethod - def reject_sub_daily_rrule(cls, value: Optional[str]) -> Optional[str]: - if value: - from open_webui.utils.automations import rrule_interval_seconds - - try: - interval = rrule_interval_seconds(value) - except ValueError: - raise - except Exception as e: - raise ValueError(ERROR_MESSAGES.AUTOMATION_INVALID_RRULE(e)) - if interval is not None and interval < MIN_CALENDAR_RRULE_INTERVAL_SECONDS: - raise ValueError(ERROR_MESSAGES.CALENDAR_RRULE_TOO_FREQUENT) - return value - class CalendarEventUpdateForm(BaseModel): calendar_id: Optional[str] = None @@ -225,22 +223,6 @@ class CalendarEventUpdateForm(BaseModel): is_cancelled: Optional[bool] = None attendees: Optional[list[dict]] = None - @field_validator('rrule') - @classmethod - def reject_sub_daily_rrule(cls, value: Optional[str]) -> Optional[str]: - if value: - from open_webui.utils.automations import rrule_interval_seconds - - try: - interval = rrule_interval_seconds(value) - except ValueError: - raise - except Exception as e: - raise ValueError(ERROR_MESSAGES.AUTOMATION_INVALID_RRULE(e)) - if interval is not None and interval < MIN_CALENDAR_RRULE_INTERVAL_SECONDS: - raise ValueError(ERROR_MESSAGES.CALENDAR_RRULE_TOO_FREQUENT) - return value - class RSVPForm(BaseModel): status: str # 'accepted' | 'declined' | 'tentative' | 'pending' @@ -465,6 +447,7 @@ class CalendarEventTable: async def insert_new_event( self, user_id: str, form_data: CalendarEventForm, db: Optional[AsyncSession] = None ) -> Optional[CalendarEventModel]: + await validate_calendar_rrule(form_data.rrule) async with get_async_db_context(db) as db: now = int(time.time_ns()) event = CalendarEvent( @@ -695,6 +678,7 @@ class CalendarEventTable: async def update_event_by_id( self, id: str, form_data: CalendarEventUpdateForm, db: Optional[AsyncSession] = None ) -> Optional[CalendarEventModel]: + await validate_calendar_rrule(form_data.rrule) async with get_async_db_context(db) as db: result = await db.execute(select(CalendarEvent).filter(CalendarEvent.id == id)) event = result.scalars().first() diff --git a/backend/open_webui/models/chats.py b/backend/open_webui/models/chats.py index 508b0d0b4e..2af71fd944 100644 --- a/backend/open_webui/models/chats.py +++ b/backend/open_webui/models/chats.py @@ -258,6 +258,7 @@ class ChatForm(BaseModel): class ChatImportForm(ChatForm): meta: dict | None = {} pinned: bool | None = False + archived: bool | None = False current_message_id: str | None = None created_at: int | None = None updated_at: int | None = None @@ -304,6 +305,7 @@ class ChatTitleIdResponse(BaseModel): last_read_at: int | None = None snippet: str | None = None active: bool = False + archived: bool = False class SharedChatResponse(BaseModel): @@ -644,6 +646,7 @@ class ChatTable: 'meta': form_data.meta, 'variables': form_data.variables or {}, 'pinned': form_data.pinned, + 'archived': form_data.archived, 'folder_id': form_data.folder_id, 'current_message_id': form_data.current_message_id or self.get_current_message_id(form_data.chat), 'created_at': (form_data.created_at if form_data.created_at else int(time.time())), @@ -1985,10 +1988,8 @@ class ChatTable: ] # Extract folder names - folders = await Folders.search_folders_by_names( - user_id, - [word.replace('folder:', '') for word in search_text_words if word.startswith('folder:')], - ) + folder_names = [word.replace('folder:', '') for word in search_text_words if word.startswith('folder:')] + folders = await Folders.search_folders_by_names(user_id, folder_names) folder_ids = [folder.id for folder in folders] is_pinned = None @@ -2032,7 +2033,7 @@ class ChatTable: else: stmt = stmt.filter(Chat.share_id.is_(None)) - if folder_ids: + if folder_names: stmt = stmt.filter(Chat.folder_id.in_(folder_ids)) # Check if the database dialect is either 'sqlite' or 'postgresql' @@ -2535,18 +2536,15 @@ class ChatTable: except Exception: return False - async def move_chats_by_user_id_and_folder_id( + async def move_chats_by_folder_id( self, - user_id: str, folder_id: str, new_folder_id: str | None, db: AsyncSession | None = None, ) -> bool: try: async with get_async_db_context(db) as session: - await session.execute( - update(Chat).filter_by(user_id=user_id, folder_id=folder_id).values(folder_id=new_folder_id) - ) + await session.execute(update(Chat).filter_by(folder_id=folder_id).values(folder_id=new_folder_id)) await session.commit() return True diff --git a/backend/open_webui/models/prompts.py b/backend/open_webui/models/prompts.py index e6e07eacc7..f0e4a3e9b8 100644 --- a/backend/open_webui/models/prompts.py +++ b/backend/open_webui/models/prompts.py @@ -506,14 +506,16 @@ class PromptsTable: ) # Update prompt fields - prompt.name = form_data.name prompt.command = form_data.command - prompt.content = form_data.content - prompt.data = form_data.data or prompt.data - prompt.meta = form_data.meta or prompt.meta - if form_data.tags is not None: - prompt.tags = form_data.tags + if form_data.is_production: + prompt.name = form_data.name + prompt.content = form_data.content + prompt.data = form_data.data or prompt.data + prompt.meta = form_data.meta or prompt.meta + + if form_data.tags is not None: + prompt.tags = form_data.tags if form_data.access_grants is not None: await AccessGrants.set_access_grants('prompt', prompt.id, form_data.access_grants, db=session) @@ -531,7 +533,7 @@ class PromptsTable: 'command': prompt.command, 'data': form_data.data or {}, 'meta': form_data.meta or {}, - 'tags': prompt.tags or [], + 'tags': form_data.tags if form_data.tags is not None else (prompt.tags or []), 'access_grants': [grant.model_dump() for grant in current_access_grants], } diff --git a/backend/open_webui/models/users.py b/backend/open_webui/models/users.py index 0e9d2bb6c9..0235b7d787 100644 --- a/backend/open_webui/models/users.py +++ b/backend/open_webui/models/users.py @@ -803,7 +803,9 @@ class UsersTable: return None scim = dict(user.scim or {}) scim[provider] = {'external_id': external_id} - user.scim = scim + if scim != user.scim: + user.scim = scim + user.updated_at = int(time.time()) await session.commit() return UserModel.model_validate(user) diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index 9188c723c1..74221f1aea 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -91,6 +91,7 @@ known_source_ext = [ 'yaml', 'yml', 'toml', + 'svg', ] known_archive_ext = {'docx', 'epub', 'odt', 'pptx', 'xlsx'} @@ -290,8 +291,17 @@ class DoclingLoader: ) if r.ok: result = r.json() + # Docling reports failed and skipped conversions inside HTTP 200 responses. + conversion_status = result.get('status') + if conversion_status in ['failure', 'skipped']: + error_details = ( + '; '.join(filter(None, (error.get('error_message') for error in result.get('errors', [])))) + or 'no error message provided' + ) + raise Exception(f'Error calling Docling: conversion status {conversion_status} - {error_details}') + document_data = result.get('document', {}) - md_content = document_data.get('md_content', '') + md_content = document_data.get('md_content') or '' text = md_content or '' metadata = {'Content-Type': self.mime_type} if self.mime_type else {} diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index 531b2605e2..055f750122 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -34,6 +34,7 @@ from open_webui.env import ( ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS, MPS_INFERENCE_LOCK, OFFLINE_MODE, + RAG_SOURCE_METADATA_KEYS, USE_SLIM, ) from open_webui.models.access_grants import AccessGrants @@ -1361,6 +1362,11 @@ async def filter_accessible_collections( return validated +def filter_source_metadata(metadata: dict) -> dict: + """Keep only the chunk metadata keys the operator allowed the model to see.""" + return {key: metadata[key] for key in RAG_SOURCE_METADATA_KEYS if metadata.get(key) is not None} + + async def get_sources_from_items( request, items, diff --git a/backend/open_webui/retrieval/vector/dbs/opengauss.py b/backend/open_webui/retrieval/vector/dbs/opengauss.py index cfb7166250..d8f84a2580 100644 --- a/backend/open_webui/retrieval/vector/dbs/opengauss.py +++ b/backend/open_webui/retrieval/vector/dbs/opengauss.py @@ -311,6 +311,7 @@ class OpenGaussClient(VectorDBBase): results = query.all() if not results: + self.session.rollback() return None ids = [[result.id for result in results]] @@ -333,6 +334,7 @@ class OpenGaussClient(VectorDBBase): results = query.all() if not results: + self.session.rollback() return None ids = [[result.id for result in results]] diff --git a/backend/open_webui/retrieval/vector/dbs/pgvector.py b/backend/open_webui/retrieval/vector/dbs/pgvector.py index 75daeccd12..04fdaed1c1 100644 --- a/backend/open_webui/retrieval/vector/dbs/pgvector.py +++ b/backend/open_webui/retrieval/vector/dbs/pgvector.py @@ -8,6 +8,7 @@ from open_webui.config import ( PGVECTOR_HNSW_M, PGVECTOR_INDEX_METHOD, PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH, + PGVECTOR_ITERATIVE_SCAN, PGVECTOR_IVFFLAT_LISTS, PGVECTOR_PGCRYPTO, PGVECTOR_PGCRYPTO_KEY, @@ -154,6 +155,7 @@ class PgvectorClient(VectorDBBase): index_method, index_options = self._vector_index_configuration() self._ensure_vector_index(index_method, index_options) self._ensure_text_search_index() + self.iterative_scan_sql = self._iterative_scan_setting(index_method) self.session.execute( text( @@ -223,6 +225,9 @@ class PgvectorClient(VectorDBBase): ) if not existing_index_def: + if index_method == 'ivfflat' and not self._has_enough_ivfflat_training_rows(): + return + index_sql = ( f'CREATE INDEX IF NOT EXISTS {index_name} ' f'ON document_chunk USING {index_method} (vector {VECTOR_OPCLASS})' @@ -237,6 +242,38 @@ class PgvectorClient(VectorDBBase): f' {index_options}' if index_options else '', ) + def _has_enough_ivfflat_training_rows(self) -> bool: + # ivfflat samples 50 rows per list to place its centroids, so recall stays poor until the table holds that many + min_training_rows = 50 * PGVECTOR_IVFFLAT_LISTS + row_count = self.session.execute( + text('SELECT count(*) FROM (SELECT 1 FROM document_chunk LIMIT :min_training_rows) AS sample'), + {'min_training_rows': min_training_rows}, + ).scalar() + + if row_count < min_training_rows: + log.info( + "Deferring vector index 'idx_document_chunk_vector' until document_chunk holds %s rows to cluster on, " + 'it has %s. Searches run as an exact scan until then.', + min_training_rows, + row_count, + ) + return False + return True + + def _iterative_scan_setting(self, index_method: str) -> Optional[str]: + if PGVECTOR_ITERATIVE_SCAN == 'off': + return None + + version = self.session.execute(text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")).scalar() + version_parts = [int(part) for part in (version or '').split('.') if part.isdigit()] + if version_parts[:2] < [0, 8]: + log.info('Iterative scan needs pgvector 0.8 or newer, the server has %s.', version or 'none') + return None + + # ivfflat only accepts relaxed_order + mode = 'relaxed_order' if index_method == 'ivfflat' else PGVECTOR_ITERATIVE_SCAN + return f'SET LOCAL {index_method}.iterative_scan = {mode}' + def _ensure_text_search_index(self) -> None: if PGVECTOR_PGCRYPTO: return @@ -503,6 +540,9 @@ class PgvectorClient(VectorDBBase): .order_by(query_vectors.c.qid, subq.c.distance) ) + if self.iterative_scan_sql: + self.session.execute(text(self.iterative_scan_sql)) + result_proxy = self.session.execute(stmt) results = result_proxy.all() @@ -512,6 +552,7 @@ class PgvectorClient(VectorDBBase): metadatas = [[] for _ in range(num_queries)] if not results: + self.session.rollback() return SearchResult( ids=ids, distances=distances, @@ -631,6 +672,7 @@ class PgvectorClient(VectorDBBase): results = query.all() if not results: + self.session.rollback() return None ids = [[result.id for result in results]] @@ -670,6 +712,7 @@ class PgvectorClient(VectorDBBase): results = query.all() if not results: + self.session.rollback() return None ids = [[result.id for result in results]] diff --git a/backend/open_webui/retrieval/web/staan.py b/backend/open_webui/retrieval/web/staan.py new file mode 100644 index 0000000000..c1af2b5b2a --- /dev/null +++ b/backend/open_webui/retrieval/web/staan.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import requests +from open_webui.retrieval.web.main import SearchResult, get_filtered_results + + +def search_staan( + api_key: str, + query: str, + count: int, + filter_list: list[str] | None = None, + market: str | None = None, + max_snippets: int | None = None, +) -> list[SearchResult]: + """Search using Staan's Web Search API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A Staan API key + query (str): The query to search for + count (int): The maximum number of results to return + filter_list (list[str] | None): The domains to allow or block + market (str | None): The market to search in, e.g. 'en-us' + max_snippets (int | None): The maximum extra snippets to request per result + + Returns: + A list of SearchResult objects. + """ + url = 'https://api.staan.ai/v2/search/web' + headers = { + 'Accept': 'application/json', + 'Authorization': f'Bearer {api_key}', + } + params = {'q': query, 'market': market} + + if max_snippets: + params['extra_snippets'] = 'true' + params['max_snippets'] = max_snippets + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + + results = response.json().get('web', {}).get('results', []) + if filter_list: + results = get_filtered_results(results, filter_list) + + return [ + SearchResult( + link=result.get('url', ''), + title=result.get('title'), + snippet=_build_snippet(result), + ) + for result in results[:count] + ] + + +def _build_snippet(result: dict) -> str: + """Combine the snippet and the extra snippets list into a single string.""" + parts = [result.get('snippet')] + parts.extend(extra.get('chunk') for extra in result.get('extra_snippets', [])) + return '\n\n'.join(part for part in parts if part) diff --git a/backend/open_webui/routers/audio.py b/backend/open_webui/routers/audio.py index dfc0ecde65..b9ab84ea0e 100644 --- a/backend/open_webui/routers/audio.py +++ b/backend/open_webui/routers/audio.py @@ -297,8 +297,8 @@ async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm raise HTTPException(400, 'Local TTS is unavailable in slim. Select an external text-to-speech engine.') await Config.upsert( { - **config_updates(form_data.tts.model_dump(), TTS_CONFIG_KEYS), - **config_updates(form_data.stt.model_dump(), STT_CONFIG_KEYS), + **config_updates(form_data.tts.model_dump(exclude_unset=True), TTS_CONFIG_KEYS), + **config_updates(form_data.stt.model_dump(exclude_unset=True), STT_CONFIG_KEYS), } ) diff --git a/backend/open_webui/routers/auths.py b/backend/open_webui/routers/auths.py index 75a3e0961d..16ae0422e8 100644 --- a/backend/open_webui/routers/auths.py +++ b/backend/open_webui/routers/auths.py @@ -1752,6 +1752,7 @@ async def token_exchange( user=user, user_data=user_data, provider=provider, + access_token=form_data.token, db=db, ) if await Config.get('oauth.enable_group_mapping'): diff --git a/backend/open_webui/routers/automations.py b/backend/open_webui/routers/automations.py index caf8879e28..4b44a8b8ed 100644 --- a/backend/open_webui/routers/automations.py +++ b/backend/open_webui/routers/automations.py @@ -87,7 +87,7 @@ async def check_automation_limits(request, user, rrule_str: str, db, is_create: if min_interval: min_interval = int(min_interval) if min_interval > 0: - interval = rrule_interval_seconds(rrule_str) + interval = await rrule_interval_seconds(rrule_str) if interval is not None and interval < min_interval: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -150,7 +150,7 @@ async def enrich_automation(automation: AutomationModel, db: AsyncSession, tz: s return AutomationResponse( **automation.model_dump(), last_run=last_run, - next_runs=next_n_runs_ns(automation.data['rrule'], tz=tz), + next_runs=await next_n_runs_ns(automation.data['rrule'], tz=tz), ) @@ -216,7 +216,7 @@ async def create_new_automation( await check_automation_folder_access(form_data.folder_id, user, db) await check_automation_channel_access(form_data, user, db) try: - validate_rrule(form_data.data.rrule, tz=user.timezone) + await validate_rrule(form_data.data.rrule, tz=user.timezone) except ValueError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -226,7 +226,7 @@ async def create_new_automation( await check_automation_limits(request, user, form_data.data.rrule, db, is_create=True) tz = user.timezone - automation = await Automations.insert(user.id, form_data, next_run_ns(form_data.data.rrule, tz=tz), db=db) + automation = await Automations.insert(user.id, form_data, await next_run_ns(form_data.data.rrule, tz=tz), db=db) response = await enrich_automation(automation, db, tz=tz) await publish_event( request, @@ -276,7 +276,7 @@ async def update_automation_by_id( await check_automation_channel_access(form_data, user, db) try: - validate_rrule(form_data.data.rrule, tz=user.timezone) + await validate_rrule(form_data.data.rrule, tz=user.timezone) except ValueError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -286,7 +286,7 @@ async def update_automation_by_id( await check_automation_limits(request, user, form_data.data.rrule, db, is_create=False) tz = user.timezone - updated = await Automations.update_by_id(id, form_data, next_run_ns(form_data.data.rrule, tz=tz), db=db) + updated = await Automations.update_by_id(id, form_data, await next_run_ns(form_data.data.rrule, tz=tz), db=db) response = await enrich_automation(updated, db, tz=tz) await publish_event( request, @@ -313,7 +313,7 @@ async def toggle_automation_by_id( await check_automations_permission(request, user) automation = await Automations.get_by_id(id, db=db) check_automation_access(automation, user) - toggled = await Automations.toggle(id, next_run_ns(automation.data['rrule'], tz=user.timezone), db=db) + toggled = await Automations.toggle(id, await next_run_ns(automation.data['rrule'], tz=user.timezone), db=db) response = await enrich_automation(toggled, db, tz=user.timezone) await publish_event( request, diff --git a/backend/open_webui/routers/calendar.py b/backend/open_webui/routers/calendar.py index f95be48e84..86d9824803 100644 --- a/backend/open_webui/routers/calendar.py +++ b/backend/open_webui/routers/calendar.py @@ -273,7 +273,10 @@ async def get_events( async def create_event(request: Request, form_data: CalendarEventForm, user: UserModel = Depends(get_verified_user)): await check_calendar_permission(request, user) await _check_calendar_access(form_data.calendar_id, user, 'write') - event = await CalendarEvents.insert_new_event(user.id, form_data) + try: + event = await CalendarEvents.insert_new_event(user.id, form_data) + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) from e await publish_event( request, EVENTS.CALENDAR_EVENT_CREATED, @@ -325,7 +328,10 @@ async def update_event( if form_data.calendar_id is not None and form_data.calendar_id != event.calendar_id: await _check_calendar_access(form_data.calendar_id, user, 'write') - updated = await CalendarEvents.update_event_by_id(event_id, form_data) + try: + updated = await CalendarEvents.update_event_by_id(event_id, form_data) + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) from e if not updated: raise HTTPException(status_code=500, detail='Failed to update') await publish_event( diff --git a/backend/open_webui/routers/chats.py b/backend/open_webui/routers/chats.py index 13cf13605b..ed339a601a 100644 --- a/backend/open_webui/routers/chats.py +++ b/backend/open_webui/routers/chats.py @@ -888,6 +888,7 @@ async def search_user_chats( created_at=chat.created_at, last_read_at=chat.last_read_at, snippet=chat_search_snippet(chat.chat, search_text), + archived=chat.archived, ) ) @@ -1592,7 +1593,7 @@ async def delete_chat_by_id( # Cancel any in-flight LLM tasks (streaming, title/tags generation) before # deleting the chat to prevent orphaned requests. await stop_item_tasks(request.app.state.redis, id) - await Chats.delete_orphan_tags_for_user(chat.meta.get('tags', []), user.id, threshold=1, db=db) + await Chats.delete_orphan_tags_for_user(chat.meta.get('tags', []), chat.user_id, threshold=1, db=db) # Cascade to internal child chats spawned from this one. for child_id in await Chats.get_internal_chat_ids_by_parent_id(id, chat.user_id): @@ -1690,15 +1691,6 @@ async def fork_chat_by_id( history = (chat.chat or {}).get('history') or {} messages_map = await Chats.get_messages_map_by_chat_id(id) or history.get('messages') or {} - if any( - message.get('role') == 'assistant' and message.get('done') is False - for message in messages_map.values() - if isinstance(message, dict) - ): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail='Wait for the current response to finish before forking.', - ) source_message_id = ( (form_data.message_id if form_data else None) or chat.current_message_id or history.get('currentId') @@ -1715,6 +1707,22 @@ async def fork_chat_by_id( detail=detail, ) from exc + # An unfinished message is stale unless it is awaiting tool approval + for message in fork_history['messages'].values(): + if message.get('role') != 'assistant' or message.get('done') is not False: + continue + + output = message.get('output') + if isinstance(output, list) and any( + isinstance(item, dict) + and item.get('type') == 'function_call' + and item.get('status') in {'pending', 'queued', 'requires_approval'} + for item in output + ): + continue + + message['done'] = True + updated_chat = {**(chat.chat or {})} updated_chat.pop('currentId', None) updated_chat.update( diff --git a/backend/open_webui/routers/folders.py b/backend/open_webui/routers/folders.py index 7bc8641d68..046a2d8834 100644 --- a/backend/open_webui/routers/folders.py +++ b/backend/open_webui/routers/folders.py @@ -401,7 +401,7 @@ async def update_folder_parent_id_by_id( form_data.parent_id, user.id, folder.name, db=db ) - if existing_folder: + if existing_folder and existing_folder.id != id: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT('Folder already exists'), @@ -683,7 +683,7 @@ async def delete_folder_by_id( folder_owner_id = folder.user_id folder_ids = await Folders.get_folder_ids_by_id_and_user_id_in_subtree(id, folder_owner_id, db=db) - if await Chats.count_chats_by_folder_ids_and_user_id(folder_ids, folder_owner_id, db=db): + if delete_contents and await Chats.count_chats_by_folder_ids_and_user_id(folder_ids, folder_owner_id, db=db): chat_delete_permission = await has_permission( user.id, 'chat.delete', await Config.get('user.permissions'), db=db ) @@ -704,8 +704,8 @@ async def delete_folder_by_id( for folder_id in folder_ids: if delete_contents: await Chats.delete_chats_by_user_id_and_folder_id(folder_owner_id, folder_id, db=db) - else: - await Chats.move_chats_by_user_id_and_folder_id(folder_owner_id, folder_id, None, db=db) + + await Chats.move_chats_by_folder_id(folder_id, None, db=db) # Clean up access grants for this folder await AccessGrants.revoke_all_access('folder', folder_id, db=db) diff --git a/backend/open_webui/routers/images.py b/backend/open_webui/routers/images.py index a93623ee66..8c6e772896 100644 --- a/backend/open_webui/routers/images.py +++ b/backend/open_webui/routers/images.py @@ -329,38 +329,34 @@ def get_automatic1111_api_auth(image_config): return f'Basic {auth1111_base64_encoded_string}' -@router.get('/config/url/verify') -async def verify_url(request: Request, user=Depends(get_admin_user)): - image_config = await get_image_config() - if image_config.IMAGE_GENERATION_ENGINE == 'automatic1111': - try: - session = await get_session() - async with session.get( - url=f'{image_config.AUTOMATIC1111_BASE_URL}/sdapi/v1/options', - headers={'authorization': get_automatic1111_api_auth(image_config)}, - ssl=AIOHTTP_CLIENT_SESSION_SSL, - ) as r: - r.raise_for_status() - return True - except Exception: - raise HTTPException(status_code=400, detail=ERROR_MESSAGES.INVALID_URL) - elif image_config.IMAGE_GENERATION_ENGINE == 'comfyui': - headers = None - if image_config.COMFYUI_API_KEY: - headers = {'Authorization': f'Bearer {image_config.COMFYUI_API_KEY}'} - try: - session = await get_session() - async with session.get( - url=f'{image_config.COMFYUI_BASE_URL}/object_info', - headers=headers, - ssl=AIOHTTP_CLIENT_SESSION_SSL, - ) as r: - r.raise_for_status() - return True - except Exception: - raise HTTPException(status_code=400, detail=ERROR_MESSAGES.INVALID_URL) +class ConnectionVerificationForm(BaseModel): + engine: str + url: str + key: str | None = None + + +@router.post('/verify') +async def verify_connection(form_data: ConnectionVerificationForm, user=Depends(get_admin_user)): + url = form_data.url.rstrip('/') + headers = {} + if form_data.engine == 'automatic1111': + url = f'{url}/sdapi/v1/options' + if form_data.key is not None: + headers['Authorization'] = f'Basic {base64.b64encode(form_data.key.encode("utf-8")).decode("utf-8")}' + elif form_data.engine == 'comfyui': + url = f'{url}/object_info' + if form_data.key: + headers['Authorization'] = f'Bearer {form_data.key}' else: - return True + raise HTTPException(status_code=400, detail='Unsupported image engine') + + try: + session = await get_session() + async with session.get(url=url, headers=headers, ssl=AIOHTTP_CLIENT_SESSION_SSL) as r: + r.raise_for_status() + return True + except Exception: + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.INVALID_URL) @router.get('/models') @@ -671,7 +667,9 @@ async def image_generations( if image_url := image.get('url', None): image_data, content_type = await get_image_data( image_url, - {k: v for k, v in headers.items() if k != 'Content-Type'}, + {k: v for k, v in headers.items() if k != 'Content-Type'} + if _is_same_origin(image_url, image_config.IMAGES_OPENAI_API_BASE_URL) + else None, ) else: image_data, content_type = await get_image_data(image['b64_json']) @@ -922,11 +920,8 @@ async def image_edits( if data.startswith('http://') or data.startswith('https://'): parsed = urlparse(data) - if ( - parsed.netloc == urlparse(str(request.base_url)).netloc - and parsed.path.startswith('/api/v1/files/') - and '/content' in parsed.path - ): + # Fetching /api/v1/files/{id}/content over the network would be unauthenticated. + if parsed.path.startswith('/api/v1/files/') and '/content' in parsed.path: return await load_url_image(parsed.path) # Validate URL to prevent SSRF attacks against local/private networks. @@ -1050,7 +1045,9 @@ async def image_edits( if image_url := image.get('url', None): image_data, content_type = await get_image_data( image_url, - {k: v for k, v in headers.items() if k != 'Content-Type'}, + {k: v for k, v in headers.items() if k != 'Content-Type'} + if _is_same_origin(image_url, image_config.IMAGES_EDIT_OPENAI_API_BASE_URL) + else None, ) else: image_data, content_type = await get_image_data(image['b64_json']) diff --git a/backend/open_webui/routers/notes.py b/backend/open_webui/routers/notes.py index c1951d2097..a84dd3f94e 100644 --- a/backend/open_webui/routers/notes.py +++ b/backend/open_webui/routers/notes.py @@ -554,14 +554,15 @@ async def update_note_by_id( ): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.DEFAULT()) - form_data.access_grants = await filter_allowed_access_grants( - await Config.get('user.permissions'), - user.id, - user.role, - form_data.access_grants, - 'sharing.public_notes', - db=db, - ) + if form_data.access_grants is not None: + form_data.access_grants = await filter_allowed_access_grants( + await Config.get('user.permissions'), + user.id, + user.role, + form_data.access_grants, + 'sharing.public_notes', + db=db, + ) try: note = await Notes.update_note_by_id(id, form_data, db=db) diff --git a/backend/open_webui/routers/openai.py b/backend/open_webui/routers/openai.py index 854a8f8389..63676d166e 100644 --- a/backend/open_webui/routers/openai.py +++ b/backend/open_webui/routers/openai.py @@ -1425,6 +1425,11 @@ def convert_to_responses_payload(payload: dict) -> dict: converted_tools.append(tool) responses_payload['tools'] = converted_tools + # Responses API expects a forced function choice as {"type": "function", "name": ...} + tool_choice = responses_payload.get('tool_choice') + if isinstance(tool_choice, dict) and isinstance(tool_choice.get('function'), dict): + responses_payload['tool_choice'] = {'type': 'function', 'name': tool_choice['function'].get('name', '')} + return responses_payload @@ -1432,17 +1437,32 @@ def convert_responses_result(response: dict) -> dict: """ Convert non-streaming Responses API result to Chat Completions format. - Extracts text from message output items so all downstream consumers + Extracts text and function calls from output items so all downstream consumers (frontend tasks, get_content_from_response) work without modification. """ output_items = response.get('output', []) content = '' + tool_calls = [] for item in output_items: if item.get('type') == 'message': for part in item.get('content', []): if part.get('type') == 'output_text': content += part.get('text', '') + elif item.get('type') == 'function_call': + arguments = item.get('arguments', '{}') + if not isinstance(arguments, str): + arguments = JSONCodec.dumps(arguments) + tool_calls.append( + { + 'id': item.get('call_id', ''), + 'type': 'function', + 'function': { + 'name': item.get('name', ''), + 'arguments': arguments, + }, + } + ) return { 'id': response.get('id', ''), @@ -1454,8 +1474,9 @@ def convert_responses_result(response: dict) -> dict: 'message': { 'role': 'assistant', 'content': content, + **({'tool_calls': tool_calls} if tool_calls else {}), }, - 'finish_reason': 'stop', + 'finish_reason': 'tool_calls' if tool_calls else 'stop', } ], 'usage': response.get('usage', {}), diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 885f45d46a..c36b0b8a53 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -115,6 +115,7 @@ from open_webui.retrieval.web.serphouse import search_serphouse from open_webui.retrieval.web.serply import search_serply from open_webui.retrieval.web.serpstack import search_serpstack from open_webui.retrieval.web.sougou import search_sougou +from open_webui.retrieval.web.staan import search_staan from open_webui.retrieval.web.tavily import search_tavily from open_webui.retrieval.web.utils import get_web_loader from open_webui.retrieval.web.yacy import search_yacy @@ -380,6 +381,9 @@ RETRIEVAL_CONFIG_KEYS = { 'SERPSTACK_HTTPS': 'web.search.serpstack_https', 'SOUGOU_API_SID': 'web.search.sougou_api_sid', 'SOUGOU_API_SK': 'web.search.sougou_api_sk', + 'STAAN_API_KEY': 'web.search.staan_api_key', + 'STAAN_MARKET': 'web.search.staan_market', + 'STAAN_MAX_SNIPPETS': 'web.search.staan_max_snippets', 'TAVILY_API_KEY': 'web.search.tavily_api_key', 'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth', 'TEXT_SPLITTER': 'rag.text_splitter', @@ -737,6 +741,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): 'SERPLY_API_KEY': config.SERPLY_API_KEY, 'DDGS_BACKEND': config.DDGS_BACKEND, 'TAVILY_API_KEY': config.TAVILY_API_KEY, + 'STAAN_API_KEY': config.STAAN_API_KEY, + 'STAAN_MARKET': config.STAAN_MARKET, + 'STAAN_MAX_SNIPPETS': config.STAAN_MAX_SNIPPETS, 'SEARCHAPI_API_KEY': config.SEARCHAPI_API_KEY, 'SEARCHAPI_ENGINE': config.SEARCHAPI_ENGINE, 'SERPAPI_API_KEY': config.SERPAPI_API_KEY, @@ -817,6 +824,9 @@ class WebConfig(BaseModel): SERPLY_API_KEY: str | None = None DDGS_BACKEND: str | None = None TAVILY_API_KEY: str | None = None + STAAN_API_KEY: str | None = None + STAAN_MARKET: str | None = None + STAAN_MAX_SNIPPETS: int | None = None SEARCHAPI_API_KEY: str | None = None SEARCHAPI_ENGINE: str | None = None SERPAPI_API_KEY: str | None = None @@ -1337,6 +1347,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend config.SERPLY_API_KEY = form_data.web.SERPLY_API_KEY config.DDGS_BACKEND = form_data.web.DDGS_BACKEND config.TAVILY_API_KEY = form_data.web.TAVILY_API_KEY + config.STAAN_API_KEY = form_data.web.STAAN_API_KEY + config.STAAN_MARKET = form_data.web.STAAN_MARKET + config.STAAN_MAX_SNIPPETS = form_data.web.STAAN_MAX_SNIPPETS config.SEARCHAPI_API_KEY = form_data.web.SEARCHAPI_API_KEY config.SEARCHAPI_ENGINE = form_data.web.SEARCHAPI_ENGINE config.SERPAPI_API_KEY = form_data.web.SERPAPI_API_KEY @@ -1490,6 +1503,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend 'SERPHOUSE_DOMAIN': config.SERPHOUSE_DOMAIN, 'SERPLY_API_KEY': config.SERPLY_API_KEY, 'TAVILY_API_KEY': config.TAVILY_API_KEY, + 'STAAN_API_KEY': config.STAAN_API_KEY, + 'STAAN_MARKET': config.STAAN_MARKET, + 'STAAN_MAX_SNIPPETS': config.STAAN_MAX_SNIPPETS, 'SEARCHAPI_API_KEY': config.SEARCHAPI_API_KEY, 'SEARCHAPI_ENGINE': config.SEARCHAPI_ENGINE, 'SERPAPI_API_KEY': config.SERPAPI_API_KEY, @@ -2689,6 +2705,19 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li ) else: raise Exception('No TAVILY_API_KEY found in environment variables') + elif engine == 'staan': + if config.STAAN_API_KEY: + return await asyncio.to_thread( + search_staan, + config.STAAN_API_KEY, + query, + config.WEB_SEARCH_RESULT_COUNT, + config.WEB_SEARCH_DOMAIN_FILTER_LIST, + market=config.STAAN_MARKET, + max_snippets=config.STAAN_MAX_SNIPPETS, + ) + else: + raise Exception('No STAAN_API_KEY found in environment variables') elif engine == 'exa': if config.EXA_API_KEY: return await asyncio.to_thread( diff --git a/backend/open_webui/routers/scim.py b/backend/open_webui/routers/scim.py index a9bb999cb7..a2d093dba8 100644 --- a/backend/open_webui/routers/scim.py +++ b/backend/open_webui/routers/scim.py @@ -753,31 +753,60 @@ async def patch_user( ) update_data = {} + fields = { + 'userName': 'email', + 'displayName': 'name', + 'emails[primary eq true].value': 'email', + 'name.formatted': 'name', + } for operation in patch_data.Operations: op = operation.op.lower() path = operation.path - value = operation.value - if op == 'replace': + if op not in ('add', 'replace', 'remove'): + return scim_error(400, f'Unsupported PATCH operation: {operation.op}') + if op == 'remove': + if not path: + return scim_error(400, 'Remove requires a path', 'noTarget') + if path != 'externalId': + return scim_error(400, f'Removing {path} is not supported', 'mutability') + values = {path: None} + elif path is None: + if not isinstance(operation.value, dict) or not operation.value: + return scim_error(400, 'A pathless operation requires an attribute object', 'invalidValue') + values = operation.value + else: + values = {path: operation.value} + + for path, value in values.items(): if path == 'active': + if not isinstance(value, bool): + return scim_error(400, 'active must be a boolean', 'invalidValue') # Same guard as update_user: never demote an existing admin via SCIM. if user.role != 'admin': update_data['role'] = 'user' if value else 'pending' - elif path == 'userName': - update_data['email'] = value - elif path == 'displayName': - update_data['name'] = value - elif path == 'emails[primary eq true].value': - update_data['email'] = value - elif path == 'name.formatted': - update_data['name'] = value + elif path in fields: + if not isinstance(value, str): + return scim_error(400, f'{path} must be a string', 'invalidValue') + update_data[fields[path]] = value elif path == 'externalId': + if value is not None and not isinstance(value, str): + return scim_error(400, 'externalId must be a string or null', 'invalidValue') provider = get_scim_provider() - await Users.update_user_scim_by_id(user_id, provider, value, db=db) + scim = dict(update_data.get('scim', user.scim) or {}) + scim[provider] = {'external_id': value} + update_data['scim'] = scim + else: + return scim_error(400, f'Unsupported PATCH path: {path}', 'invalidPath') + + # Validate all operations before persisting once, and leave identical writes unchanged. + update_data = {key: value for key, value in update_data.items() if value != getattr(user, key)} + user_updated_fields = ['externalId' if field == 'scim' else field for field in update_data if field != 'role'] # Update user if update_data: + update_data['updated_at'] = int(time.time()) updated_user = await Users.update_user_by_id(user_id, update_data, db=db) if not updated_user: raise HTTPException( @@ -788,7 +817,6 @@ async def patch_user( updated_user = user role_changed = updated_user.role != user.role - user_updated_fields = [field for field in update_data.keys() if field != 'role'] if user_updated_fields: await publish_event( diff --git a/backend/open_webui/routers/tools.py b/backend/open_webui/routers/tools.py index a814eb30f3..12c3771420 100644 --- a/backend/open_webui/routers/tools.py +++ b/backend/open_webui/routers/tools.py @@ -27,7 +27,7 @@ from open_webui.models.tools import ( ) from open_webui.utils.access_control import ( filter_allowed_access_grants, - has_access, + has_connection_access, has_permission, ) from open_webui.utils.auth import get_admin_user, get_verified_user @@ -102,7 +102,7 @@ async def get_tools( ) # OpenAPI Tool Servers - server_access_grants = {} + server_connections = {} for server in await get_tool_servers(request): server_idx = server.get('idx', 0) connections = await Config.get('tool_server.connections', []) @@ -113,10 +113,8 @@ async def get_tools( ) continue connection = connections[server_idx] - server_config = connection.get('config', {}) - server_id = f'server:{server.get("id")}' - server_access_grants[server_id] = server_config.get('access_grants', []) + server_connections[server_id] = connection tools.append( ToolUserResponse( @@ -149,10 +147,8 @@ async def get_tools( user.id, f'mcp:{server_id}' ) - server_config = server.get('config') or {} - tool_id = f'server:mcp:{info.get("id")}' - server_access_grants[tool_id] = server_config.get('access_grants', []) + server_connections[tool_id] = server tools.append( ToolUserResponse( @@ -181,12 +177,10 @@ async def get_tools( tool for tool in tools if not str(tool.id).startswith('server:') - or await has_access( - user.id, - 'read', - server_access_grants.get(str(tool.id), []), + or await has_connection_access( + user, + server_connections[str(tool.id)], user_group_ids, - db=db, ) ] diff --git a/backend/open_webui/socket/main.py b/backend/open_webui/socket/main.py index d4ac18f655..c060db6de9 100644 --- a/backend/open_webui/socket/main.py +++ b/backend/open_webui/socket/main.py @@ -6,6 +6,7 @@ import logging import random import sys import time +from contextlib import suppress from typing import Any import pycrdt as Y @@ -38,17 +39,23 @@ from open_webui.models.folders import Folders from open_webui.models.notes import Notes, NoteUpdateForm from open_webui.models.users import UserNameResponse, Users from open_webui.socket.utils import RedisDict, RedisLock, YdocManager -from open_webui.tasks import create_task, stop_item_tasks +from open_webui.tasks import ( + REDIS_PUBSUB_MAX_RECONNECT_INTERVAL, + REDIS_PUBSUB_RECONNECT_INTERVAL, + create_task, + stop_item_tasks, +) from open_webui.utils.access_control import has_permission from open_webui.utils.auth import get_verified_user_by_token from open_webui.utils.chat_id import is_saved_chat_id -from open_webui.utils.json_codec import SOCKETIO_JSON +from open_webui.utils.json_codec import SOCKETIO_JSON, JSONCodec, dumps_bytes from open_webui.utils.misc import get_output_text from open_webui.utils.redis import ( build_sentinel_url, get_redis_connection, get_sentinels_from_env, ) +from redis.exceptions import RedisError from socketio.packet import Packet logging.basicConfig(stream=sys.stdout, level=GLOBAL_LOG_LEVEL) @@ -190,6 +197,11 @@ YDOC_MANAGER = YdocManager( redis_key_prefix=f'{REDIS_KEY_PREFIX}:ydoc:documents', ) +REDIS_EVENT_CHANNEL = f'{REDIS_KEY_PREFIX}:direct_completion' + +EVENT_QUEUES: dict[str, asyncio.Queue] = {} +EVENT_PUBLISH_LOCK = asyncio.Lock() + def get_session_pool_batches(): """All session pool entries, in bounded batches for the Redis backing.""" @@ -828,27 +840,28 @@ async def yjs_document_update(sid, data): log.warning(f'User {user.get("id")} does not have write access to note {note_id}. Rejecting update.') return - user_id = data.get('user_id', sid) + update = data.get('update') # List of bytes from frontend - update = data['update'] # List of bytes from frontend + if update: + user_id = data.get('user_id', sid) - await YDOC_MANAGER.append_to_updates( - document_id=document_id, - update=update, # Convert list of bytes to bytes - ) + await YDOC_MANAGER.append_to_updates( + document_id=document_id, + update=update, # Convert list of bytes to bytes + ) - # Broadcast update to all other users in the document - await sio.emit( - 'ydoc:document:update', - { - 'document_id': document_id, - 'user_id': user_id, - 'update': update, - 'socket_id': sid, # Add socket_id to match frontend filtering - }, - room=f'doc_{document_id}', - skip_sid=sid, - ) + # Broadcast update to all other users in the document + await sio.emit( + 'ydoc:document:update', + { + 'document_id': document_id, + 'user_id': user_id, + 'update': update, + 'socket_id': sid, # Add socket_id to match frontend filtering + }, + room=f'doc_{document_id}', + skip_sid=sid, + ) async def debounced_save(): await asyncio.sleep(0.5) @@ -948,6 +961,62 @@ async def disconnect(sid, reason=None): # print(f"Unknown session ID {sid} disconnected") +async def redis_event_listener() -> None: + """Route events received over Redis to their local queues.""" + reconnect_interval = REDIS_PUBSUB_RECONNECT_INTERVAL + + while True: + pubsub = None + try: + # RedisCluster can't route a pubsub subscribe until initialize() fills its slot cache. + await REDIS.initialize() + + pubsub = REDIS.pubsub() + await pubsub.subscribe(REDIS_EVENT_CHANNEL) + reconnect_interval = REDIS_PUBSUB_RECONNECT_INTERVAL + + async for message in pubsub.listen(): + if message['type'] != 'message': + continue + event = JSONCodec.loads(message['data']) + queue = EVENT_QUEUES.get(event['channel']) + if queue is not None: + await queue.put(event['data']) + log.warning('Redis event listener stopped. Retrying.') + except asyncio.CancelledError: + raise + except Exception: + log.exception('Redis event listener failed. Retrying.') + finally: + if pubsub: + with suppress(Exception): + await pubsub.aclose() + + await asyncio.sleep(reconnect_interval) + reconnect_interval = min(reconnect_interval * 2, REDIS_PUBSUB_MAX_RECONNECT_INTERVAL) + + +@sio.on('*') +async def socket_event_handler(event: Any, sid: str, *args: Any) -> None: + """Route user-owned stream events to a local queue or another worker.""" + if not isinstance(event, str) or event.count(':') != 2 or not args: + return + + user = await get_socket_session_user(sid) + if not user or user.get('id') != event.split(':', 1)[0]: + return + + queue = EVENT_QUEUES.get(event) + if queue is not None: + await queue.put(args[0]) + elif WEBSOCKET_MANAGER == 'redis': + try: + async with EVENT_PUBLISH_LOCK: + await REDIS.publish(REDIS_EVENT_CHANNEL, dumps_bytes({'channel': event, 'data': args[0]})) + except RedisError as e: + log.debug('Failed to relay socket event %s: %s', event, e) + + async def _make_channel_emitter(request_info): """Event emitter that routes pipeline output to a channel message. diff --git a/backend/open_webui/tasks.py b/backend/open_webui/tasks.py index 322cd32e60..05f622ef06 100644 --- a/backend/open_webui/tasks.py +++ b/backend/open_webui/tasks.py @@ -6,7 +6,7 @@ from uuid import uuid4 from redis.asyncio import Redis -from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL +from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL, REDIS_TASK_TTL from open_webui.utils.json_codec import JSONCodec, dumps_bytes log = logging.getLogger(__name__) @@ -66,13 +66,28 @@ async def redis_task_command_listener(app): reconnect_interval = min(reconnect_interval * 2, REDIS_PUBSUB_MAX_RECONNECT_INTERVAL) +async def redis_task_heartbeat(app): + redis: Redis = app.state.redis + while True: + await asyncio.sleep(REDIS_TASK_TTL / 4) + try: + pipe = redis.pipeline(transaction=False) + for task_id in list(tasks): + # EXPIRE cannot recreate a task already removed by cleanup. + pipe.expire(f'{REDIS_TASKS_KEY}:{task_id}', REDIS_TASK_TTL) + await pipe.execute() + except Exception: + log.exception('Redis task heartbeat failed') + + ### ------------------------------ ### REDIS-ENABLED HANDLERS ### ------------------------------ async def redis_save_task(redis: Redis, task_id: str, item_id: str | None): - pipe = redis.pipeline() + pipe = redis.pipeline(transaction=False) + pipe.set(f'{REDIS_TASKS_KEY}:{task_id}', '1', ex=REDIS_TASK_TTL or None) pipe.hset(REDIS_TASKS_KEY, task_id, item_id or '') if item_id: pipe.sadd(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) @@ -80,25 +95,36 @@ async def redis_save_task(redis: Redis, task_id: str, item_id: str | None): async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None): - pipe = redis.pipeline() + pipe = redis.pipeline(transaction=False) + pipe.delete(f'{REDIS_TASKS_KEY}:{task_id}') pipe.hdel(REDIS_TASKS_KEY, task_id) pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id) if item_id: pipe.srem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) - await pipe.execute() - # Remove the set key entirely if no tasks remain for this item - if await redis.scard(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') == 0: - await redis.delete(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') - else: - await pipe.execute() + await pipe.execute() -async def redis_list_tasks(redis: Redis) -> list[str]: - return list(await redis.hkeys(REDIS_TASKS_KEY)) +async def redis_list_tasks(redis: Redis, item_id: str | None = None) -> list[str]: + task_ids = list( + await redis.smembers(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') + if item_id is not None + else await redis.hkeys(REDIS_TASKS_KEY) + ) + if not task_ids or REDIS_TASK_TTL == 0: + return task_ids + pipe = redis.pipeline(transaction=False) + for task_id in task_ids: + pipe.exists(f'{REDIS_TASKS_KEY}:{task_id}') -async def redis_list_item_tasks(redis: Redis, item_id: str) -> list[str]: - return list(await redis.smembers(f'{REDIS_ITEM_TASKS_KEY}:{item_id}')) + active = [] + for task_id, exists in zip(task_ids, await pipe.execute()): + if exists: + active.append(task_id) + else: + task_item_id = item_id if item_id is not None else await redis.hget(REDIS_TASKS_KEY, task_id) + await redis_cleanup_task(redis, task_id, task_item_id or None) + return active async def redis_send_command(redis: Redis, command: dict): @@ -166,7 +192,7 @@ async def list_task_ids_by_item_id(redis, id): List all tasks associated with a specific ID. """ if redis: - return await redis_list_item_tasks(redis, id) + return await redis_list_tasks(redis, id) return list(item_tasks.get(id, [])) diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index f3277e3b8f..d80884d1cd 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -29,7 +29,7 @@ from open_webui.models.memories import Memories from open_webui.models.messages import Message, Messages from open_webui.models.notes import Notes from open_webui.models.users import UserModel -from open_webui.retrieval.utils import get_content_from_url +from open_webui.retrieval.utils import filter_source_metadata, get_content_from_url from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.routers.images import ( CreateImageForm, @@ -526,6 +526,7 @@ async def ask_user( Use this when the next step depends on user intent, preference, or a tradeoff that cannot be inferred safely. :param questions: 1-3 question objects, each with id, header, question, and 2-3 options. Each option needs label and description. + List the option you recommend first; the UI labels the first option Recommended. :param allow_other: Whether users may enter a free-form answer instead of choosing one of the options :param timeout_ms: How long the browser should keep the prompt open before cancelling it :return: JSON with status and answers keyed by question id @@ -2461,6 +2462,7 @@ async def grep_chat_files( """ Search exact text across files attached to the current chat. Pass file_id from the attached_files block to search one file. + Auto-detected regex uses RE2 syntax; no lookarounds/backreferences, and shorthand classes are ASCII-only. :param pattern: The text pattern to search for :param file_id: Optional attached file ID to search within a single file @@ -2609,6 +2611,7 @@ async def query_chat_files( for idx, doc in enumerate(documents): metadata = metadatas[idx] if idx < len(metadatas) and isinstance(metadatas[idx], dict) else {} chunk = { + **filter_source_metadata(metadata), 'content': doc, 'source': metadata.get('source', metadata.get('name', source_info.get('name', 'Unknown'))), 'file_id': metadata.get('file_id', source_info.get('id', '')), @@ -2636,6 +2639,7 @@ async def grep_knowledge_files( Search for exact text across knowledge files. Returns matching lines with line numbers. Unlike query_knowledge_files (semantic/vector search), this performs exact string matching. Automatically detects regex patterns (e.g. "error|warn", "version \\d+"). + Regex uses RE2 syntax; no lookarounds/backreferences, and shorthand character classes are ASCII-only. Helpful for literal strings, identifiers, error messages, or regex-style searches. :param pattern: The text pattern to search for (regex auto-detected) @@ -3321,6 +3325,7 @@ async def query_knowledge_files( for idx, doc in enumerate(documents): chunk_info = { + **filter_source_metadata(metadatas[idx]), 'content': doc, 'source': metadatas[idx].get('source', metadatas[idx].get('name', 'Unknown')), 'file_id': metadatas[idx].get('file_id', ''), @@ -3344,6 +3349,7 @@ async def query_knowledge_files( for idx, doc in enumerate(documents): metadata = metadatas[idx] if idx < len(metadatas) else {} chunk_info = { + **filter_source_metadata(metadata), 'content': doc, 'source': metadata.get('source', metadata.get('name', knowledge.name)), 'file_id': metadata.get('file_id', f'external-{knowledge.id}'), @@ -3762,7 +3768,7 @@ async def create_automation( # Validate the RRULE try: - validate_rrule(rrule, tz=user.timezone) + await validate_rrule(rrule, tz=user.timezone) except ValueError as e: return JSONCodec.dumps({'error': f'Invalid schedule: {e}'}) @@ -3788,7 +3794,7 @@ async def create_automation( is_active=True, ) - automation = await Automations.insert(user_id, form, next_run_ns(rrule, tz=tz)) + automation = await Automations.insert(user_id, form, await next_run_ns(rrule, tz=tz)) return JSONCodec.dumps( { @@ -3799,7 +3805,7 @@ async def create_automation( 'model_id': model_id, 'target': automation.data.get('target'), 'is_active': automation.is_active, - 'next_runs': next_n_runs_ns(rrule, tz=tz), + 'next_runs': await next_n_runs_ns(rrule, tz=tz), }, ensure_ascii=False, ) @@ -3870,7 +3876,7 @@ async def update_automation( # Validate RRULE if changed if rrule is not None: try: - validate_rrule(new_rrule, tz=user.timezone) + await validate_rrule(new_rrule, tz=user.timezone) except ValueError as e: return JSONCodec.dumps({'error': f'Invalid schedule: {e}'}) @@ -3892,7 +3898,7 @@ async def update_automation( is_active=automation.is_active, ) - updated = await Automations.update_by_id(automation_id, form, next_run_ns(new_rrule, tz=tz)) + updated = await Automations.update_by_id(automation_id, form, await next_run_ns(new_rrule, tz=tz)) return JSONCodec.dumps( { @@ -3903,7 +3909,7 @@ async def update_automation( 'model_id': new_model_id, 'target': updated.data.get('target'), 'is_active': updated.is_active, - 'next_runs': next_n_runs_ns(new_rrule, tz=tz), + 'next_runs': await next_n_runs_ns(new_rrule, tz=tz), }, ensure_ascii=False, ) @@ -3971,7 +3977,7 @@ async def list_automations( 'rrule': rrule, 'is_active': item.is_active, 'last_run_at': item.last_run_at, - 'next_runs': next_n_runs_ns(rrule, tz=user.timezone if user else None), + 'next_runs': await next_n_runs_ns(rrule, tz=user.timezone if user else None), } ) @@ -4018,7 +4024,7 @@ async def toggle_automation( rrule = automation.data.get('rrule', '') toggled = await Automations.toggle( automation_id, - next_run_ns(rrule, tz=user.timezone if user else None), + await next_run_ns(rrule, tz=user.timezone if user else None), ) return JSONCodec.dumps( diff --git a/backend/open_webui/tools/knowledge_fs.py b/backend/open_webui/tools/knowledge_fs.py index 53440b7bbf..355ae3cc58 100644 --- a/backend/open_webui/tools/knowledge_fs.py +++ b/backend/open_webui/tools/knowledge_fs.py @@ -17,7 +17,7 @@ from collections.abc import Callable from contextlib import contextmanager from typing import Optional -import regex +import re2 from fastapi import Request from open_webui.env import ( @@ -31,12 +31,9 @@ log = logging.getLogger(__name__) DEFAULT_HEAD_LINES = 10 DEFAULT_TAIL_LINES = 10 -# Matching time allowed per tool call. Backtracking cost is exponential in the length of the -# matched text, so capping the pattern or the line does not bound it. +# Total matching time allowed per tool call, checked between RE2's linear-time searches. MATCH_BUDGET_SECONDS = 2.0 -MAX_REGEX_QUANTIFIER_COUNT = 2_000 -MAX_REGEX_QUANTIFIER_EXPANSION = 100_000 -_COUNTED_QUANTIFIER_RE = re.compile(r'(? str: return pattern.replace(r'\|', '|').replace(r'\|', '|') -def validate_regex_quantifiers(pattern: str) -> str | None: - """Reject counted quantifiers that make regex compilation expand too much.""" - quantifier_expansion = 1 - for quantifier in _COUNTED_QUANTIFIER_RE.finditer(pattern): - count_text = quantifier.group(1) - count = int(count_text) if len(count_text) <= 6 else MAX_REGEX_QUANTIFIER_COUNT + 1 - if count > MAX_REGEX_QUANTIFIER_COUNT: - return f'Regex quantifier counts over {MAX_REGEX_QUANTIFIER_COUNT:g} are not supported' - - # ponytail: conservative expansion catches nested quantifier bombs without mirroring regex syntax. - quantifier_expansion *= max(count, 1) - if quantifier_expansion > MAX_REGEX_QUANTIFIER_EXPANSION: - return 'Regex quantifiers expand too much, lower the counts' - - return None - - def build_matcher(pattern: str, case_insensitive: bool = False, use_regex: bool = False) -> tuple: """Build a matcher function. Returns (match_fn, error_str_or_None).""" + if len(pattern) > MAX_SEARCH_PATTERN_LENGTH: + return None, f'Search patterns over {MAX_SEARCH_PATTERN_LENGTH} characters are not supported' + if not use_regex and is_regex_pattern(pattern): use_regex = True if use_regex: normalized = normalize_regex(pattern) - quantifier_error = validate_regex_quantifiers(normalized) - if quantifier_error: - return None, quantifier_error try: - re_flags = regex.IGNORECASE if case_insensitive else 0 - compiled = regex.compile(normalized, re_flags) - except regex.error as e: - return None, f'Invalid regex: {e}' + options = re2.Options() + options.case_sensitive = not case_insensitive + options.max_mem = 1 << 20 # Bound compiled programs and the engine's matching cache to 1 MiB. + options.log_errors = False + compiled = re2.compile(normalized, options=options) + except re2.error as e: + return None, f'Invalid or unsupported regex (RE2 syntax): {e}' budget = _active_budget.get() or MatchBudget() def matches(line: str) -> bool: started = time.monotonic() try: - # A negative timeout disables it, so an exhausted budget must not reach search(). if budget.remaining <= 0: raise TimeoutError - return bool(compiled.search(line, timeout=budget.remaining)) + matched = bool(compiled.search(line)) + if time.monotonic() - started >= budget.remaining: + raise TimeoutError + return matched except TimeoutError: raise MatchBudgetExceeded(f'Search exceeded {MATCH_BUDGET_SECONDS:g}s, narrow the pattern') from None finally: @@ -1188,6 +1173,7 @@ async def kb_exec( Pipes: grep "auth" | head -5 Files: reference by path (docs/api/auth.md), filename, or file ID + Regex: RE2 syntax; no lookarounds/backreferences. Shorthand character classes are ASCII-only. :param command: A filesystem command string :return: Command output as text diff --git a/backend/open_webui/utils/auth.py b/backend/open_webui/utils/auth.py index 612a56ef92..7d6c52fe26 100644 --- a/backend/open_webui/utils/auth.py +++ b/backend/open_webui/utils/auth.py @@ -328,6 +328,10 @@ async def invalidate_token(request, token): ttl = exp - int(datetime.now(UTC).timestamp()) # Calculate time-to-live for the token if ttl > 0: + # Revoked tokens must not be able to disconnect newer sessions. + if not await is_valid_token(decoded, request.app.state.redis): + return + # Store the revoked token in Redis with an expiration time await request.app.state.redis.set( f'{REDIS_KEY_PREFIX}:auth:token:{jti}:revoked', @@ -335,6 +339,12 @@ async def invalidate_token(request, token): ex=ttl, ) + user_id = decoded.get('id') + if user_id: + from open_webui.socket.main import disconnect_user_sessions + + await disconnect_user_sessions(user_id) + async def revoke_user_tokens(request, user_id: str): """Reject every token already issued to a user. Requires Redis.""" @@ -356,6 +366,10 @@ async def revoke_user_tokens(request, user_id: str): ex=int(expires_delta.total_seconds()) if expires_delta else None, ) + from open_webui.socket.main import disconnect_user_sessions + + await disconnect_user_sessions(user_id) + def extract_token_from_auth_header(auth_header: str): return auth_header[len('Bearer ') :] diff --git a/backend/open_webui/utils/automations.py b/backend/open_webui/utils/automations.py index 7df4c399aa..54887467c0 100644 --- a/backend/open_webui/utils/automations.py +++ b/backend/open_webui/utils/automations.py @@ -20,12 +20,10 @@ import logging import os import random import time -from datetime import datetime, timedelta +from datetime import timedelta from typing import Optional from uuid import uuid4 -from zoneinfo import ZoneInfo -from dateutil.rrule import HOURLY, MINUTELY, SECONDLY, rruleset, rrulestr from fastapi import Request from fastapi.security import HTTPAuthorizationCredentials from open_webui.constants import ERROR_MESSAGES @@ -39,6 +37,13 @@ from open_webui.models.messages import MessageForm from open_webui.models.users import Users from open_webui.utils.auth import create_token from open_webui.utils.misc import parse_duration +from open_webui.utils.recurrence import ( + _resolve_tz, + next_n_runs_ns, + next_run_ns, + rrule_interval_seconds, + validate_rrule, +) from open_webui.utils.task import prompt_template from open_webui.utils.terminals import get_terminal_server_url from starlette.datastructures import Headers @@ -50,153 +55,6 @@ TIMER_POLL_INTERVAL = int(os.getenv('TIMER_POLL_INTERVAL', '1')) CALENDAR_ALERT_LOOKAHEAD_MINUTES = int(os.getenv('CALENDAR_ALERT_LOOKAHEAD_MINUTES', '10')) -#################### -# RRULE Helpers -#################### - - -def _resolve_tz(tz: str = None) -> Optional[ZoneInfo]: - """Safely resolve a timezone string to ZoneInfo. - - Returns None (→ server-local fallback) when *tz* is empty, None, - or an unrecognised IANA key. Logs a warning on bad keys so - misconfiguration is visible in the server logs. - """ - if not tz: - return None - try: - return ZoneInfo(tz) - except (KeyError, Exception): - log.warning('Unknown timezone %r — falling back to server time', tz) - return None - - -def _parse_rule(s: str, now: Optional[datetime] = None): - """Parse RRULE with clock-aligned DTSTART for sub-daily frequencies. - - SECONDLY/MINUTELY/HOURLY rules use a fixed epoch DTSTART (2000-01-01 00:00) - so intervals snap to clock boundaries (e.g. every 5min = :00, :05, :10). - """ - upper = s.upper() - if 'EXRULE' in upper: - raise ValueError('EXRULE is not supported in recurrence rules') - - parsed = rrulestr(s, ignoretz=True) - rules = parsed._rrule if isinstance(parsed, rruleset) else [parsed] - if len(rules) > 1: - raise ValueError('only one RRULE is supported per recurrence rule') - - rule = rules[0] - start = rule._dtstart.replace(tzinfo=None) - anchor = now or datetime.now() - lines = s.splitlines() - stripped = '\n'.join(line for line in lines if not line.upper().startswith('DTSTART')) or s - has_dtstart = any(line.upper().startswith('DTSTART') for line in lines) - step = { - SECONDLY: timedelta(seconds=rule._interval), - MINUTELY: timedelta(minutes=rule._interval), - HOURLY: timedelta(hours=rule._interval), - }.get(rule._freq) - - if step is None: - if not rule._dtstart.tzinfo: - return parsed - return rrulestr(stripped, dtstart=start, ignoretz=True) - - if rule._interval < 1: - raise ValueError('RRULE INTERVAL must be a positive integer') - dtstart = None - if has_dtstart: - emitted = ((anchor - start) // step) if anchor > start else 0 - emitted *= len(rule._byminute or (0,)) * len(rule._bysecond or (0,)) - if emitted <= 100_000: - if rule._dtstart.tzinfo: - dtstart = start - else: - return parsed - if not has_dtstart or dtstart is None: - epoch = datetime(2000, 1, 1) - dtstart = epoch + ((anchor - epoch) // step) * step - - return rrulestr(stripped, dtstart=dtstart, ignoretz=True) - - -def validate_rrule(s: str, tz: str = None) -> None: - """Raise ValueError if the RRULE is malformed or exhausted. - - When *tz* is provided the "now" reference uses the user's local - clock so that near-future schedules are not incorrectly rejected - on servers whose system clock is ahead (e.g. UTC vs US timezones). - """ - upper = s.upper() - if 'COUNT=' in upper and 'DTSTART' not in upper: - raise ValueError(ERROR_MESSAGES.AUTOMATION_COUNT_REQUIRES_DTSTART) - zi = _resolve_tz(tz) - now = datetime.now(zi).replace(tzinfo=None) if zi else datetime.now() - try: - rule = _parse_rule(s, now) - except Exception as e: - raise ValueError(ERROR_MESSAGES.AUTOMATION_INVALID_RRULE(e)) - if rule.after(now) is None: - raise ValueError(ERROR_MESSAGES.AUTOMATION_NO_FUTURE_RUNS) - - -def next_run_ns(s: str, tz: str = None) -> Optional[int]: - """Next occurrence as epoch nanoseconds, respecting user timezone.""" - zi = _resolve_tz(tz) - now = datetime.now(zi) if zi else datetime.now() - now_naive = now.replace(tzinfo=None) - dt = _parse_rule(s, now_naive).after(now_naive) - if dt is None: - return None - if zi: - dt = dt.replace(tzinfo=zi) - return int(dt.timestamp() * 1_000_000_000) - - -def next_n_runs_ns(s: str, n: int = 5, tz: str = None) -> list[int]: - """Compute next N occurrences for UI preview. - - Uses the user's timezone for the starting "now" so that the - preview matches the user's local clock (same as next_run_ns). - """ - zi = _resolve_tz(tz) - result = [] - now = datetime.now(zi).replace(tzinfo=None) if zi else datetime.now() - rule = _parse_rule(s, now) - dt = now - for _ in range(n): - dt = rule.after(dt) - if not dt: - break - if zi: - dt_tz = dt.replace(tzinfo=zi) - result.append(int(dt_tz.timestamp() * 1_000_000_000)) - else: - result.append(int(dt.timestamp() * 1_000_000_000)) - return result - - -def rrule_interval_seconds(s: str) -> Optional[int]: - """Approximate interval between recurrences in seconds. - - Returns None for one-shot (COUNT=1) schedules or rules - with fewer than two future occurrences. - """ - if 'COUNT=1' in s: - return None - s = '\n'.join(line for line in s.splitlines() if not line.upper().startswith('DTSTART')) or s - now = datetime.now() - rule = _parse_rule(s, now) - first = rule.after(now) - if first is None: - return None - second = rule.after(first) - if second is None: - return None - return int((second - first).total_seconds()) - - ############################ # Worker Loop ############################ diff --git a/backend/open_webui/utils/calendar.py b/backend/open_webui/utils/calendar.py index 1ce590203a..0406db38f6 100644 --- a/backend/open_webui/utils/calendar.py +++ b/backend/open_webui/utils/calendar.py @@ -44,7 +44,7 @@ def expand_recurring_event( original_start_ns = event_dict['start_at'] original_start = to_local_datetime(original_start_ns) - rule_str = '\n'.join(line for line in rrule_str.splitlines() if not line.upper().startswith('DTSTART')) or rrule_str + rule_str = '\n'.join(part for part in rrule_str.split() if not part.upper().startswith('DTSTART')) or rrule_str try: # Anchor to the event's real start so day-of-week / day-of-month are correct diff --git a/backend/open_webui/utils/chat.py b/backend/open_webui/utils/chat.py index 0b64a52165..5d5f6f60c4 100644 --- a/backend/open_webui/utils/chat.py +++ b/backend/open_webui/utils/chat.py @@ -23,9 +23,9 @@ from open_webui.routers.pipelines import ( process_pipeline_outlet_filter, ) from open_webui.socket.main import ( + EVENT_QUEUES, get_event_call, get_event_emitter, - sio, ) from open_webui.utils.filter import ( get_filter_functions, @@ -71,19 +71,8 @@ async def generate_direct_chat_completion( logging.info('WebSocket channel: %s', channel) if form_data.get('stream'): - q = asyncio.Queue() - - async def message_listener(sid, data): - """ - Handle received socket messages and push them into the queue. - """ - await q.put(data) - - def remove_message_listener(): - sio.handlers['/'].pop(channel, None) - - # Register the listener - sio.on(channel, message_listener) + queue = asyncio.Queue() + EVENT_QUEUES[channel] = queue # Start processing chat completion in background try: @@ -103,16 +92,15 @@ async def generate_direct_chat_completion( status = res.get('status', False) except BaseException: - remove_message_listener() + EVENT_QUEUES.pop(channel, None) raise if status: # Define a generator to stream responses async def event_generator(): - nonlocal q try: while True: - data = await q.get() # Wait for new messages + data = await queue.get() # Wait for new messages if isinstance(data, dict): if 'done' in data and data['done']: break # Stop streaming when 'done' is received @@ -127,16 +115,16 @@ async def generate_direct_chat_completion( log.debug('Error in event generator: %s', e) pass finally: - remove_message_listener() + EVENT_QUEUES.pop(channel, None) # Define a background task to run the event generator async def background(): - remove_message_listener() + EVENT_QUEUES.pop(channel, None) # Return the streaming response return StreamingResponse(event_generator(), media_type='text/event-stream', background=background) else: - remove_message_listener() + EVENT_QUEUES.pop(channel, None) raise Exception(str(res)) else: res = await event_caller( diff --git a/backend/open_webui/utils/files.py b/backend/open_webui/utils/files.py index f982da1675..e9c926e5f8 100644 --- a/backend/open_webui/utils/files.py +++ b/backend/open_webui/utils/files.py @@ -75,9 +75,16 @@ async def get_image_base64_from_url(url: str, user=None) -> Optional[str]: # rebinding DNS answer that passed validate_url cannot reach an internal address. async with get_ssrf_safe_session() as session: async with session.get( - url, ssl=AIOHTTP_CLIENT_SESSION_SSL, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS + url, + ssl=AIOHTTP_CLIENT_SESSION_SSL, + allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS, + headers={'Accept-Encoding': 'identity'}, ) as response: response.raise_for_status() + # Accept-Encoding is only a request; the sender can still compress and pick our decompressed size. + encodings = response.headers.getall('Content-Encoding', ()) + if any(encoding.lower() not in ('', 'identity') for encoding in encodings): + return None image_data = bytearray() total = 0 async for chunk in response.content.iter_chunked(64 * 1024): diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 5b59fbd7ad..ad784658c8 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -2,6 +2,7 @@ import ast import asyncio import base64 import copy +import html import inspect import json import logging @@ -52,7 +53,7 @@ from open_webui.models.models import Models from open_webui.models.notes import Notes from open_webui.models.oauth_sessions import OAuthSessions from open_webui.models.users import UserModel, Users -from open_webui.retrieval.utils import get_sources_from_items +from open_webui.retrieval.utils import filter_source_metadata, get_sources_from_items from open_webui.routers.images import ( CreateImageForm, EditImageForm, @@ -117,6 +118,7 @@ from open_webui.utils.misc import ( get_response_error_detail, get_reasoning_details, get_system_message, + is_raster_image_content_type, is_string_allowed, merge_system_messages, prepend_to_first_user_message_content, @@ -951,11 +953,17 @@ def get_source_context(sources: list, source_ids: dict = None, include_content: src_type = source.get('source', {}).get('type') src_rid = source.get('source', {}).get('id') body = doc if include_content else '' + extra_attrs = '' + for key, value in filter_source_metadata(meta).items(): + if key in ('id', 'name', 'resource-type', 'resource-id'): + continue + extra_attrs += f' {key}="{html.escape(str(value))}"' context_string += ( f'{body}\n' ) return context_string @@ -1258,6 +1266,14 @@ async def process_tool_result( return tool_result, tool_result_files, tool_result_embeds +def parse_terminal_tool_result(tool_result: Any) -> dict: + try: + result = JSONCodec.loads(tool_result) + except (JSONCodec.JSONDecodeError, TypeError): + return {} + return result if isinstance(result, dict) else {} + + async def terminal_event_handler( tool_function_name: str, tool_function_params: dict, @@ -1276,17 +1292,13 @@ async def terminal_event_handler( if tool_function_name == 'display_file': if tool_function_params.get('inline') is True: return - path = tool_function_params.get('path', '') + result = parse_terminal_tool_result(tool_result) + # Open Terminal resolves the argument against the session cwd, so prefer its resolved path + path = result.get('path') or tool_function_params.get('path', '') if not path: return # Only emit if the file actually exists - parsed = tool_result - if isinstance(parsed, str): - try: - parsed = JSONCodec.loads(parsed) - except (JSONCodec.JSONDecodeError, TypeError): - pass - if isinstance(parsed, dict) and parsed.get('exists') is False: + if result.get('exists') is False: return page = tool_function_params.get('page') @@ -1300,7 +1312,8 @@ async def terminal_event_handler( } ) elif tool_function_name in ('write_file', 'replace_file_content'): - path = tool_function_params.get('path', '') + result = parse_terminal_tool_result(tool_result) + path = result.get('path') or tool_function_params.get('path', '') if not path: return await event_emitter( @@ -1725,7 +1738,7 @@ def get_images_from_messages(message_list): for file in message.get('files', []): if file.get('type') == 'image': message_images.append(file.get('url')) - elif file.get('content_type', '').startswith('image/'): + elif is_raster_image_content_type(file.get('content_type')): message_images.append(file.get('url')) if message_images: @@ -2221,7 +2234,11 @@ async def load_messages_from_db(chat_id: str, message_id: str) -> Optional[list[ if not db_messages: return None - return [{k: v for k, v in msg.items() if k in MESSAGE_REPLAY_KEYS} for msg in db_messages] + return [ + {k: v for k, v in msg.items() if k in MESSAGE_REPLAY_KEYS} + for msg in db_messages + if not (msg.get('role') == 'assistant' and msg.get('error') and not msg.get('content') and not msg.get('output')) + ] def get_reasoning_format(model: dict) -> str | None: @@ -2273,6 +2290,8 @@ def process_messages_with_output( if output_messages: processed.extend(output_messages) continue + if not message.get('content'): + continue clean_message = dict(message) for key in ('id', 'files', 'output', 'model', 'contextSummary', 'context_summary', 'usage'): @@ -2440,7 +2459,7 @@ async def process_chat_payload(request, form_data, user, metadata, model): image_files = [ f for f in message.get('files', []) - if f.get('type') == 'image' or (f.get('content_type') or '').startswith('image/') + if f.get('type') == 'image' or is_raster_image_content_type(f.get('content_type')) ] if message.get('role') == 'user' and image_files: text_content = message.get('content', '') @@ -2664,7 +2683,12 @@ async def process_chat_payload(request, form_data, user, metadata, model): form_data['messages'], ) - if 'memory' in features and features['memory'] and await Config.get('memories.system_context.enable'): + if ( + 'memory' in features + and features['memory'] + and await Config.get('memories.enable') + and await Config.get('memories.system_context.enable') + ): # features is client-supplied; re-check the permission the native FC path enforces. if getattr(user, 'role', None) == 'admin' or await has_permission( getattr(user, 'id', ''), @@ -3036,6 +3060,8 @@ async def process_chat_payload(request, form_data, user, metadata, model): if direct_tool_servers: for tool_server in direct_tool_servers: + if tool_server.get('is_terminal') is True and not terminal_capability: + continue system_prompt = tool_server.pop('system_prompt', None) if system_prompt: form_data['messages'] = add_or_update_system_message( @@ -3053,6 +3079,13 @@ async def process_chat_payload(request, form_data, user, metadata, model): 'server': tool_server, } + if terminal_id and terminal_capability: + from open_webui.utils.terminals import add_terminal_agents_md, get_terminal_agents_md + + agents_md = await get_terminal_agents_md(request, user, metadata, extra_params) + if agents_md: + form_data['messages'] = add_terminal_agents_md(form_data['messages'], agents_md) + if mcp_clients: metadata['mcp_clients'] = mcp_clients @@ -3100,6 +3133,50 @@ async def process_chat_payload(request, form_data, user, metadata, model): if name not in tools_dict: tools_dict[name] = tool_dict + # Only advertise user-shell tools when the originating browser has a connected shell. + shell_tools = { + name: tool + for name, tool in tools_dict.items() + if name in {'read_user_terminal', 'send_user_terminal_input'} + and (tool.get('type') == 'terminal' or tool.get('server', {}).get('is_terminal') is True) + } + selected = { + name + for name, tool in shell_tools.items() + if terminal_id + and ( + tool.get('tool_id') == f'terminal:{terminal_id}' + or (tool.get('direct') and tool.get('server', {}).get('url') == terminal_id) + ) + } + connected = False + if ( + selected + and event_caller + and metadata.get('session_id') + and metadata.get('chat_id') + and not metadata.get('automation_id') + and not metadata.get('internal') + ): + try: + state = await asyncio.wait_for( + event_caller( + { + 'type': 'request:terminal:state', + 'data': {'terminal_id': terminal_id, 'session_id': metadata['session_id']}, + } + ), + timeout=2, + ) + connected = isinstance(state, dict) and state.get('connected') is True + except Exception: + # Old/disconnected browsers cannot confirm availability; other tools still work. + pass + + for name in shell_tools: + if not connected or name not in selected: + tools_dict.pop(name) + if tools_dict: # Always store resolved tools in metadata so downstream consumers # (e.g. pipe functions) can access all tools including MCP and builtins. @@ -4162,13 +4239,14 @@ async def non_streaming_chat_response_handler(response, ctx): metadata['message_id'], { 'error': {'content': error}, + 'done': True, }, ) if isinstance(error, str) or isinstance(error, dict): await event_emitter( { 'type': 'chat:message:error', - 'data': {'error': {'content': error}}, + 'data': {'error': {'content': error}, 'done': True}, } ) diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index 247ab72441..ad959d3bc7 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -1223,6 +1223,12 @@ def strict_match_mime_type(supported: list[str] | str, header: str) -> str | Non return None +def is_raster_image_content_type(content_type: str | None) -> bool: + """Return True if the content type is an image that decodes as a bitmap; SVG is XML.""" + base_content_type = (content_type or '').split(';')[0].strip().lower() + return base_content_type.startswith('image/') and base_content_type != 'image/svg+xml' + + def extract_urls(text: str) -> list[str]: # Regex pattern to match URLs url_pattern = re.compile(r'(https?://[^\s]+)', re.IGNORECASE) # Matches http and https URLs diff --git a/backend/open_webui/utils/oauth.py b/backend/open_webui/utils/oauth.py index ed24b32e87..10dedb1e6c 100644 --- a/backend/open_webui/utils/oauth.py +++ b/backend/open_webui/utils/oauth.py @@ -188,6 +188,16 @@ def _default_value(value): return getattr(value, 'value', value) +def _get_roles_claim(claims: dict, claim: str) -> list | str | int | None: + """Read nested or flat claims, preserving explicit empty values and zero.""" + value = claims + for key in claim.split('.'): + value = value.get(key) if isinstance(value, dict) else None + if not isinstance(value, (list, str, int)): + value = claims.get(claim) + return value if isinstance(value, (list, str, int)) else None + + async def get_oauth_runtime_config() -> SimpleNamespace: keys = [key for key, _default in OAUTH_RUNTIME_CONFIG.values()] stored = await Config.get_many(*keys) @@ -1505,7 +1515,7 @@ class OAuthManager: log.error(f'Exception during token refresh for provider {provider}: {e}') return None - async def get_user_role(self, user, user_data): + async def get_user_role(self, user, user_data, *, access_token: str | None = None): auth_config = await get_oauth_runtime_config() user_count = await Users.get_num_users() if user and user_count == 1: @@ -1529,18 +1539,15 @@ class OAuthManager: # Keep existing users at their current role unless the provider sent roles. role = user.role if user else auth_config.DEFAULT_USER_ROLE - # Next block extracts the roles from the user data, accepting nested claims of any depth - if oauth_claim and oauth_allowed_roles: - claim_data = user_data - nested_claims = oauth_claim.split('.') - for nested_claim in nested_claims: - claim_data = claim_data.get(nested_claim, {}) - - # Try flat claim structure as alternative - if not claim_data: - claim_data = user_data.get(oauth_claim, {}) - - oauth_roles = [] + if oauth_claim: + claim_data = _get_roles_claim(user_data, oauth_claim) + if claim_data is None and access_token is not None: + # The exchange endpoint has already validated this token with the provider's userinfo endpoint. + try: + token_claims = jwt.decode(access_token, options={'verify_signature': False}) + claim_data = _get_roles_claim(token_claims, oauth_claim) + except jwt.PyJWTError as e: + log.debug('Token exchange: cannot decode token claims: %s', e) if isinstance(claim_data, list): oauth_roles = claim_data @@ -1553,6 +1560,10 @@ class OAuthManager: elif isinstance(claim_data, int): oauth_roles = [str(claim_data)] + if access_token is not None and not oauth_roles and oauth_allowed_roles and '*' not in oauth_allowed_roles: + log.warning('Token exchange denied: no readable roles claim in userinfo or the token') + raise HTTPException(status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.ACCESS_PROHIBITED) + log.debug('Oauth Roles claim: %s', oauth_claim) log.debug('User roles from oauth: %s', oauth_roles) log.debug('Accepted user roles: %s', oauth_allowed_roles) @@ -1599,9 +1610,10 @@ class OAuthManager: user_data, provider, *, + access_token: str | None = None, db=None, ): - determined_role = await self.get_user_role(user, user_data) + determined_role = await self.get_user_role(user, user_data, access_token=access_token) if user.role == determined_role: return user diff --git a/backend/open_webui/utils/recurrence.py b/backend/open_webui/utils/recurrence.py new file mode 100644 index 0000000000..d5c2b6b1f2 --- /dev/null +++ b/backend/open_webui/utils/recurrence.py @@ -0,0 +1,171 @@ +"""Recurrence calculations isolated from application/DB imports for worker processes.""" + +import logging +from datetime import datetime, timedelta +from typing import Optional +from zoneinfo import ZoneInfo + +from anyio import fail_after, to_process +from dateutil.rrule import HOURLY, MINUTELY, SECONDLY, rruleset, rrulestr +from open_webui.constants import ERROR_MESSAGES + +log = logging.getLogger(__name__) +RRULE_TIMEOUT_SECONDS = 2 + + +class RecurrenceEvaluationTimeout(ValueError): + """The evaluation budget expired; the schedule may still have occurrences.""" + + +def _resolve_tz(tz: str = None) -> Optional[ZoneInfo]: + """Safely resolve a timezone string to ZoneInfo. + + Returns None (→ server-local fallback) when *tz* is empty, None, + or an unrecognised IANA key. Logs a warning on bad keys so + misconfiguration is visible in the server logs. + """ + if not tz: + return None + try: + return ZoneInfo(tz) + except (KeyError, Exception): + log.warning('Unknown timezone %r — falling back to server time', tz) + return None + + +def _parse_rule(s: str, now: Optional[datetime] = None): + """Parse RRULE with clock-aligned DTSTART for sub-daily frequencies. + + SECONDLY/MINUTELY/HOURLY rules use a fixed epoch DTSTART (2000-01-01 00:00) + so intervals snap to clock boundaries (e.g. every 5min = :00, :05, :10). + """ + upper = s.upper() + if 'EXRULE' in upper: + raise ValueError('EXRULE is not supported in recurrence rules') + + parsed = rrulestr(s, ignoretz=True) + rules = parsed._rrule if isinstance(parsed, rruleset) else [parsed] + if len(rules) > 1: + raise ValueError('only one RRULE is supported per recurrence rule') + + rule = rules[0] + start = rule._dtstart.replace(tzinfo=None) + anchor = now or datetime.now() + parts = s.split() + stripped = '\n'.join(part for part in parts if not part.upper().startswith('DTSTART')) or s + has_dtstart = any(part.upper().startswith('DTSTART') for part in parts) + step = { + SECONDLY: timedelta(seconds=rule._interval), + MINUTELY: timedelta(minutes=rule._interval), + HOURLY: timedelta(hours=rule._interval), + }.get(rule._freq) + + if step is None: + if not rule._dtstart.tzinfo: + return parsed + return rrulestr(stripped, dtstart=start, ignoretz=True) + + if rule._interval < 1: + raise ValueError('RRULE INTERVAL must be a positive integer') + dtstart = None + if has_dtstart: + emitted = ((anchor - start) // step) if anchor > start else 0 + emitted *= len(rule._byminute or (0,)) * len(rule._bysecond or (0,)) + if emitted <= 100_000: + if rule._dtstart.tzinfo: + dtstart = start + else: + return parsed + if not has_dtstart or dtstart is None: + epoch = datetime(2000, 1, 1) + dtstart = epoch + ((anchor - epoch) // step) * step + + return rrulestr(stripped, dtstart=dtstart, ignoretz=True) + + +def _next_occurrences(s: str, now: datetime, n: int) -> list[datetime]: + rule = _parse_rule(s, now) + occurrences = [] + for _ in range(n): + now = rule.after(now) + if now is None: + break + occurrences.append(now) + return occurrences + + +async def _get_next_occurrences(s: str, now: datetime, n: int) -> list[datetime]: + # A result-count or date limit cannot bound work before the first match. + try: + with fail_after(RRULE_TIMEOUT_SECONDS): + return await to_process.run_sync(_next_occurrences, s, now, n, cancellable=True) + except TimeoutError as e: + raise RecurrenceEvaluationTimeout('Schedule took too long to evaluate; simplify its recurrence rule.') from e + + +async def validate_rrule(s: str, tz: str = None) -> None: + """Raise ValueError if the RRULE is malformed or exhausted. + + When *tz* is provided the "now" reference uses the user's local + clock so that near-future schedules are not incorrectly rejected + on servers whose system clock is ahead (e.g. UTC vs US timezones). + """ + upper = s.upper() + if 'COUNT=' in upper and 'DTSTART' not in upper: + raise ValueError(ERROR_MESSAGES.AUTOMATION_COUNT_REQUIRES_DTSTART) + zi = _resolve_tz(tz) + now = datetime.now(zi).replace(tzinfo=None) if zi else datetime.now() + try: + occurrences = await _get_next_occurrences(s, now, 1) + except RecurrenceEvaluationTimeout: + raise + except Exception as e: + raise ValueError(ERROR_MESSAGES.AUTOMATION_INVALID_RRULE(e)) + if not occurrences: + raise ValueError(ERROR_MESSAGES.AUTOMATION_NO_FUTURE_RUNS) + + +async def next_run_ns(s: str, tz: str = None) -> Optional[int]: + """Next occurrence as epoch nanoseconds, respecting user timezone.""" + zi = _resolve_tz(tz) + now = datetime.now(zi) if zi else datetime.now() + now_naive = now.replace(tzinfo=None) + occurrences = await _get_next_occurrences(s, now_naive, 1) + if not occurrences: + return None + dt = occurrences[0] + if zi: + dt = dt.replace(tzinfo=zi) + return int(dt.timestamp() * 1_000_000_000) + + +async def next_n_runs_ns(s: str, n: int = 5, tz: str = None) -> list[int]: + """Compute next N occurrences for UI preview. + + Uses the user's timezone for the starting "now" so that the + preview matches the user's local clock (same as next_run_ns). + """ + zi = _resolve_tz(tz) + result = [] + now = datetime.now(zi).replace(tzinfo=None) if zi else datetime.now() + for dt in await _get_next_occurrences(s, now, n): + if zi: + dt_tz = dt.replace(tzinfo=zi) + result.append(int(dt_tz.timestamp() * 1_000_000_000)) + else: + result.append(int(dt.timestamp() * 1_000_000_000)) + return result + + +async def rrule_interval_seconds(s: str) -> Optional[int]: + """Approximate interval between recurrences in seconds. + + Returns None for one-shot (COUNT=1) schedules or rules + with fewer than two future occurrences. + """ + s = '\n'.join(part for part in s.split() if not part.upper().startswith('DTSTART')) or s + now = datetime.now() + occurrences = await _get_next_occurrences(s, now, 2) + if len(occurrences) < 2: + return None + return int((occurrences[1] - occurrences[0]).total_seconds()) diff --git a/backend/open_webui/utils/terminals.py b/backend/open_webui/utils/terminals.py index 9054c1f042..e7d34550d3 100644 --- a/backend/open_webui/utils/terminals.py +++ b/backend/open_webui/utils/terminals.py @@ -1,5 +1,8 @@ """Shared routing helpers for admin-configured terminal servers.""" +import asyncio +import logging +import posixpath from urllib.parse import quote from open_webui.utils.chat_id import is_saved_chat_id @@ -9,6 +12,9 @@ TERMINAL_CONTEXT_DEFAULT = 'default' TERMINAL_CONTEXT_TYPES = {'chat', 'automation'} TERMINAL_CONTEXT_ID_SOURCES = {'chat': 'chat_id', 'automation': 'automation_id'} TERMINAL_CHAT_UPLOAD_MODES = {'default', 'filesystem'} +MAX_AGENTS_MD_BYTES = 32 * 1024 + +log = logging.getLogger(__name__) def is_terminal_orchestrator(connection: dict) -> bool: @@ -131,7 +137,14 @@ async def get_terminal_request_info(request, user, metadata: dict, extra_params: connection = next((item for item in connections if item.get('id') == terminal_id), None) if connection: - if not connection.get('enabled', True): + terminal_context = 'automation' if metadata.get('automation_id') else 'chat' + context_id = terminal_context_id(connection, metadata, terminal_context) + config = terminal_context_config(connection, terminal_context) + if ( + not connection.get('enabled', True) + or not terminal_context_available(connection, terminal_context) + or (config.get('context_id') in {'chat_id', 'automation_id'} and not context_id) + ): return None user_group_ids = {group.id for group in await Groups.get_groups_by_member_id(user_model.id)} if not await has_connection_access(user_model, connection, user_group_ids): @@ -148,15 +161,17 @@ async def get_terminal_request_info(request, user, metadata: dict, extra_params: headers['X-User-Id'] = user_model.id if metadata.get('chat_id'): headers['X-Session-Id'] = metadata['chat_id'] - terminal_context = 'automation' if metadata.get('automation_id') else 'chat' - context_id = terminal_context_id(connection, metadata, terminal_context) if context_id: headers[TERMINAL_CONTEXT_HEADER] = context_id return get_terminal_server_url(connection), headers, cookies selector = str(terminal_id).rstrip('/') direct_terminal = next( - (server for server in metadata.get('tool_servers') or [] if str(server.get('url') or '').rstrip('/') == selector), + ( + server + for server in metadata.get('tool_servers') or [] + if str(server.get('url') or '').rstrip('/') == selector + ), None, ) if not direct_terminal: @@ -171,10 +186,70 @@ async def get_terminal_request_info(request, user, metadata: dict, extra_params: return selector, headers, {} -async def get_terminal_skill(request, user, metadata: dict, skill_name: str, extra_params: dict | None = None) -> dict | None: +async def get_terminal_agents_md(request, user, metadata: dict, extra_params: dict | None = None) -> str | None: + """Load the selected terminal user's home AGENTS.md afresh for this turn.""" import aiohttp + from open_webui.env import AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL + + try: + async with asyncio.timeout(5): + terminal_request = await get_terminal_request_info(request, user, metadata, extra_params) + if not terminal_request: + return None + base_url, headers, cookies = terminal_request + async with aiohttp.ClientSession( + headers=headers, cookies=cookies, timeout=aiohttp.ClientTimeout(total=5), trust_env=True + ) as session: + async with session.get( + f'{base_url.rstrip("/")}/files/cwd', + ssl=AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL, + allow_redirects=False, + ) as response: + if response.status != 200: + log.debug('Skipping terminal AGENTS.md: home lookup returned HTTP %s', response.status) + return None + data = await response.json() + home = data.get('home') if isinstance(data, dict) else None + if not isinstance(home, str) or not posixpath.isabs(home): + return None + path = posixpath.join(home, 'AGENTS.md') + async with session.get( + f'{base_url.rstrip("/")}/files/read', + params={'path': path}, + ssl=AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL, + allow_redirects=False, + ) as response: + if response.status == 404: + return None + response.raise_for_status() + data = await response.json() + + content = data.get('content') if isinstance(data, dict) else None + if not isinstance(content, str) or not content.strip(): + return None + if len(content.encode('utf-8')) > MAX_AGENTS_MD_BYTES: + log.warning('Skipping terminal AGENTS.md: exceeds %s bytes', MAX_AGENTS_MD_BYTES) + return None + return f'# AGENTS.md\n\n{content}' + except Exception as e: + log.debug('Failed to load terminal AGENTS.md (%s)', type(e).__name__) + return None + + +def add_terminal_agents_md(messages: list[dict], agents_md: str) -> list[dict]: + """Place file instructions before user requests without changing their content.""" + for index, message in enumerate(messages): + if message.get('role') == 'user': + return [*messages[:index], {'role': 'user', 'content': agents_md}, *messages[index:]] + return messages + + +async def get_terminal_skill( + request, user, metadata: dict, skill_name: str, extra_params: dict | None = None +) -> dict | None: from urllib.parse import quote + import aiohttp from open_webui.env import AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL, AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER_DATA terminal_request = await get_terminal_request_info(request, user, metadata, extra_params) diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 31485112a9..7bd6a8dd69 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -542,6 +542,7 @@ async def get_builtin_tools( # Helper to check user-level feature permission (admins always pass) user = extra_params.get('__user__', {}) config = await Config.get_many( + 'memories.enable', 'web.search.enable', 'image_generation.enable', 'images.edit.enable', @@ -655,6 +656,7 @@ async def get_builtin_tools( # Add memory tools when memory is enabled and the model allows this builtin category. if ( is_builtin_tool_enabled('memory') + and config.get('memories.enable') and features.get('memory') and get_model_capability('memory') and await has_user_permission('memories') diff --git a/backend/requirements-slim.txt b/backend/requirements-slim.txt index 8171a6f9e6..e482bf330d 100644 --- a/backend/requirements-slim.txt +++ b/backend/requirements-slim.txt @@ -19,7 +19,8 @@ authlib==1.7.2 joserfc==1.7.4 requests==2.34.2 -regex==2026.5.9 # supports a per-search timeout, which `re` does not +regex==2026.5.9 +google-re2==1.1.20251105 # bounded compilation and linear-time knowledge searches aiohttp==3.13.5 # do not update to 3.13.3 - broken aiodns==3.6.1 # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER aiocache==0.12.3 diff --git a/backend/requirements.txt b/backend/requirements.txt index e929beafb6..11282b2807 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -14,7 +14,8 @@ authlib==1.7.2 joserfc==1.7.4 requests==2.34.2 -regex==2026.5.9 # supports a per-search timeout, which `re` does not +regex==2026.5.9 +google-re2==1.1.20251105 # bounded compilation and linear-time knowledge searches aiohttp==3.13.5 # do not update to 3.13.3 - broken aiodns==3.6.1 # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER aiocache==0.12.3 diff --git a/i18next-parser.config.ts b/i18next-parser.config.ts index 37ce57ee14..79c57d047a 100644 --- a/i18next-parser.config.ts +++ b/i18next-parser.config.ts @@ -26,7 +26,7 @@ export default { namespaceSeparator: false, output: 'src/lib/i18n/locales/$LOCALE/$NAMESPACE.json', pluralSeparator: '_', - input: 'src/**/*.{js,svelte}', + input: 'src/**/*.{js,ts,svelte}', sort: true, verbose: true, failOnWarnings: false, diff --git a/pyproject.toml b/pyproject.toml index 2fc362ae97..0f67b1b120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dependencies = [ "tiktoken==0.13.0", "regex==2026.5.9", + "google-re2==1.1.20251105", "mcp==1.27.2", "openai==2.29.0", diff --git a/scripts/prepare-pyodide.js b/scripts/prepare-pyodide.js index 64c380e3a7..8ec57426cb 100644 --- a/scripts/prepare-pyodide.js +++ b/scripts/prepare-pyodide.js @@ -11,18 +11,31 @@ const packages = [ 'regex', 'sympy', 'tiktoken', - 'seaborn', 'pytz', 'black', - 'openai', - 'openpyxl' + 'openai' ]; // Pure-Python packages whose wheels must be downloaded from PyPI and saved into // static/pyodide/ so that the browser can install them offline via micropip. // Packages already provided by the Pyodide distribution (click, platformdirs, // typing_extensions, etc.) do NOT need to be listed here. -const pypiPackages = ['black', 'pathspec', 'mypy_extensions', 'pytokens']; +// Spell them canonically (dashed): that is the only form pyodide resolves lock entries by. +const pypiPackages = [ + 'black', + 'pathspec', + 'mypy-extensions', + 'pytokens', + 'openpyxl', + 'et-xmlfile', + 'seaborn' +]; + +const pypiDepends = { + black: ['click', 'mypy-extensions', 'packaging', 'pathspec', 'platformdirs', 'pytokens'], + openpyxl: ['et-xmlfile'], + seaborn: ['matplotlib', 'numpy', 'pandas'] +}; import { loadPyodide } from 'pyodide'; import { setGlobalDispatcher, ProxyAgent } from 'undici'; @@ -175,19 +188,18 @@ async function downloadPyPIWheels() { } // Inject into pyodide-lock.json so micropip resolves locally - const normalizedName = pkg.replace(/-/g, '_'); - if (!lockData.packages[normalizedName]) { - lockData.packages[normalizedName] = { - name: normalizedName, + if (!lockData.packages[pkg]) { + lockData.packages[pkg] = { + name: pkg, version: version, file_name: wheel.filename, install_dir: 'site', sha256: wheel.digests?.sha256 || '', package_type: 'package', - imports: [normalizedName], - depends: [] + imports: [pkg.replace(/-/g, '_')], + depends: pypiDepends[pkg] || [] }; - console.log(` Added ${normalizedName}==${version} to pyodide-lock.json`); + console.log(` Added ${pkg}==${version} to pyodide-lock.json`); } } @@ -195,6 +207,31 @@ async function downloadPyPIWheels() { console.log('Updated pyodide-lock.json with PyPI packages'); } +// A package with no bundled wheel is installed from PyPI in the user's browser instead. +async function verifyBundledWheels() { + const lockPath = 'static/pyodide/pyodide-lock.json'; + const lockData = JSON.parse(await readFile(lockPath, 'utf-8')); + const missing = []; + + for (const pkg of new Set([...packages, ...pypiPackages, ...Object.values(pypiDepends).flat()])) { + const entry = lockData.packages[pkg.toLowerCase().replace(/[-_.]+/g, '-')]; + if (!entry) { + missing.push(pkg); + continue; + } + try { + await access(`static/pyodide/${entry.file_name}`); + } catch { + missing.push(pkg); + } + } + + if (missing.length) { + throw new Error(`No wheel bundled for: ${missing.join(', ')}`); + } + console.log('All listed packages are bundled'); +} + initNetworkProxyFromEnv(); if (process.env.USE_SLIM === 'true') { // Rebuild generated assets so a previous full build cannot leave bundled wheels behind. @@ -216,4 +253,5 @@ if (process.env.USE_SLIM === 'true') { await downloadPackages(); await copyPyodide(); await downloadPyPIWheels(); + await verifyBundledWheels(); } diff --git a/src/lib/apis/calendar/index.ts b/src/lib/apis/calendar/index.ts index eb02b184d6..8be3b9f52b 100644 --- a/src/lib/apis/calendar/index.ts +++ b/src/lib/apis/calendar/index.ts @@ -53,7 +53,7 @@ export type CalendarEventForm = { start_at: number; end_at?: number; all_day?: boolean; - rrule?: string; + rrule?: string | null; color?: string; location?: string | null; data?: Record; diff --git a/src/lib/apis/images/index.ts b/src/lib/apis/images/index.ts index 5d87c13a18..ad2f79d731 100644 --- a/src/lib/apis/images/index.ts +++ b/src/lib/apis/images/index.ts @@ -67,16 +67,20 @@ export const updateConfig = async (token: string = '', config: object) => { return res; }; -export const verifyConfigUrl = async (token: string = '') => { +export const verifyConnection = async ( + token: string = '', + connection: { engine: string; url: string; key?: string | null } +) => { let error = null; - const res = await fetch(`${IMAGES_API_BASE_URL}/config/url/verify`, { - method: 'GET', + const res = await fetch(`${IMAGES_API_BASE_URL}/verify`, { + method: 'POST', headers: { Accept: 'application/json', 'Content-Type': 'application/json', ...(token && { authorization: `Bearer ${token}` }) - } + }, + body: JSON.stringify(connection) }) .then(async (res) => { if (!res.ok) throw await res.json(); diff --git a/src/lib/apis/models/index.ts b/src/lib/apis/models/index.ts index eac4b9408a..11b3829cc2 100644 --- a/src/lib/apis/models/index.ts +++ b/src/lib/apis/models/index.ts @@ -1,10 +1,10 @@ import { WEBUI_API_BASE_URL } from '$lib/constants'; -export const exportModels = async (token: string, ids: string[]) => { +export const exportModels = async (token: string, ids?: string[]) => { const query = new URLSearchParams(); - ids.forEach((id) => query.append('ids', id)); - if (!ids.length) return []; - const response = await fetch(`${WEBUI_API_BASE_URL}/models/export?${query}`, { + (ids ?? []).forEach((id) => query.append('ids', id)); + if (ids && !ids.length) return []; + const response = await fetch(`${WEBUI_API_BASE_URL}/models/export${ids ? `?${query}` : ''}`, { headers: { authorization: `Bearer ${token}` } }); if (!response.ok) throw await response.json(); diff --git a/src/lib/components/AddToolServerModal.svelte b/src/lib/components/AddToolServerModal.svelte index 5e02ad8996..afde028abe 100644 --- a/src/lib/components/AddToolServerModal.svelte +++ b/src/lib/components/AddToolServerModal.svelte @@ -559,15 +559,15 @@ class=" text-xs text-gray-700 dark:text-gray-300" > {#if ['', 'openapi'].includes(type)} - {$i18n.t('OpenAPI')} + {$i18n.t('settings.admin.integrations.openApi.label')} {:else if type === 'mcp'} - {$i18n.t('MCP')} + {$i18n.t('settings.admin.integrations.mcp.label')} {$i18n.t('Streamable HTTP')} {/if} {:else}
- {$i18n.t('OpenAPI')} + {$i18n.t('settings.personal.tools.openApi.label')}
{/if} diff --git a/src/lib/components/ChangelogModal.svelte b/src/lib/components/ChangelogModal.svelte index aff4b15fce..af6cf935e1 100644 --- a/src/lib/components/ChangelogModal.svelte +++ b/src/lib/components/ChangelogModal.svelte @@ -41,7 +41,7 @@ const closeModal = async () => { localStorage.version = $config.version; await settings.set({ ...$settings, ...{ version: $config.version } }); - await updateUserSettings(localStorage.token, { ui: $settings }); + await updateUserSettings(localStorage.token, { ui: { version: $config.version } }); show = false; }; diff --git a/src/lib/components/admin/Evaluations/Feedbacks.svelte b/src/lib/components/admin/Evaluations/Feedbacks.svelte index d9bade0b7e..ee211f46ef 100644 --- a/src/lib/components/admin/Evaluations/Feedbacks.svelte +++ b/src/lib/components/admin/Evaluations/Feedbacks.svelte @@ -321,11 +321,11 @@ setSortKey('user')} + on:click={() => setSortKey('username')} >
{$i18n.t('User')} - {#if orderBy === 'user'} + {#if orderBy === 'username'} {#if direction === 'asc'} diff --git a/src/lib/components/admin/Settings/Audio.svelte b/src/lib/components/admin/Settings/Audio.svelte index ae3d2b07ef..1aa9e444eb 100644 --- a/src/lib/components/admin/Settings/Audio.svelte +++ b/src/lib/components/admin/Settings/Audio.svelte @@ -244,7 +244,9 @@ dispatch('save'); }} > -

{$i18n.t('Audio')}

+

+ {$i18n.t('settings.admin.audio.title')} +

{#if $config?.features?.slim === true}

{$i18n.t( @@ -254,10 +256,10 @@ {/if}

- +
@@ -313,7 +315,7 @@ - + {:else if STT_ENGINE === 'deepgram'} - + {:else if STT_ENGINE === 'azure'} - +
- + - + - + - + {:else if STT_ENGINE === 'mistral'}
- + - +
@@ -420,10 +422,8 @@ @@ -433,8 +433,8 @@ Do not alter, remove, obscure, or replace it except as LICENSE permits: https://docs.openwebui.com/license. -->
{#if STT_WHISPER_MODEL_LOADING} @@ -481,19 +481,15 @@ {/if} - + { - await updateConfigHandler(); - await getVoices(); - await getModels(); - const value = (e.currentTarget as HTMLSelectElement).value; if (value === 'openai') { @@ -506,6 +502,10 @@ TTS_VOICE = ''; TTS_MODEL = ''; } + + await updateConfigHandler(); + await getVoices(); + await getModels(); }} > @@ -521,50 +521,50 @@ {#if TTS_ENGINE === 'openai'}
- + - +
{:else if TTS_ENGINE === 'elevenlabs'} - + {:else if TTS_ENGINE === 'azure'} - +
- + - + {:else if TTS_ENGINE === 'mistral'}
- + - + @@ -593,7 +593,7 @@ {/if} {#if TTS_ENGINE === ''} - + {#each voices as voice} @@ -610,8 +610,8 @@ Do not alter, remove, obscure, or replace it except as LICENSE permits: https://docs.openwebui.com/license. --> {:else if TTS_ENGINE === 'openai'}
- + - +

- {$i18n.t('Leave empty for the built-in default.')} + {$i18n.t('settings.admin.subagents.systemPrompt.description')}

{/if} diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 3570a03cde..ce9690a83d 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -36,6 +36,7 @@ 'serpapi', 'duckduckgo', 'tavily', + 'staan', 'jina', 'bing', 'exa', @@ -163,22 +164,24 @@ saveHandler(); }} > -

{$i18n.t('Web Search')}

+

+ {$i18n.t('settings.admin.web.title')} +

{#if webConfig} - +