diff --git a/.dockerignore b/.dockerignore index 2b4f7b5fcf..55ab35b985 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ uploads **/*.db _test backend/data/* + +.venv +.git diff --git a/Dockerfile b/Dockerfile index d70244583b..d9ddd4590f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,24 +123,30 @@ RUN echo -n 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry # Make sure the user has access to the app and root directory RUN chown -R $UID:$GID /app $HOME -# Install common system dependencies +# Slim cannot bundle a local model server or GPU runtime. +RUN if [ "$USE_SLIM" = "true" ] && { [ "$USE_CUDA" = "true" ] || [ "$USE_OLLAMA" = "true" ]; }; then \ + echo "USE_SLIM cannot be combined with USE_CUDA or USE_OLLAMA" >&2; exit 1; fi + +# Keep the slim runtime free of local document/audio processing tools. RUN apt-get update && \ apt-get install -y --no-install-recommends \ - git build-essential pandoc gcc curl jq ca-certificates \ - libmariadb-dev \ - python3-dev \ - ffmpeg libsm6 libxext6 zstd \ - && rm -rf /var/lib/apt/lists/* + git curl jq ca-certificates zstd \ + && if [ "$USE_SLIM" != "true" ]; then \ + apt-get install -y --no-install-recommends \ + build-essential pandoc gcc libmariadb-dev ffmpeg libsm6 libxext6; \ + fi && rm -rf /var/lib/apt/lists/* # install python dependencies -COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt +COPY --chown=$UID:$GID ./backend/requirements*.txt ./ # Set UV_LINK_MODE to copy to prevent 0-byte file corruption in QEMU arm64 cross-builds ENV UV_LINK_MODE=copy RUN set -e; \ pip3 install --no-cache-dir uv; \ - if [ "$USE_CUDA" = "true" ]; then \ + if [ "$USE_SLIM" = "true" ]; then \ + uv pip install --system -r requirements-slim.txt --no-cache-dir; \ + elif [ "$USE_CUDA" = "true" ]; then \ # If you use CUDA the whisper and embedding model will be downloaded on first use # fix: pin torch<=2.9.1 - torch 2.10.0 aarch64 wheels cause SIGILL on ARM devices (RPi 4 Cortex-A72) #21349 pip3 install 'torch<=2.9.1' torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir; \ @@ -149,7 +155,6 @@ RUN set -e; \ python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ.get('AUXILIARY_EMBEDDING_MODEL', 'TaylorAI/bge-micro-v2'), device='cpu')"; \ python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \ python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \ - python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')"; \ else \ pip3 install 'torch<=2.9.1' torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir; \ uv pip install --system -r requirements.txt --no-cache-dir; \ @@ -158,7 +163,6 @@ RUN set -e; \ python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ.get('AUXILIARY_EMBEDDING_MODEL', 'TaylorAI/bge-micro-v2'), device='cpu')"; \ python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \ python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \ - python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')"; \ fi; \ fi; \ mkdir -p /app/backend/data; chown -R $UID:$GID /app/backend/data/; \ diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index ff77fbcf4e..822631ef69 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -17,6 +17,7 @@ from authlib.integrations.starlette_client import OAuth from pydantic import BaseModel from open_webui.env import ( + USE_SLIM, DATA_DIR, DATABASE_URL, ENABLE_ADMIN_CHAT_ACCESS, @@ -644,7 +645,7 @@ SSL_ASSERT_FINGERPRINT = os.getenv('SSL_ASSERT_FINGERPRINT', None) ELASTICSEARCH_INDEX_PREFIX = os.getenv('ELASTICSEARCH_INDEX_PREFIX', 'open_webui_collections') # Pgvector PGVECTOR_DB_URL = os.getenv('PGVECTOR_DB_URL', DATABASE_URL) -if VECTOR_DB == 'pgvector' and not PGVECTOR_DB_URL.startswith('postgres'): +if not USE_SLIM and VECTOR_DB == 'pgvector' and not PGVECTOR_DB_URL.startswith('postgres'): raise ValueError( 'Pgvector requires setting PGVECTOR_DB_URL or using Postgres with vector extension as the primary database.' ) @@ -805,7 +806,7 @@ ORACLE_DB_POOL_MAX = int(os.getenv('ORACLE_DB_POOL_MAX', 10)) ORACLE_DB_POOL_INCREMENT = int(os.getenv('ORACLE_DB_POOL_INCREMENT', 1)) -if VECTOR_DB == 'oracle23ai': +if not USE_SLIM and VECTOR_DB == 'oracle23ai': if not ORACLE_DB_USER or not ORACLE_DB_PASSWORD or not ORACLE_DB_DSN: raise ValueError('Oracle23ai requires setting ORACLE_DB_USER, ORACLE_DB_PASSWORD, and ORACLE_DB_DSN.') if ORACLE_DB_USE_WALLET and (not ORACLE_WALLET_DIR or not ORACLE_WALLET_PASSWORD): diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 9017cafe6c..e37aeb4052 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -7,7 +7,9 @@ import pkgutil import re import shutil import sys +import threading import traceback +from contextlib import nullcontext from pathlib import Path from typing import Any, Optional from uuid import uuid4 @@ -40,12 +42,13 @@ except ImportError: print('dotenv not installed, skipping...') DOCKER = os.getenv('DOCKER', 'False').lower() == 'true' +USE_SLIM = os.getenv('USE_SLIM_DOCKER', 'False').lower() == 'true' USE_CUDA = os.getenv('USE_CUDA_DOCKER', 'false') DEVICE_TYPE = 'cpu' _cuda_error: Optional[str] = None -if USE_CUDA.lower() == 'true': +if not USE_SLIM and USE_CUDA.lower() == 'true': try: import torch # noqa: E402 @@ -57,7 +60,7 @@ if USE_CUDA.lower() == 'true': os.environ['USE_CUDA_DOCKER'] = 'false' USE_CUDA = 'false' -if sys.platform == 'darwin' and DEVICE_TYPE == 'cpu': +if not USE_SLIM and sys.platform == 'darwin' and DEVICE_TYPE == 'cpu': try: import torch # noqa: E402 @@ -66,6 +69,9 @@ if sys.platform == 'darwin' and DEVICE_TYPE == 'cpu': except Exception: pass +# Torch MPS inference is not thread-safe and a concurrent call kills the whole process. +MPS_INFERENCE_LOCK = threading.Lock() if DEVICE_TYPE == 'mps' else nullcontext() + #################################### # LOGGING #################################### @@ -842,7 +848,7 @@ MINERU_MAX_MARKDOWN_BYTES = ( # When enabled, skips pydub-based preprocessing (format conversion, compression, # and chunked splitting) before sending files to processing engines. Useful when # the upstream provider handles these steps or when ffmpeg is unavailable. -BYPASS_PYDUB_PREPROCESSING = os.getenv('BYPASS_PYDUB_PREPROCESSING', 'False').lower() == 'true' +BYPASS_PYDUB_PREPROCESSING = USE_SLIM or os.getenv('BYPASS_PYDUB_PREPROCESSING', 'False').lower() == 'true' # When disabled (default), the OpenAI catch-all proxy endpoint (/{path:path}) # is blocked. Enable only if you need direct passthrough to upstream OpenAI- diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index a3bdf4f327..ec7827be10 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -75,6 +75,7 @@ from open_webui.config import ( ) from open_webui.constants import ERROR_MESSAGES, TASKS from open_webui.env import ( + USE_SLIM, AIOHTTP_CLIENT_SESSION_SSL, AUDIT_EXCLUDED_PATHS, AUDIT_INCLUDED_PATHS, @@ -2297,6 +2298,7 @@ async def get_app_config(request: Request): 'auto_redirect': config.get('oauth.auto_redirect'), }, 'features': { + 'slim': USE_SLIM, # --- Public: required by login/signup page pre-auth --- 'auth': WEBUI_AUTH, 'auth_trusted_header': bool(WEBUI_AUTH_TRUSTED_EMAIL_HEADER), diff --git a/backend/open_webui/models/models.py b/backend/open_webui/models/models.py index 5094276bf4..e435a492ba 100755 --- a/backend/open_webui/models/models.py +++ b/backend/open_webui/models/models.py @@ -449,11 +449,13 @@ class ModelsTable: return ModelListResponse(items=models, total=total) - async def get_model_meta_by_id(self, id: str, db: AsyncSession | None = None) -> tuple[dict, int | None]: - """Return (meta, updated_at) for a model, skipping access grant resolution.""" + async def get_model_meta_by_id( + self, id: str, db: AsyncSession | None = None + ) -> tuple[dict, str, int | None] | None: + """Return (meta, user_id, updated_at) for a model, skipping access grant resolution.""" try: async with get_async_db_context(db) as db: - result = await db.execute(select(Model.meta, Model.updated_at).filter_by(id=id)) + result = await db.execute(select(Model.meta, Model.user_id, Model.updated_at).filter_by(id=id)) return result.first() except Exception: return None diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index 13570c37f0..a2d8b366db 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -7,6 +7,7 @@ import zipfile import ftfy import requests +from fastapi import HTTPException from azure.identity import DefaultAzureCredential from langchain_community.document_loaders import ( AzureAIDocumentIntelligenceLoader, @@ -20,6 +21,7 @@ from langchain_core.documents import Document from open_webui.env import ( AIOHTTP_CLIENT_SESSION_SSL, GLOBAL_LOG_LEVEL, + USE_SLIM, MINERU_MAX_MARKDOWN_BYTES, REQUESTS_VERIFY, ) @@ -314,7 +316,11 @@ class Loader: def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: loader = self._get_loader(filename, file_content_type, file_path) docs = loader.load() - return [Document(page_content=ftfy.fix_text(doc.page_content), metadata=doc.metadata) for doc in docs] + # ftfy's auto mode unescapes entities on every line before the first literal '<', rewriting the document. + return [ + Document(page_content=ftfy.fix_text(doc.page_content, unescape_html=False), metadata=doc.metadata) + for doc in docs + ] async def aload(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: """ @@ -667,6 +673,19 @@ class Loader: file_path=file_path, ) else: + if USE_SLIM: + if file_ext == 'csv': + return CSVLoaderWithSummary(file_path, filename, self._detect_text_encoding(file_path)) + if file_ext in ['htm', 'html']: + return BSHTMLLoader(file_path, open_encoding=self._detect_text_encoding(file_path)) + if file_ext in ['txt', 'md', 'markdown', 'rst', 'xml'] or self._is_text_file( + file_ext, file_content_type + ): + return TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) + raise HTTPException( + 503, + 'This file type requires an external document extractor in slim. Configure one that supports it.', + ) if file_ext == 'pdf': loader = PyPDFLoader( file_path, diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index b1946f630d..e32c756279 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -10,8 +10,9 @@ from typing import Awaitable, Optional, Union from urllib.parse import quote import aiohttp +import numpy as np import requests -from huggingface_hub import snapshot_download +from fastapi import HTTPException from langchain_classic.retrievers import ( ContextualCompressionRetriever, EnsembleRetriever, @@ -32,8 +33,10 @@ from open_webui.env import ( BYPASS_RETRIEVAL_ACCESS_CONTROL, ENABLE_FORWARD_USER_INFO_HEADERS, ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS, + MPS_INFERENCE_LOCK, OFFLINE_MODE, RAG_SOURCE_METADATA_KEYS, + USE_SLIM, ) from open_webui.models.access_grants import AccessGrants from open_webui.models.chats import Chats @@ -46,7 +49,7 @@ from open_webui.models.users import UserModel from open_webui.retrieval.loaders.youtube import YoutubeLoader from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.retrieval.external import retrieve_external_knowledge -from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT +from open_webui.retrieval.vector.factory import get_vector_db_client from open_webui.retrieval.vector.main import GetResult, SearchResult from open_webui.retrieval.web.utils import get_web_loader from open_webui.utils.access_control.files import get_owner_accessible_folder_files, has_access_to_file @@ -333,7 +336,7 @@ class VectorSearchRetriever(BaseRetriever): def query_doc(collection_name: str, query_embedding: list[float], k: int, user: UserModel = None): try: log.debug('query_doc:doc %s', collection_name) - result = VECTOR_DB_CLIENT.search( + result = get_vector_db_client().search( collection_name=collection_name, vectors=[query_embedding], limit=k, @@ -351,7 +354,7 @@ def query_doc(collection_name: str, query_embedding: list[float], k: int, user: def get_doc(collection_name: str, user: UserModel = None): try: log.debug('get_doc:doc %s', collection_name) - result = VECTOR_DB_CLIENT.get(collection_name=collection_name) + result = get_vector_db_client().get(collection_name=collection_name) if result: log.info('query_doc:result %s %s', result.ids, result.metadatas) @@ -1110,6 +1113,8 @@ def get_embedding_function( if embedding_engine == '': # Sentence transformers: CPU-bound sync operation async def async_embedding_function(query, prefix=None, user=None): + if USE_SLIM: + raise HTTPException(503, 'Configure an external embedding engine (openai, ollama, azure_openai).') # Deferred so a missing local model degrades RAG instead of crashing boot. if embedding_function is None: raise ValueError( @@ -1117,17 +1122,16 @@ def get_embedding_function( 'SentenceTransformer model name, or configure an external ' 'RAG_EMBEDDING_ENGINE (ollama, openai, azure_openai).' ) - return await asyncio.to_thread( - ( - lambda query, prefix=None: embedding_function.encode( + + def encode(): + with MPS_INFERENCE_LOCK: + return embedding_function.encode( query, batch_size=int(embedding_batch_size), **({'prompt': prefix} if prefix else {}), ).tolist() - ), - query, - prefix, - ) + + return await asyncio.to_thread(encode) return async_embedding_function elif embedding_engine in ['ollama', 'openai', 'azure_openai']: @@ -1244,6 +1248,14 @@ async def generate_embeddings( def get_reranking_function(reranking_engine, reranking_model, reranking_function, reranking_batch_size=32): + if USE_SLIM and reranking_model and reranking_engine != 'external': + + def unavailable(query, documents, user=None): + raise HTTPException( + 503, 'Configure an external reranker, or clear the reranking model to use cosine scoring.' + ) + + return unavailable if reranking_function is None: return None if reranking_engine == 'external': @@ -1251,9 +1263,14 @@ def get_reranking_function(reranking_engine, reranking_model, reranking_function [(query, doc.page_content) for doc in documents], user=user ) else: - return lambda query, documents, user=None: reranking_function.predict( - [(query, doc.page_content) for doc in documents], batch_size=int(reranking_batch_size) - ) + + def predict(query, documents, user=None): + with MPS_INFERENCE_LOCK: + return reranking_function.predict( + [(query, doc.page_content) for doc in documents], batch_size=int(reranking_batch_size) + ) + + return predict # UUIDs, SHA-256 digests, and prefixed variants thereof all fit [A-Za-z0-9_-]. @@ -1689,6 +1706,8 @@ async def get_sources_from_items( def get_model_path(model: str, update_model: bool = False): + from huggingface_hub import snapshot_download + # Construct huggingface_hub kwargs with local_files_only to return the snapshot path cache_dir = os.getenv('SENTENCE_TRANSFORMERS_HOME') @@ -1734,6 +1753,17 @@ from langchain_core.callbacks import Callbacks from langchain_core.documents import BaseDocumentCompressor, Document +def cosine_similarity(query, documents) -> np.ndarray: + """Score one query against documents without loading a model runtime.""" + if len(documents) == 0: + return np.array([], dtype=float) + query = np.asarray(query, dtype=float).reshape(-1) + documents = np.asarray(documents, dtype=float) + query = query / max(np.linalg.norm(query), 1e-12) + documents = documents / np.maximum(np.linalg.norm(documents, axis=1, keepdims=True), 1e-12) + return documents @ query + + class RerankCompressor(BaseDocumentCompressor): embedding_function: Any top_n: int @@ -1769,18 +1799,18 @@ class RerankCompressor(BaseDocumentCompressor): query: str, callbacks: Callbacks | None = None, ) -> Sequence[Document]: + if not documents: + return [] reranking = self.reranking_function is not None scores = None if reranking: scores = await asyncio.to_thread(self.reranking_function, query, documents) else: - from sentence_transformers import util as st_util - query_embedding = await self.embedding_function(query, RAG_EMBEDDING_QUERY_PREFIX) doc_texts = [doc.page_content for doc in documents] document_embedding = await self.embedding_function(doc_texts, RAG_EMBEDDING_CONTENT_PREFIX) - scores = st_util.cos_sim(query_embedding, document_embedding)[0] + scores = cosine_similarity(query_embedding, document_embedding) if scores is not None: docs_with_scores = list( diff --git a/backend/open_webui/retrieval/vector/async_client.py b/backend/open_webui/retrieval/vector/async_client.py index 481f26e19f..af8df23b87 100644 --- a/backend/open_webui/retrieval/vector/async_client.py +++ b/backend/open_webui/retrieval/vector/async_client.py @@ -15,9 +15,8 @@ transparently dispatches each call to a worker thread via `asyncio.to_thread`. Async callers can `await ASYNC_VECTOR_DB_CLIENT.x(...)` in place of `VECTOR_DB_CLIENT.x(...)` and the loop stays responsive. -The original `VECTOR_DB_CLIENT` is unchanged, so callers already running -inside `run_in_threadpool` (e.g. `save_docs_to_vector_db`) are not -affected. +Client initialization and calls run in the worker thread. Synchronous callers +already inside `run_in_threadpool` use `get_vector_db_client()` directly. Thread-safety expectations -------------------------- @@ -55,7 +54,7 @@ from __future__ import annotations import asyncio from typing import Dict, List, Optional, Union -from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT +from open_webui.retrieval.vector.factory import get_vector_db_client from open_webui.retrieval.vector.main import ( GetResult, SearchResult, @@ -73,30 +72,30 @@ class AsyncVectorDBClient: typically swallowed by surrounding ``try/except``). """ - def __init__(self, sync_client: VectorDBBase) -> None: + def __init__(self, sync_client: Optional[VectorDBBase] = None) -> None: self._sync = sync_client @property def sync(self) -> VectorDBBase: """Escape hatch for code that must call the sync client directly (e.g. already inside a worker thread).""" - return self._sync + return self._sync if self._sync is not None else get_vector_db_client() @property def supports_hybrid_search(self) -> bool: - return type(self._sync).hybrid_search is not VectorDBBase.hybrid_search + return type(self.sync).hybrid_search is not VectorDBBase.hybrid_search async def has_collection(self, collection_name: str) -> bool: - return await asyncio.to_thread(self._sync.has_collection, collection_name) + return await asyncio.to_thread(lambda: self.sync.has_collection(collection_name)) async def delete_collection(self, collection_name: str) -> None: - return await asyncio.to_thread(self._sync.delete_collection, collection_name) + return await asyncio.to_thread(lambda: self.sync.delete_collection(collection_name)) async def insert(self, collection_name: str, items: List[VectorItem]) -> None: - return await asyncio.to_thread(self._sync.insert, collection_name, items) + return await asyncio.to_thread(lambda: self.sync.insert(collection_name, items)) async def upsert(self, collection_name: str, items: List[VectorItem]) -> None: - return await asyncio.to_thread(self._sync.upsert, collection_name, items) + return await asyncio.to_thread(lambda: self.sync.upsert(collection_name, items)) async def search( self, @@ -105,7 +104,7 @@ class AsyncVectorDBClient: filter: Optional[Dict] = None, limit: int = 10, ) -> Optional[SearchResult]: - return await asyncio.to_thread(self._sync.search, collection_name, vectors, filter, limit) + return await asyncio.to_thread(lambda: self.sync.search(collection_name, vectors, filter, limit)) async def hybrid_search( self, @@ -117,13 +116,7 @@ class AsyncVectorDBClient: hybrid_bm25_weight: float = 0.5, ) -> Optional[SearchResult]: return await asyncio.to_thread( - self._sync.hybrid_search, - collection_name, - query, - vectors, - filter, - limit, - hybrid_bm25_weight, + lambda: self.sync.hybrid_search(collection_name, query, vectors, filter, limit, hybrid_bm25_weight) ) async def query( @@ -132,10 +125,10 @@ class AsyncVectorDBClient: filter: Dict, limit: Optional[int] = None, ) -> Optional[GetResult]: - return await asyncio.to_thread(self._sync.query, collection_name, filter, limit) + return await asyncio.to_thread(lambda: self.sync.query(collection_name, filter, limit)) async def get(self, collection_name: str) -> Optional[GetResult]: - return await asyncio.to_thread(self._sync.get, collection_name) + return await asyncio.to_thread(lambda: self.sync.get(collection_name)) async def delete( self, @@ -143,10 +136,10 @@ class AsyncVectorDBClient: ids: Optional[List[str]] = None, filter: Optional[Dict] = None, ) -> None: - return await asyncio.to_thread(self._sync.delete, collection_name, ids, filter) + return await asyncio.to_thread(lambda: self.sync.delete(collection_name, ids, filter)) async def reset(self) -> None: - return await asyncio.to_thread(self._sync.reset) + return await asyncio.to_thread(lambda: self.sync.reset()) -ASYNC_VECTOR_DB_CLIENT = AsyncVectorDBClient(VECTOR_DB_CLIENT) +ASYNC_VECTOR_DB_CLIENT = AsyncVectorDBClient() diff --git a/backend/open_webui/retrieval/vector/dbs/chroma.py b/backend/open_webui/retrieval/vector/dbs/chroma.py index f6629164ff..2c51aeca93 100755 --- a/backend/open_webui/retrieval/vector/dbs/chroma.py +++ b/backend/open_webui/retrieval/vector/dbs/chroma.py @@ -16,6 +16,8 @@ from open_webui.config import ( CHROMA_HTTP_SSL, CHROMA_TENANT, ) +from open_webui.env import USE_SLIM +from fastapi import HTTPException from open_webui.retrieval.vector.main import ( GetResult, SearchResult, @@ -29,6 +31,8 @@ log = logging.getLogger(__name__) class ChromaClient(VectorDBBase): def __init__(self): + if USE_SLIM and not CHROMA_HTTP_HOST: + raise HTTPException(503, 'Configure CHROMA_HTTP_HOST: embedded Chroma is unavailable in slim.') settings_dict = { 'allow_reset': True, 'anonymized_telemetry': False, @@ -58,7 +62,7 @@ class ChromaClient(VectorDBBase): def has_collection(self, collection_name: str) -> bool: try: - self.client.get_collection(name=collection_name) + self.client.get_collection(name=collection_name, embedding_function=None) return True except NotFoundError: return False @@ -76,7 +80,7 @@ class ChromaClient(VectorDBBase): ) -> Optional[SearchResult]: # Search for the nearest neighbor items based on the vectors and return 'limit' number of results. try: - collection = self.client.get_collection(name=collection_name) + collection = self.client.get_collection(name=collection_name, embedding_function=None) if collection: result = collection.query( query_embeddings=vectors, @@ -105,7 +109,7 @@ class ChromaClient(VectorDBBase): def query(self, collection_name: str, filter: dict, limit: Optional[int] = None) -> Optional[GetResult]: # Query the items from the collection based on the filter. try: - collection = self.client.get_collection(name=collection_name) + collection = self.client.get_collection(name=collection_name, embedding_function=None) if collection: result = collection.get( where=filter, @@ -125,7 +129,7 @@ class ChromaClient(VectorDBBase): def get(self, collection_name: str) -> Optional[GetResult]: # Get all the items in the collection. - collection = self.client.get_collection(name=collection_name) + collection = self.client.get_collection(name=collection_name, embedding_function=None) if collection: result = collection.get() return GetResult( @@ -139,7 +143,9 @@ class ChromaClient(VectorDBBase): def insert(self, collection_name: str, items: list[VectorItem]): # Insert the items into the collection, if the collection does not exist, it will be created. - collection = self.client.get_or_create_collection(name=collection_name, metadata={'hnsw:space': 'cosine'}) + collection = self.client.get_or_create_collection( + name=collection_name, metadata={'hnsw:space': 'cosine'}, embedding_function=None + ) ids = [item['id'] for item in items] documents = [item['text'] for item in items] @@ -157,7 +163,9 @@ class ChromaClient(VectorDBBase): def upsert(self, collection_name: str, items: list[VectorItem]): # Update the items in the collection, if the items are not present, insert them. If the collection does not exist, it will be created. - collection = self.client.get_or_create_collection(name=collection_name, metadata={'hnsw:space': 'cosine'}) + collection = self.client.get_or_create_collection( + name=collection_name, metadata={'hnsw:space': 'cosine'}, embedding_function=None + ) ids = [item['id'] for item in items] documents = [item['text'] for item in items] @@ -174,7 +182,7 @@ class ChromaClient(VectorDBBase): ): # Delete the items from the collection based on the ids. try: - collection = self.client.get_collection(name=collection_name) + collection = self.client.get_collection(name=collection_name, embedding_function=None) if collection: if ids: collection.delete(ids=ids) diff --git a/backend/open_webui/retrieval/vector/factory.py b/backend/open_webui/retrieval/vector/factory.py index 3080956163..8173bc1b6f 100644 --- a/backend/open_webui/retrieval/vector/factory.py +++ b/backend/open_webui/retrieval/vector/factory.py @@ -1,8 +1,12 @@ +from threading import Lock + +from fastapi import HTTPException from open_webui.config import ( ENABLE_MILVUS_MULTITENANCY_MODE, ENABLE_QDRANT_MULTITENANCY_MODE, VECTOR_DB, ) +from open_webui.env import USE_SLIM from open_webui.retrieval.vector.main import VectorDBBase from open_webui.retrieval.vector.type import VectorType @@ -88,4 +92,35 @@ class Vector: raise ValueError(f'Unsupported vector type: {vector_type}') -VECTOR_DB_CLIENT = Vector.get_vector(VECTOR_DB) +VECTOR_DB_CLIENT = None if USE_SLIM else Vector.get_vector(VECTOR_DB) +_vector_client_lock = Lock() + + +def get_vector_db_client() -> VectorDBBase: + """Initialize slim's remote client on first use so chat can start without it.""" + global VECTOR_DB_CLIENT + if VECTOR_DB_CLIENT is not None: + return VECTOR_DB_CLIENT + with _vector_client_lock: + if VECTOR_DB_CLIENT is None: + from open_webui import config + + if VECTOR_DB == VectorType.CHROMA and not config.CHROMA_HTTP_HOST: + raise HTTPException( + 503, 'Slim requires remote vector storage. Set CHROMA_HTTP_HOST or configure another VECTOR_DB.' + ) + if VECTOR_DB == VectorType.MILVUS and not config.MILVUS_URI.startswith(('http://', 'https://', 'tcp://')): + raise HTTPException(503, 'Slim requires an external MILVUS_URI, not a local database file.') + if VECTOR_DB == VectorType.QDRANT and not config.QDRANT_URI: + raise HTTPException(503, 'Configure QDRANT_URI for remote vector storage.') + if VECTOR_DB == VectorType.PGVECTOR and not config.PGVECTOR_DB_URL.startswith('postgres'): + raise HTTPException(503, 'Configure PGVECTOR_DB_URL for remote vector storage.') + try: + VECTOR_DB_CLIENT = Vector.get_vector(VECTOR_DB) + except HTTPException: + raise + except Exception as exc: + raise HTTPException( + 503, f'Unable to connect to configured vector database ({VECTOR_DB}): {exc}' + ) from exc + return VECTOR_DB_CLIENT diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index a6099afe1d..25ec988279 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -29,7 +29,9 @@ import urllib3.connection import urllib3.connectionpool import validators from requests.adapters import HTTPAdapter +from fastapi import HTTPException from fastapi.concurrency import run_in_threadpool +from bs4 import BeautifulSoup from langchain_community.document_loaders import PlaywrightURLLoader, WebBaseLoader from langchain_community.document_loaders.base import BaseLoader from langchain_core.documents import Document @@ -58,6 +60,7 @@ from open_webui.env import ( AIOHTTP_CLIENT_SSL_CERT_FILE, AIOHTTP_CLIENT_TIMEOUT, USER_AGENT, + USE_SLIM, ) from open_webui.retrieval.loaders.external_web import ExternalWebLoader from open_webui.retrieval.loaders.microsoft_web_iq import MicrosoftWebIQLoader @@ -643,6 +646,26 @@ class SafeMicrosoftWebIQLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): raise e +class TextHtmlEvaluator: + """Extract rendered text without Unstructured or its local NLP dependencies.""" + + def __init__(self, remove_selectors=None): + self.remove_selectors = remove_selectors or [] + + def text(self, html): + soup = BeautifulSoup(html, 'lxml') + for selector in ['script', 'style', 'noscript', *self.remove_selectors]: + for element in soup.select(selector): + element.decompose() + return soup.get_text(separator='\n', strip=True) + + def evaluate(self, page, browser, response): + return self.text(page.content()) + + async def evaluate_async(self, page, browser, response): + return self.text(await page.content()) + + class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessingMixin): """Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection. @@ -674,6 +697,8 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing playwright_timeout: Optional[int] = 10000, ): """Initialize with additional safety parameters and remote browser support.""" + if USE_SLIM and not playwright_ws_url: + raise HTTPException(503, 'Configure PLAYWRIGHT_WS_URL. Slim requires a remote browser.') proxy_server = proxy.get('server') if proxy else None if trust_env and not proxy_server: @@ -690,7 +715,8 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing urls=web_paths, continue_on_failure=continue_on_failure, headless=headless if playwright_ws_url is None else False, - remove_selectors=remove_selectors, + remove_selectors=None if USE_SLIM else remove_selectors, + evaluator=TextHtmlEvaluator(remove_selectors) if USE_SLIM else None, proxy=proxy, ) self.verify_ssl = verify_ssl diff --git a/backend/open_webui/routers/audio.py b/backend/open_webui/routers/audio.py index 9fa5d89304..dfc0ecde65 100644 --- a/backend/open_webui/routers/audio.py +++ b/backend/open_webui/routers/audio.py @@ -46,6 +46,7 @@ from open_webui.env import ( DEVICE_TYPE, ENABLE_FORWARD_USER_INFO_HEADERS, ENV, + USE_SLIM, ) from open_webui.events import EVENTS, publish_event from open_webui.models.config import Config @@ -58,9 +59,10 @@ from open_webui.utils.session_pool import get_session from pydantic import BaseModel # pydub needs stdlib audioop (gone in 3.13); keep requires-python capped < 3.13 -from pydub import AudioSegment -from pydub.silence import split_on_silence -from pydub.utils import mediainfo +if not USE_SLIM: + from pydub import AudioSegment + from pydub.silence import split_on_silence + from pydub.utils import mediainfo log = logging.getLogger(__name__) router = APIRouter() @@ -213,6 +215,8 @@ def transcode_audio_to_mp3(audio_data: bytes, content_type_header: str, output_p def set_faster_whisper_model(model: str, auto_update: bool = False): + if USE_SLIM: + raise HTTPException(503, 'Configure an external speech-to-text engine. Local Whisper is unavailable in slim.') whisper_model = None if model: from faster_whisper import WhisperModel @@ -285,6 +289,12 @@ async def get_audio_config(request: Request, user=Depends(get_admin_user)): @router.post('/config/update') async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm, user=Depends(get_admin_user)): + if USE_SLIM: + current = await Config.get_many('audio.stt.engine', 'audio.tts.engine') + if form_data.stt.ENGINE == '' and current.get('audio.stt.engine') != '': + raise HTTPException(400, 'Local Whisper is unavailable in slim. Select an external speech-to-text engine.') + if form_data.tts.ENGINE == 'transformers' and current.get('audio.tts.engine') != 'transformers': + raise HTTPException(400, 'Local TTS is unavailable in slim. Select an external text-to-speech engine.') await Config.upsert( { **config_updates(form_data.tts.model_dump(), TTS_CONFIG_KEYS), @@ -292,7 +302,7 @@ async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm } ) - if form_data.stt.ENGINE == '': + if form_data.stt.ENGINE == '' and not USE_SLIM: request.app.state.faster_whisper_model = await asyncio.to_thread( set_faster_whisper_model, form_data.stt.WHISPER_MODEL, WHISPER_MODEL_AUTO_UPDATE ) @@ -314,6 +324,8 @@ async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm def load_speech_pipeline(request): + if USE_SLIM: + raise HTTPException(503, 'Configure an external text-to-speech engine. Local TTS is unavailable in slim.') from datasets import load_dataset from transformers import pipeline @@ -328,7 +340,9 @@ def load_speech_pipeline(request): async def _raise_tts_error(exc: Exception, r=None) -> None: """Raise a standardised HTTPException from a TTS provider failure.""" - code = r.status if r is not None else 500 + if isinstance(exc, HTTPException): + raise exc + code = r.status if r is not None and r.status >= 400 else 500 # LICENSE covers this Open WebUI error identifier. # Do not alter, remove, obscure, or replace it except as LICENSE permits: # https://docs.openwebui.com/license. @@ -351,8 +365,29 @@ async def _write_tts_cache( audio: bytes, body_path: Path, payload: dict, + content_type: str = 'audio/mpeg', ) -> None: """Persist audio + request metadata to the speech cache.""" + if USE_SLIM: + mime_type = content_type.split(';')[0].strip().lower() + if mime_type not in { + 'audio/mpeg', + 'audio/mp3', + 'audio/wav', + 'audio/x-wav', + 'audio/ogg', + 'audio/opus', + 'audio/webm', + 'audio/flac', + 'audio/aac', + 'audio/mp4', + }: + raise HTTPException( + 502, + f'TTS returned unsupported format {mime_type}. Configure the provider to return MP3, WAV, Ogg, or another browser-playable audio format.', + ) + async with aiofiles.open(file_path.with_suffix('.mime'), 'w') as f: + await f.write(content_type) async with aiofiles.open(file_path, 'wb') as f: await f.write(audio) async with aiofiles.open(body_path, 'w') as f: @@ -389,6 +424,10 @@ async def _tts_openai(request, payload, file_path, file_body_path, user): audio_data = await r.read() content_type = r.headers.get('Content-Type', 'audio/mpeg') + if USE_SLIM: + await _write_tts_cache(file_path, audio_data, file_body_path, payload, content_type) + return FileResponse(file_path, media_type=content_type) + if not await asyncio.to_thread(transcode_audio_to_mp3, audio_data, content_type, file_path): async with aiofiles.open(file_path, 'wb') as f: await f.write(audio_data) @@ -430,8 +469,9 @@ async def _tts_elevenlabs(request, payload, file_path, file_body_path, user): ssl=AIOHTTP_CLIENT_SESSION_SSL, ) as r: r.raise_for_status() - await _write_tts_cache(file_path, await r.read(), file_body_path, payload) - return FileResponse(file_path) + content_type = r.headers.get('Content-Type', 'audio/mpeg') + await _write_tts_cache(file_path, await r.read(), file_body_path, payload, content_type) + return FileResponse(file_path, media_type=content_type if USE_SLIM else None) except Exception as exc: log.exception(exc) await _raise_tts_error(exc, r) @@ -465,8 +505,9 @@ async def _tts_azure(request, payload, file_path, file_body_path, user): ssl=AIOHTTP_CLIENT_SESSION_SSL, ) as r: r.raise_for_status() - await _write_tts_cache(file_path, await r.read(), file_body_path, payload) - return FileResponse(file_path) + content_type = r.headers.get('Content-Type', 'audio/mpeg') + await _write_tts_cache(file_path, await r.read(), file_body_path, payload, content_type) + return FileResponse(file_path, media_type=content_type if USE_SLIM else None) except Exception as exc: log.exception(exc) await _raise_tts_error(exc, r) @@ -474,6 +515,8 @@ async def _tts_azure(request, payload, file_path, file_body_path, user): async def _tts_transformers(request, payload, file_path, file_body_path, user): """Generate speech via the local HuggingFace SpeechT5 pipeline (thread-offloaded).""" + if USE_SLIM: + raise HTTPException(503, 'Configure an external text-to-speech engine. Local TTS is unavailable in slim.') import soundfile as sf import torch @@ -558,6 +601,8 @@ _TTS_ENGINES = { @router.post('/speech') async def speech(request: Request, user=Depends(get_verified_user)): engine = await Config.get('audio.tts.engine') + if USE_SLIM and engine in ('', 'transformers'): + raise HTTPException(503, 'Configure an external text-to-speech engine.') if engine == '': raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -572,7 +617,10 @@ async def speech(request: Request, user=Depends(get_verified_user)): body = await request.body() name = hashlib.sha256( - body + str(engine).encode('utf-8') + str(await Config.get('audio.tts.model')).encode('utf-8') + body + + str(engine).encode('utf-8') + + str(await Config.get('audio.tts.model')).encode('utf-8') + + (b':slim' if USE_SLIM else b'') ).hexdigest() file_path = SPEECH_CACHE_DIR.joinpath(f'{name}.mp3') @@ -587,7 +635,11 @@ async def speech(request: Request, user=Depends(get_verified_user)): subject_id=name, data={'engine': engine, 'cached': True}, ) - return FileResponse(file_path) + content_type = None + if USE_SLIM: + async with aiofiles.open(file_path.with_suffix('.mime')) as f: + content_type = await f.read() + return FileResponse(file_path, media_type=content_type) try: payload = JSONCodec.loads(body) @@ -960,7 +1012,11 @@ async def _transcribe_mistral(request, file_path, filename, metadata, file_dir, session = await get_session() if use_chat_completions: audio_file_to_use = file_path - if is_audio_conversion_required(file_path): + if USE_SLIM and Path(filename).suffix.lower() not in ('.mp3', '.wav'): + raise HTTPException( + 400, 'Mistral chat transcription requires MP3 or WAV in slim; local conversion is unavailable.' + ) + if not BYPASS_PYDUB_PREPROCESSING and is_audio_conversion_required(file_path): log.debug('Converting audio to mp3 for chat completions API') converted_path = await asyncio.to_thread(convert_audio_to_mp3, file_path) if converted_path: diff --git a/backend/open_webui/routers/evaluations.py b/backend/open_webui/routers/evaluations.py index 8a6bf71f4a..00c1517c88 100644 --- a/backend/open_webui/routers/evaluations.py +++ b/backend/open_webui/routers/evaluations.py @@ -4,7 +4,10 @@ from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.concurrency import run_in_threadpool from open_webui.constants import ERROR_MESSAGES +from open_webui.env import MPS_INFERENCE_LOCK from open_webui.events import EVENTS, publish_event +from open_webui.env import USE_SLIM +from open_webui.retrieval.utils import cosine_similarity from open_webui.internal.db import get_async_session from open_webui.models.config import Config from open_webui.models.feedbacks import ( @@ -68,6 +71,8 @@ _embedding_model = None def _get_embedding_model(): global _embedding_model + if USE_SLIM: + return None if _embedding_model is None: try: from sentence_transformers import SentenceTransformer @@ -179,8 +184,9 @@ def _compute_similarities(feedbacks: list[LeaderboardFeedbackData], query: str) return {} try: - tag_embeddings = embedding_model.encode(all_tags) - query_embedding = embedding_model.encode([query])[0] + with MPS_INFERENCE_LOCK: + tag_embeddings = embedding_model.encode(all_tags) + query_embedding = embedding_model.encode([query])[0] except Exception as e: log.error(f'Embedding error: {e}') return {} @@ -215,6 +221,7 @@ class LeaderboardResponse(BaseModel): @router.get('/leaderboard', response_model=LeaderboardResponse) async def get_leaderboard( + request: Request, query: Optional[str] = None, user=Depends(get_admin_user), db: AsyncSession = Depends(get_async_session), @@ -224,7 +231,17 @@ async def get_leaderboard( similarities = None if query and query.strip(): - similarities = await run_in_threadpool(_compute_similarities, feedbacks, query.strip()) + if USE_SLIM: + tags = list({tag for feedback in feedbacks for tag in (feedback.data or {}).get('tags', [])}) + embeddings = await request.app.state.EMBEDDING_FUNCTION([query.strip(), *tags], user=user) + scores = cosine_similarity(embeddings[0], embeddings[1:]) + tag_scores = dict(zip(tags, scores.tolist())) + similarities = { + feedback.id: max((tag_scores.get(tag, 0) for tag in (feedback.data or {}).get('tags', [])), default=0) + for feedback in feedbacks + } + else: + similarities = await run_in_threadpool(_compute_similarities, feedbacks, query.strip()) elo_stats = _calculate_elo(feedbacks, similarities) tags_by_model = _get_top_tags(feedbacks) diff --git a/backend/open_webui/routers/models.py b/backend/open_webui/routers/models.py index a0fd97705d..baa8bae604 100644 --- a/backend/open_webui/routers/models.py +++ b/backend/open_webui/routers/models.py @@ -19,7 +19,11 @@ from fastapi import ( from fastapi.responses import RedirectResponse, StreamingResponse from open_webui.config import BYPASS_ADMIN_ACCESS_CONTROL from open_webui.constants import ERROR_MESSAGES -from open_webui.env import ENABLE_PROFILE_IMAGE_URL_FORWARDING, PROFILE_IMAGE_ALLOWED_MIME_TYPES +from open_webui.env import ( + BYPASS_MODEL_ACCESS_CONTROL, + ENABLE_PROFILE_IMAGE_URL_FORWARDING, + PROFILE_IMAGE_ALLOWED_MIME_TYPES, +) from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_session from open_webui.models.access_grants import AccessGrants @@ -36,7 +40,7 @@ from open_webui.models.models import ( ModelResponse, Models, ) -from open_webui.utils.access_control import filter_allowed_access_grants, has_permission +from open_webui.utils.access_control import filter_allowed_access_grants, has_access, has_permission from open_webui.utils.access_control.files import has_access_to_file from open_webui.utils.auth import get_admin_user, get_verified_user from open_webui.utils.chat_variables import get_chat_variables_schema @@ -649,18 +653,37 @@ async def get_model_profile_image( profile_image_url = None updated_at = None + bypass_access_control = BYPASS_MODEL_ACCESS_CONTROL or (user.role == 'admin' and BYPASS_ADMIN_ACCESS_CONTROL) + # First, check the database for regular models model_meta = await Models.get_model_meta_by_id(id, db=db) if model_meta: - meta, updated_at = model_meta - profile_image_url = (meta or {}).get('profile_image_url') + meta, model_user_id, model_updated_at = model_meta + # Denied callers get the default image rather than an error, so model ids stay unprobeable. + if ( + bypass_access_control + or user.id == model_user_id + or await AccessGrants.has_access( + user_id=user.id, + resource_type='model', + resource_id=id, + permission='read', + db=db, + ) + ): + profile_image_url = (meta or {}).get('profile_image_url') + updated_at = model_updated_at # Fallback: check arena models stored in config (not in the DB) if not profile_image_url: arena_models = await Config.get('evaluation.arena.models', []) or [] for arena_model in arena_models: if arena_model.get('id') == id: - profile_image_url = arena_model.get('meta', {}).get('profile_image_url') + arena_meta = arena_model.get('meta', {}) + if bypass_access_control or await has_access( + user.id, permission='read', access_grants=arena_meta.get('access_grants', []), db=db + ): + profile_image_url = arena_meta.get('profile_image_url') break if profile_image_url: diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 9437649d4d..824743fd5d 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -58,6 +58,7 @@ from open_webui.env import ( SENTENCE_TRANSFORMERS_CROSS_ENCODER_MODEL_KWARGS, SENTENCE_TRANSFORMERS_CROSS_ENCODER_SIGMOID_ACTIVATION_FUNCTION, SENTENCE_TRANSFORMERS_MODEL_KWARGS, + USE_SLIM, ) from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_db, get_async_session @@ -82,7 +83,7 @@ from open_webui.retrieval.utils import ( query_doc_with_hybrid_search, ) from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT -from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT +from open_webui.retrieval.vector.factory import get_vector_db_client from open_webui.retrieval.vector.utils import filter_metadata from open_webui.retrieval.web.azure import search_azure from open_webui.retrieval.web.bing import search_bing @@ -150,7 +151,7 @@ def get_ef( auto_update: bool = RAG_EMBEDDING_MODEL_AUTO_UPDATE, ): ef = None - if embedding_model and engine == '': + if embedding_model and engine == '' and not USE_SLIM: from sentence_transformers import SentenceTransformer try: @@ -178,6 +179,17 @@ def get_rf( rf = None # Convert timeout string to int or None (system default) timeout_value = int(external_reranker_timeout) if external_reranker_timeout else None + if reranking_model and engine == 'external': + from open_webui.retrieval.models.external import ExternalReranker + + return ExternalReranker( + url=external_reranker_url, + api_key=external_reranker_api_key, + model=reranking_model, + timeout=timeout_value, + ) + if USE_SLIM: + return None if reranking_model: if any(model in reranking_model for model in ['jinaai/jina-colbert-v2']): try: @@ -192,55 +204,39 @@ def get_rf( log.error(f'ColBERT: {e}') raise Exception(ERROR_MESSAGES.DEFAULT(e, 'Error loading reranking model')) else: - if engine == 'external': - try: - from open_webui.retrieval.models.external import ExternalReranker + import sentence_transformers + import torch - rf = ExternalReranker( - url=external_reranker_url, - api_key=external_reranker_api_key, - model=reranking_model, - timeout=timeout_value, - ) - except Exception as e: - log.error(f'ExternalReranking: {e}') - raise Exception(ERROR_MESSAGES.DEFAULT(e, 'Error loading reranking model')) - else: - import sentence_transformers - import torch + try: + rf = sentence_transformers.CrossEncoder( + get_model_path(reranking_model, auto_update), + device=DEVICE_TYPE, + trust_remote_code=RAG_RERANKING_MODEL_TRUST_REMOTE_CODE, + backend=SENTENCE_TRANSFORMERS_CROSS_ENCODER_BACKEND, + model_kwargs=SENTENCE_TRANSFORMERS_CROSS_ENCODER_MODEL_KWARGS, + activation_fn=( + torch.nn.Sigmoid() if SENTENCE_TRANSFORMERS_CROSS_ENCODER_SIGMOID_ACTIVATION_FUNCTION else None + ), + ) + except Exception as e: + log.error(f'CrossEncoder: {e}') + raise Exception(ERROR_MESSAGES.DEFAULT(e, 'CrossEncoder error')) - try: - rf = sentence_transformers.CrossEncoder( - get_model_path(reranking_model, auto_update), - device=DEVICE_TYPE, - trust_remote_code=RAG_RERANKING_MODEL_TRUST_REMOTE_CODE, - backend=SENTENCE_TRANSFORMERS_CROSS_ENCODER_BACKEND, - model_kwargs=SENTENCE_TRANSFORMERS_CROSS_ENCODER_MODEL_KWARGS, - activation_fn=( - torch.nn.Sigmoid() - if SENTENCE_TRANSFORMERS_CROSS_ENCODER_SIGMOID_ACTIVATION_FUNCTION - else None - ), - ) - except Exception as e: - log.error(f'CrossEncoder: {e}') - raise Exception(ERROR_MESSAGES.DEFAULT(e, 'CrossEncoder error')) - - # Safely adjust pad_token_id if missing as some models do not have this in config - try: - model_cfg = getattr(rf, 'model', None) - if model_cfg and hasattr(model_cfg, 'config'): - cfg = model_cfg.config - if getattr(cfg, 'pad_token_id', None) is None: - # Fallback to eos_token_id when available - eos = getattr(cfg, 'eos_token_id', None) - if eos is not None: - cfg.pad_token_id = eos - log.debug('Missing pad_token_id detected; set to eos_token_id=%s', eos) - else: - log.warning('Neither pad_token_id nor eos_token_id present in model config') - except Exception as e2: - log.warning(f'Failed to adjust pad_token_id on CrossEncoder: {e2}') + # Safely adjust pad_token_id if missing as some models do not have this in config + try: + model_cfg = getattr(rf, 'model', None) + if model_cfg and hasattr(model_cfg, 'config'): + cfg = model_cfg.config + if getattr(cfg, 'pad_token_id', None) is None: + # Fallback to eos_token_id when available + eos = getattr(cfg, 'eos_token_id', None) + if eos is not None: + cfg.pad_token_id = eos + log.debug('Missing pad_token_id detected; set to eos_token_id=%s', eos) + else: + log.warning('Neither pad_token_id nor eos_token_id present in model config') + except Exception as e2: + log.warning(f'Failed to adjust pad_token_id on CrossEncoder: {e2}') return rf @@ -535,6 +531,8 @@ async def unload_embedding_model(request: Request): @router.post('/embedding/update') async def update_embedding_config(request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user)): + if USE_SLIM and form_data.RAG_EMBEDDING_ENGINE == '': + raise HTTPException(400, 'Slim requires an external embedding engine (openai, ollama, azure_openai).') config = await get_retrieval_config() log.info('Updating embedding model: %s to %s', config.RAG_EMBEDDING_MODEL, form_data.RAG_EMBEDDING_MODEL) await unload_embedding_model(request) @@ -952,6 +950,44 @@ class ConfigForm(BaseModel): async def update_rag_config(request: Request, form_data: ConfigForm, user=Depends(get_admin_user)): # RAG settings config = await get_retrieval_config() + if USE_SLIM: + if form_data.web: + web_engine = ( + form_data.web.WEB_LOADER_ENGINE + if form_data.web.WEB_LOADER_ENGINE is not None + else config.WEB_LOADER_ENGINE + ) + browser_url = ( + form_data.web.PLAYWRIGHT_WS_URL + if form_data.web.PLAYWRIGHT_WS_URL is not None + else config.PLAYWRIGHT_WS_URL + ) + if ( + web_engine == 'playwright' + and not browser_url + and (web_engine != config.WEB_LOADER_ENGINE or browser_url != config.PLAYWRIGHT_WS_URL) + ): + raise HTTPException(400, 'Configure PLAYWRIGHT_WS_URL. Slim requires a remote browser.') + if form_data.TEXT_SPLITTER == 'token_transformers' and config.TEXT_SPLITTER != 'token_transformers': + raise HTTPException( + 400, 'Transformers tokenization is unavailable in slim. Use character or token splitting.' + ) + reranker_engine = ( + form_data.RAG_RERANKING_ENGINE + if form_data.RAG_RERANKING_ENGINE is not None + else config.RAG_RERANKING_ENGINE + ) + reranker_model = ( + form_data.RAG_RERANKING_MODEL if form_data.RAG_RERANKING_MODEL is not None else config.RAG_RERANKING_MODEL + ) + if ( + reranker_engine != 'external' + and reranker_model + and (reranker_engine != config.RAG_RERANKING_ENGINE or reranker_model != config.RAG_RERANKING_MODEL) + ): + raise HTTPException( + 400, 'Slim requires an external reranker, or an empty reranking model for cosine scoring.' + ) config.RAG_TEMPLATE = form_data.RAG_TEMPLATE if form_data.RAG_TEMPLATE is not None else config.RAG_TEMPLATE config.TOP_K = form_data.TOP_K if form_data.TOP_K is not None else config.TOP_K config.BYPASS_EMBEDDING_AND_RETRIEVAL = ( @@ -1589,6 +1625,8 @@ def merge_docs_to_target_size( def get_transformers_tokenizer(request: Request, config: RetrievalConfig): + if USE_SLIM: + raise HTTPException(503, 'Transformers tokenization is unavailable in slim. Use character or token splitting.') if config.RAG_TOKENIZER_MODEL: from transformers import AutoTokenizer @@ -1673,7 +1711,7 @@ def save_docs_to_vector_db( # Check if entries with the same hash (metadata.hash) already exist if metadata and 'hash' in metadata: - result = VECTOR_DB_CLIENT.query( + result = get_vector_db_client().query( collection_name=collection_name, filter={'hash': metadata['hash']}, ) @@ -1773,11 +1811,11 @@ def save_docs_to_vector_db( ] try: - if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name): + if get_vector_db_client().has_collection(collection_name=collection_name): log.info('collection %s already exists', collection_name) if overwrite: - VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) + get_vector_db_client().delete_collection(collection_name=collection_name) log.info('deleting existing collection %s', collection_name) elif add is False: log.info('collection %s already exists, overwrite is False and add is False', collection_name) @@ -1840,7 +1878,7 @@ def save_docs_to_vector_db( ] log.info('adding to collection %s', collection_name) - VECTOR_DB_CLIENT.insert( + get_vector_db_client().insert( collection_name=collection_name, items=items, ) @@ -3035,7 +3073,7 @@ async def query_doc_handler( query_embedding = await request.app.state.EMBEDDING_FUNCTION( form_data.query, prefix=RAG_EMBEDDING_QUERY_PREFIX, user=user ) - # query_doc wraps a blocking VECTOR_DB_CLIENT.search call; + # query_doc wraps a blocking get_vector_db_client().search call; # offload so the request's event loop stays responsive. return await asyncio.to_thread( query_doc, diff --git a/backend/open_webui/static/assets/pdf-style.css b/backend/open_webui/static/assets/pdf-style.css index 644dd58ae6..68c10b9d1a 100644 --- a/backend/open_webui/static/assets/pdf-style.css +++ b/backend/open_webui/static/assets/pdf-style.css @@ -1,24 +1,4 @@ /* HTML and Body */ -@font-face { - font-family: 'NotoSans'; - src: url('fonts/NotoSans-Variable.ttf'); -} - -@font-face { - font-family: 'NotoSansJP'; - src: url('fonts/NotoSansJP-Variable.ttf'); -} - -@font-face { - font-family: 'NotoSansKR'; - src: url('fonts/NotoSansKR-Variable.ttf'); -} - -@font-face { - font-family: 'NotoSansSC'; - src: url('fonts/NotoSansSC-Variable.ttf'); -} - @font-face { font-family: 'NotoSansSC-Regular'; src: url('fonts/NotoSansSC-Regular.ttf'); diff --git a/backend/open_webui/static/fonts/NotoSans-Variable.ttf b/backend/open_webui/static/fonts/NotoSans-Variable.ttf deleted file mode 100644 index f7d0d78ed3..0000000000 Binary files a/backend/open_webui/static/fonts/NotoSans-Variable.ttf and /dev/null differ diff --git a/backend/open_webui/static/fonts/NotoSansJP-Variable.ttf b/backend/open_webui/static/fonts/NotoSansJP-Variable.ttf deleted file mode 100644 index 53b5ec5888..0000000000 Binary files a/backend/open_webui/static/fonts/NotoSansJP-Variable.ttf and /dev/null differ diff --git a/backend/open_webui/static/fonts/NotoSansKR-Variable.ttf b/backend/open_webui/static/fonts/NotoSansKR-Variable.ttf deleted file mode 100644 index 57016c27f9..0000000000 Binary files a/backend/open_webui/static/fonts/NotoSansKR-Variable.ttf and /dev/null differ diff --git a/backend/open_webui/static/fonts/NotoSansSC-Variable.ttf b/backend/open_webui/static/fonts/NotoSansSC-Variable.ttf deleted file mode 100644 index 4e0c62e278..0000000000 Binary files a/backend/open_webui/static/fonts/NotoSansSC-Variable.ttf and /dev/null differ diff --git a/backend/open_webui/utils/oauth.py b/backend/open_webui/utils/oauth.py index 96505a6be0..81cdec4267 100644 --- a/backend/open_webui/utils/oauth.py +++ b/backend/open_webui/utils/oauth.py @@ -1263,7 +1263,7 @@ class OAuthClientManager: if token and not token.get('access_token'): error_desc = token.get('error_description', token.get('error', 'Unknown error')) error_message = f'Token exchange failed: {error_desc}' - log.error(f'Invalid token response for client_id {client_id}: {token}') + log.error('Invalid token response for client_id %s: %s', client_id, error_desc) token = None if token: @@ -1917,7 +1917,7 @@ class OAuthManager: if provider == 'feishu' and isinstance(user_data, dict) and 'data' in user_data: user_data = user_data['data'] if not user_data: - log.warning(f'OAuth callback failed, user data is missing: {token}') + log.warning('OAuth callback failed for provider %s, user data is missing', provider) raise HTTPException(400, detail=ERROR_MESSAGES.INVALID_CRED) # Extract the "sub" claim, using custom claim if configured diff --git a/backend/requirements-min.txt b/backend/requirements-min.txt deleted file mode 100644 index a7a19a2257..0000000000 --- a/backend/requirements-min.txt +++ /dev/null @@ -1,59 +0,0 @@ -# Minimal requirements for backend to run -# WIP: use this as a reference to build a minimal docker image - -fastapi==0.136.3 -uvicorn[standard]==0.51.0 -pydantic==2.13.4 -python-multipart==0.0.32 -itsdangerous==2.2.0 - -python-socketio==5.16.2 -orjson==3.11.9 -cryptography -bcrypt==5.0.0 -argon2-cffi==25.1.0 -PyJWT[crypto]==2.13.0 -authlib==1.7.2 -joserfc==1.7.4 - -requests==2.34.2 -aiohttp==3.13.5 # do not update to 3.13.3 - broken -async-timeout -aiocache -aiofiles -starlette-compress==1.7.1 -Brotli==1.2.0 -brotlicffi==1.2.0.1 -httpx[socks,http2,zstd,cli,brotli]==0.28.1 -starsessions[redis]==2.2.1 - -sqlalchemy==2.0.50 -aiosqlite==0.22.1 -psycopg[binary]==3.3.4 -alembic==1.18.4 - -pycrdt==0.13.1 -redis -hiredis - -APScheduler==3.11.2 -RestrictedPython==8.2 - -loguru==0.7.3 -asgiref==3.11.1 - -mcp==1.27.2 -openai - -langchain==1.2.10 -langchain-community==0.4.2 -langchain-classic==1.0.7 -langchain-text-splitters==1.1.2 - -fake-useragent==2.2.0 - -chromadb==1.5.9 -black==26.5.1 -pydub -chardet==7.4.3 -beautifulsoup4 diff --git a/backend/requirements-slim.txt b/backend/requirements-slim.txt new file mode 100644 index 0000000000..e231eb0699 --- /dev/null +++ b/backend/requirements-slim.txt @@ -0,0 +1,122 @@ +# External-services image. Keep shared package pins aligned with requirements.txt. +# NumPy is used for cosine scoring; pandas is required by the remote Milvus client. +numpy==2.4.6 +typer==0.25.1 +fastapi==0.136.3 +uvicorn[standard]==0.51.0 +pydantic==2.13.4 +python-multipart==0.0.32 +itsdangerous==2.2.0 + +python-socketio==5.16.2 +orjson==3.11.9 +cryptography==48.0.0 +bcrypt==5.0.0 +argon2-cffi==25.1.0 +PyJWT[crypto]==2.13.0 +authlib==1.7.2 +joserfc==1.7.4 + +requests==2.34.2 +regex==2026.5.9 # supports a per-search timeout, which `re` does not +aiohttp==3.13.5 # do not update to 3.13.3 - broken +aiodns==3.6.1 # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER +aiocache==0.12.3 +aiofiles==25.1.0 +starlette-compress==1.7.1 +Brotli==1.2.0 +httpx[socks,http2,zstd,cli,brotli]==0.28.1 +starsessions[redis]==2.2.1 +python-mimeparse==2.0.0 + +sqlalchemy[asyncio]==2.0.50 +aiosqlite==0.22.1 +psycopg[binary]==3.3.4 +alembic==1.18.4 + +pycrdt==0.13.1 +redis==8.0.1 +hiredis==3.4.0 + +APScheduler==3.11.2 +pytz==2026.2 + +loguru==0.7.3 +asgiref==3.11.1 + +# AI libraries +tiktoken==0.13.0 +mcp==1.27.2 + +openai==2.29.0 + +langchain-community==0.4.2 +langchain-classic==1.0.7 +langchain-text-splitters==1.1.2 + +fake-useragent==2.2.0 +chromadb-client==1.5.9 +weaviate-client==4.20.3 +opensearch-py==3.2.0 + +ftfy==6.3.1 +chardet==7.4.3 +fpdf2==2.8.7 + +Markdown==3.10.2 +beautifulsoup4==4.14.3 +lxml==6.1.1 +pandas==3.0.3 +validators==0.35.0 +psutil==7.2.2 + +pillow==12.2.0 +rank-bm25==0.2.2 + +black==26.5.1 +youtube-transcript-api==1.2.4 + +ddgs==9.14.4 + +azure-ai-documentintelligence==1.0.2 +azure-identity==1.25.3 +azure-storage-blob==12.29.0 +azure-search-documents==12.0.0 + +## Google Cloud storage + +googleapis-common-protos==1.75.0 +google-cloud-storage==3.9.0 + +## Databases +psycopg2-binary==2.9.12 +pgvector==0.4.2 + +PyMySQL==1.2.0 +boto3==1.42.62 +# mariadb==1.1.14 should be added if you want to support MariaDB +# valkey-glide-sync==2.3.1 # optional: install manually if VECTOR_DB=valkey + +pymilvus==2.6.14 +qdrant-client==1.18.0 +playwright==1.60.0 # Caution: version must match docker-compose.playwright.yaml - Update the docker-compose.yaml if necessary +elasticsearch==9.4.1 +pinecone==6.0.2 +oracledb==3.4.2 + +## LDAP +ldap3==2.9.1 + +## Trace +opentelemetry-api==1.42.1 +opentelemetry-sdk==1.42.1 +opentelemetry-exporter-otlp==1.42.1 +opentelemetry-instrumentation==0.63b1 +opentelemetry-instrumentation-fastapi==0.63b1 +opentelemetry-instrumentation-sqlalchemy==0.63b1 +opentelemetry-instrumentation-redis==0.63b1 +opentelemetry-instrumentation-requests==0.63b1 +opentelemetry-instrumentation-logging==0.63b1 +opentelemetry-instrumentation-httpx==0.63b1 +opentelemetry-instrumentation-aiohttp-client==0.63b1 +opentelemetry-instrumentation-system-metrics==0.63b1 diff --git a/backend/requirements.txt b/backend/requirements.txt index a8b091c37b..306885cec6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -17,7 +17,6 @@ requests==2.34.2 regex==2026.5.9 # supports a per-search timeout, which `re` does not aiohttp==3.13.5 # do not update to 3.13.3 - broken aiodns==3.6.1 # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER -async-timeout==5.0.1 aiocache==0.12.3 aiofiles==25.1.0 starlette-compress==1.7.1 @@ -36,8 +35,6 @@ pycrdt==0.13.1 redis==8.0.1 hiredis==3.4.0 -APScheduler==3.11.2 -RestrictedPython==8.2 pytz==2026.2 loguru==0.7.3 @@ -51,7 +48,6 @@ openai==2.29.0 anthropic==0.86.0 google-genai==1.66.0 -langchain==1.2.10 langchain-community==0.4.2 langchain-classic==1.0.7 langchain-text-splitters==1.1.2 @@ -71,14 +67,12 @@ ftfy==6.3.1 chardet==7.4.3 pypdf==6.7.5 fpdf2==2.8.7 -pymdown-extensions==10.21.3 docx2txt==0.9 python-docx==1.2.0 python-pptx==1.0.2 msoffcrypto-tool==6.0.0 unstructured==0.22.31 -nltk==3.9.4 Markdown==3.10.2 beautifulsoup4==4.14.3 lxml==6.1.1 @@ -102,7 +96,6 @@ faster-whisper==1.2.1 black==26.5.1 youtube-transcript-api==1.2.4 -pytube==15.0.0 pydub==0.25.1 ddgs==9.14.4 @@ -112,16 +105,10 @@ azure-identity==1.25.3 azure-storage-blob==12.29.0 azure-search-documents==12.0.0 -## Google Drive -google-api-python-client==2.197.0 -google-auth-httplib2==0.4.0 -google-auth-oauthlib==1.4.0 - googleapis-common-protos==1.75.0 google-cloud-storage==3.9.0 ## Databases -pymongo==4.17.0 psycopg2-binary==2.9.12 pgvector==0.4.2 diff --git a/backend/start.sh b/backend/start.sh index 634f4bcc1b..ef7cb5af10 100755 --- a/backend/start.sh +++ b/backend/start.sh @@ -11,20 +11,19 @@ set -euo pipefail # expansion. The two can't be combined inline (`${VAR:-default,,}` makes # the default literal `,,`), so we normalise once up front and the simple # `${VAR,,}` form stays safe under `set -u` everywhere else. -: "${WEB_LOADER_ENGINE:=}" "${USE_OLLAMA_DOCKER:=}" "${USE_CUDA_DOCKER:=}" +: "${USE_SLIM_DOCKER:=}" "${WEB_LOADER_ENGINE:=}" "${USE_OLLAMA_DOCKER:=}" "${USE_CUDA_DOCKER:=}" SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cd "$SCRIPT_DIR" || exit 1 # ── Playwright browser installation (if configured) ────────────────────────── -if [[ "${WEB_LOADER_ENGINE,,}" == "playwright" ]]; then +if [[ "${USE_SLIM_DOCKER,,}" != "true" && "${WEB_LOADER_ENGINE,,}" == "playwright" ]]; then if [[ -z "${PLAYWRIGHT_WS_URL:-}" ]]; then echo "Installing Playwright Chromium browser..." playwright install chromium playwright install-deps chromium fi - python -c "import nltk; nltk.download('punkt_tab')" fi # ── Secret key setup ───────────────────────────────────────────────────────── diff --git a/backend/start_windows.bat b/backend/start_windows.bat index 4a57d34694..22aaaafc76 100644 --- a/backend/start_windows.bat +++ b/backend/start_windows.bat @@ -13,8 +13,6 @@ IF /I "%WEB_LOADER_ENGINE%" == "playwright" ( playwright install chromium playwright install-deps chromium ) - - python -c "import nltk; nltk.download('punkt_tab')" ) SET "KEY_FILE=.webui_secret_key" diff --git a/pyproject.toml b/pyproject.toml index 5b6b3e0749..647128b4f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "requests==2.34.2", "aiohttp==3.13.5", # do not update to 3.13.3 - broken "aiodns==3.6.1", # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER - "async-timeout==5.0.1", "aiocache==0.12.3", "aiofiles==25.1.0", "starlette-compress==1.7.1", @@ -45,8 +44,6 @@ dependencies = [ # "valkey-glide-sync==2.3.1", # optional: install manually if VECTOR_DB=valkey "pytz==2026.2", - "APScheduler==3.11.2", - "RestrictedPython==8.2", "loguru==0.7.3", "asgiref==3.11.1", @@ -59,7 +56,6 @@ dependencies = [ "anthropic==0.86.0", "google-genai==1.66.0", - "langchain==1.2.10", "langchain-community==0.4.2", "langchain-classic==1.0.7", "langchain-text-splitters==1.1.2", @@ -80,12 +76,10 @@ dependencies = [ "chardet==7.4.3", "pypdf==6.7.5", "fpdf2==2.8.7", - "pymdown-extensions==10.21.3", "docx2txt==0.9", "python-docx==1.2.0", "python-pptx==1.0.2", "msoffcrypto-tool==6.0.0", - "nltk==3.9.4", "Markdown==3.10.2", "beautifulsoup4==4.14.3", "lxml==6.1.1", @@ -110,15 +104,10 @@ dependencies = [ "black==26.5.1", "youtube-transcript-api==1.2.4", - "pytube==15.0.0", "pydub==0.25.1", "ddgs==9.14.4", - "google-api-python-client==2.197.0", - "google-auth-httplib2==0.4.0", - "google-auth-oauthlib==1.4.0", - "googleapis-common-protos==1.75.0", "google-cloud-storage==3.9.0", @@ -153,7 +142,6 @@ unstructured = [ ] all = [ - "pymongo==4.17.0", "psycopg2-binary==2.9.12", "pgvector==0.4.2", "playwright==1.60.0", # Caution: version must match docker-compose.playwright.yaml - Update the docker-compose.yaml if necessary diff --git a/src/lib/components/admin/Settings/Audio.svelte b/src/lib/components/admin/Settings/Audio.svelte index e09e4aee3d..ae3d2b07ef 100644 --- a/src/lib/components/admin/Settings/Audio.svelte +++ b/src/lib/components/admin/Settings/Audio.svelte @@ -245,6 +245,13 @@ }} >

{$i18n.t('Audio')}

+ {#if $config?.features?.slim === true} +

+ {$i18n.t( + 'Slim requires external speech providers. Local speech models and audio conversion are unavailable.' + )} +

+ {/if}
@@ -253,7 +260,9 @@ description={$i18n.t('Choose the transcription provider used for audio input.')} > - + @@ -500,7 +509,9 @@ }} > - + diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 2c49b05520..c24e3a5459 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -1,4 +1,5 @@
{#if history.messages[messageId]} {#if history.messages[messageId].role === 'user'} @@ -154,7 +158,7 @@ /* Browser-native virtualization: skip rendering of off-screen messages without destroying their component trees. Replaces the JS-based culling that caused catastrophic mount/destroy thrashing. */ - .message-listitem { + .message-virtualized { content-visibility: auto; contain-intrinsic-size: auto 150px; } diff --git a/src/lib/components/common/DocxPreview.svelte b/src/lib/components/common/DocxPreview.svelte index 011420c171..eb49021be8 100644 --- a/src/lib/components/common/DocxPreview.svelte +++ b/src/lib/components/common/DocxPreview.svelte @@ -28,6 +28,8 @@ let zoomLevel = 1; let resizeObserver: ResizeObserver | null = null; + const SAFE_LINK_SCHEMES = ['http:', 'https:', 'mailto:', 'tel:']; + $: docxScale = Math.max(0.25, fitScale * zoomLevel); const clearPreview = () => { @@ -96,6 +98,24 @@ (pages[page - 1] as HTMLElement | undefined)?.scrollIntoView({ block: 'start' }); }; + const isSafeLinkTarget = (href: string) => { + if (!href.trim()) return false; + + try { + return SAFE_LINK_SCHEMES.includes(new URL(href, document.baseURI).protocol); + } catch { + return false; + } + }; + + const stripUnsafeLinkTargets = () => { + containerEl.querySelectorAll('a[href]').forEach((link) => { + if (!isSafeLinkTarget(link.getAttribute('href') ?? '')) { + link.removeAttribute('href'); + } + }); + }; + const renderDocx = async (arrayBuffer: ArrayBuffer | null) => { const currentRender = ++renderId; clearPreview(); @@ -116,12 +136,15 @@ className: 'docx', ignoreLastRenderedPageBreak: false, inWrapper: true, + // the renderer would otherwise inline a document-supplied HTML part into a same-origin frame + renderAltChunks: false, renderEndnotes: true, renderFooters: true, renderFootnotes: true, renderHeaders: true, useBase64URL: true }); + stripUnsafeLinkTargets(); await tick(); updateFitScale(); await scrollToTargetPage(); diff --git a/src/lib/components/layout/Navbar/Menu.svelte b/src/lib/components/layout/Navbar/Menu.svelte index f4f03052d2..d862580ab5 100644 --- a/src/lib/components/layout/Navbar/Menu.svelte +++ b/src/lib/components/layout/Navbar/Menu.svelte @@ -105,7 +105,7 @@ document.body.appendChild(clonedElement); // Override content-visibility so html2canvas can capture all messages - clonedElement.querySelectorAll('.message-listitem').forEach((el) => { + clonedElement.querySelectorAll('.message-virtualized').forEach((el) => { el.style.contentVisibility = 'visible'; }); diff --git a/src/lib/components/layout/Sidebar/ChatMenu.svelte b/src/lib/components/layout/Sidebar/ChatMenu.svelte index c591f46e07..17e8cc4403 100644 --- a/src/lib/components/layout/Sidebar/ChatMenu.svelte +++ b/src/lib/components/layout/Sidebar/ChatMenu.svelte @@ -120,7 +120,7 @@ document.body.appendChild(clonedElement); // Override content-visibility so html2canvas can capture all messages - clonedElement.querySelectorAll('.message-listitem').forEach((el) => { + clonedElement.querySelectorAll('.message-virtualized').forEach((el) => { el.style.contentVisibility = 'visible'; }); diff --git a/src/lib/components/notes/NoteEditor.svelte b/src/lib/components/notes/NoteEditor.svelte index 45296173fa..d3478eff18 100644 --- a/src/lib/components/notes/NoteEditor.svelte +++ b/src/lib/components/notes/NoteEditor.svelte @@ -1263,7 +1263,7 @@ ${content}
- +
{#if editor}
diff --git a/src/lib/stores/index.ts b/src/lib/stores/index.ts index 5d5b06ccd4..270451dbf6 100644 --- a/src/lib/stores/index.ts +++ b/src/lib/stores/index.ts @@ -336,6 +336,7 @@ type Config = { default_prompt_suggestions: PromptSuggestion[]; default_prompt_suggestions_i18n?: Record; features: { + slim?: boolean; auth: boolean; auth_trusted_header: boolean; enable_api_keys: boolean; diff --git a/uv.lock b/uv.lock index cff82b6167..ffc5934945 100644 --- a/uv.lock +++ b/uv.lock @@ -210,18 +210,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" }, ] -[[package]] -name = "apscheduler" -version = "3.11.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzlocal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/07/12/3e4389e5920b4c1763390c6d371162f3784f86f85cd6d6c1bfe68eef14e2/apscheduler-3.11.2.tar.gz", hash = "sha256:2a9966b052ec805f020c8c4c3ae6e6a06e24b1bf19f2e11d91d8cca0473eef41", size = 108683, upload-time = "2025-12-22T00:39:34.884Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/64/2e54428beba8d9992aa478bb8f6de9e4ecaa5f8f513bcfd567ed7fb0262d/apscheduler-3.11.2-py3-none-any.whl", hash = "sha256:ce005177f741409db4e4dd40a7431b76feb856b9dd69d57e0da49d6715bfd26d", size = 64439, upload-time = "2025-12-22T00:39:33.303Z" }, -] - [[package]] name = "argon2-cffi" version = "25.1.0" @@ -1108,15 +1096,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] -[[package]] -name = "dnspython" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, -] - [[package]] name = "docstring-parser" version = "0.18.0" @@ -1439,22 +1418,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/40/9bdbb60b03a332bd45acb8703da08bbc27d991d35286b62e42acc86d243a/google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab", size = 173102, upload-time = "2026-06-03T14:51:26.729Z" }, ] -[[package]] -name = "google-api-python-client" -version = "2.197.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-auth-httplib2" }, - { name = "httplib2" }, - { name = "uritemplate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/09/081d66357118bd260f8f182cb1b2dd5bd32ca88e3714d7c93896cab946fc/google_api_python_client-2.197.0.tar.gz", hash = "sha256:32e03977eda4a66eafc6ae58dc9ec46426b6025636d5ef019c5703013eddd4e5", size = 14707398, upload-time = "2026-05-28T20:23:12.498Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/e5/e9cc221fd75230974d4ef45eb72d2261feca3c110d5554215d516bfe6534/google_api_python_client-2.197.0-py3-none-any.whl", hash = "sha256:0f8b89aa75768161dd4f5092d6bcb386c13236b32e0d9a938c02f71342094d14", size = 15287302, upload-time = "2026-05-28T20:23:09.683Z" }, -] - [[package]] name = "google-auth" version = "2.55.0" @@ -1473,32 +1436,6 @@ requests = [ { name = "requests" }, ] -[[package]] -name = "google-auth-httplib2" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "httplib2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/b3/f192c8bc7e41e0ebdbd95afcae4783417a34b6a6af62d22daf22c3fd38fc/google_auth_httplib2-0.4.0.tar.gz", hash = "sha256:d5b030a204b7a4b4d553ba9ca701b62481ee2b74419325580be70f7d85ffed35", size = 11161, upload-time = "2026-05-07T08:03:46.878Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/be/954c35a62b9e31de66b0a43c225c9b6bb9e0f98d6b1dc110a2308e3644f5/google_auth_httplib2-0.4.0-py3-none-any.whl", hash = "sha256:8e55cfafa3358cba85f6cad4a886138e88e158d71e7e5c9ee5936a5c1507fb91", size = 9529, upload-time = "2026-05-07T08:02:12.375Z" }, -] - -[[package]] -name = "google-auth-oauthlib" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "requests-oauthlib" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/18/90c7fac516e63cf2058166fce0c88c353647c677b51cc036c09c49bb5cbb/google_auth_oauthlib-1.4.0.tar.gz", hash = "sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2", size = 21675, upload-time = "2026-05-07T08:03:47.816Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/d3/d7dff0d58a9e9244b48044bfb6a898bfcc8ecc42e0031d1bebc695344725/google_auth_oauthlib-1.4.0-py3-none-any.whl", hash = "sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070", size = 19261, upload-time = "2026-05-07T08:02:13.798Z" }, -] - [[package]] name = "google-cloud-core" version = "2.6.0" @@ -1762,18 +1699,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] -[[package]] -name = "httplib2" -version = "0.31.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyparsing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800, upload-time = "2026-01-23T11:04:44.165Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099, upload-time = "2026-01-23T11:04:42.78Z" }, -] - [[package]] name = "httptools" version = "0.8.0" @@ -2079,20 +2004,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/5c160dbdef7123f8cc97fd8ece7e0198627a426a2a49614845e9086feb8d/kubernetes-36.0.2-py2.py3-none-any.whl", hash = "sha256:faf9b5241b58de0c4a5069f2a0ffc8ac06fece7215156cd3d3ba081a78a858b6", size = 4617568, upload-time = "2026-06-01T18:20:28.737Z" }, ] -[[package]] -name = "langchain" -version = "1.2.10" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "langchain-core" }, - { name = "langgraph" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/16/22/a4d4ac98fc2e393537130bbfba0d71a8113e6f884d96f935923e247397fe/langchain-1.2.10.tar.gz", hash = "sha256:bdcd7218d9c79a413cf15e106e4eb94408ac0963df9333ccd095b9ed43bf3be7", size = 570071, upload-time = "2026-02-10T14:56:49.74Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/06/c3394327f815fade875724c0f6cff529777c96a1e17fea066deb997f8cf5/langchain-1.2.10-py3-none-any.whl", hash = "sha256:e07a377204451fffaed88276b8193e894893b1003e25c5bca6539288ccca3698", size = 111738, upload-time = "2026-02-10T14:56:47.985Z" }, -] - [[package]] name = "langchain-classic" version = "1.0.7" @@ -2186,62 +2097,6 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0", size = 981474, upload-time = "2021-05-07T07:54:13.562Z" } -[[package]] -name = "langgraph" -version = "1.0.10" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "langchain-core" }, - { name = "langgraph-checkpoint" }, - { name = "langgraph-prebuilt" }, - { name = "langgraph-sdk" }, - { name = "pydantic" }, - { name = "xxhash" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/92/14df6fefba28c10caf1cb05aa5b8c7bf005838fe32a86d903b6c7cc4018d/langgraph-1.0.10.tar.gz", hash = "sha256:73bd10ee14a8020f31ef07e9cd4c1a70c35cc07b9c2b9cd637509a10d9d51e29", size = 511644, upload-time = "2026-02-27T21:04:38.743Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/60/260e0c04620a37ba8916b712766c341cc5fc685dabc6948c899494bbc2ae/langgraph-1.0.10-py3-none-any.whl", hash = "sha256:7c298bef4f6ea292fcf9824d6088fe41a6727e2904ad6066f240c4095af12247", size = 160920, upload-time = "2026-02-27T21:04:35.932Z" }, -] - -[[package]] -name = "langgraph-checkpoint" -version = "4.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "langchain-core" }, - { name = "ormsgpack" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/47/886af6f886f0bff2273164a45f008694e48a96ff3cd25ff0228f2aa9480e/langgraph_checkpoint-4.1.1.tar.gz", hash = "sha256:6c2bdb530c91f91d7d9c1bd100925d0fc4f498d418c17f3587d1526279482a25", size = 184020, upload-time = "2026-05-22T16:57:38.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/b4/71425e3e38be92611300b9cc5e46a5bf98ab23f5ea8a75b73d02a2f1413c/langgraph_checkpoint-4.1.1-py3-none-any.whl", hash = "sha256:25d29144b082827218e7bc3f1e9b0566a4bb007895cd6cc26f66a8428739f56e", size = 56212, upload-time = "2026-05-22T16:57:37.203Z" }, -] - -[[package]] -name = "langgraph-prebuilt" -version = "1.0.13" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "langchain-core" }, - { name = "langgraph-checkpoint" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/a4/f8ac75fa7c503103f0cf7680944e28bbaaef74c19a8d163d7346869cc369/langgraph_prebuilt-1.0.13.tar.gz", hash = "sha256:ad219782a80e1718e7e7794de49e0ae307111d45cbcffab9a52725a66a609456", size = 172913, upload-time = "2026-04-30T01:48:15.742Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/ef/5ada0bef4013ef5ae53a0ca1de5736517f1076a54d313f156ca545ec65d5/langgraph_prebuilt-1.0.13-py3-none-any.whl", hash = "sha256:7055e9fad41fbd3593800aed0aea0a6e974b17f33ed51b80d3d3a031212dd7c0", size = 37214, upload-time = "2026-04-30T01:48:14.507Z" }, -] - -[[package]] -name = "langgraph-sdk" -version = "0.3.15" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "orjson" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/af/cdd4d6f3c05b3c1112ed3f12ef830faf15951b21d22cbc622a4becbbe25c/langgraph_sdk-0.3.15.tar.gz", hash = "sha256:29e805003d2c6e296823dd71992610976fd0428cefaa8b3304fd91f2247037de", size = 201924, upload-time = "2026-05-22T16:54:27.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/a5/0196d9c05749c25bc198e4909d68c998bc3120297e14944921baf2f4c384/langgraph_sdk-0.3.15-py3-none-any.whl", hash = "sha256:3838773acf7456d158165385d49f48f1e856f28b56ccd99ea139a8f27004815d", size = 98166, upload-time = "2026-05-22T16:54:26.013Z" }, -] - [[package]] name = "langsmith" version = "0.8.18" @@ -2690,21 +2545,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, ] -[[package]] -name = "nltk" -version = "3.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, -] - [[package]] name = "numba" version = "0.65.1" @@ -2980,10 +2820,8 @@ dependencies = [ { name = "aiosqlite" }, { name = "alembic" }, { name = "anthropic" }, - { name = "apscheduler" }, { name = "argon2-cffi" }, { name = "asgiref" }, - { name = "async-timeout" }, { name = "authlib" }, { name = "azure-ai-documentintelligence" }, { name = "azure-identity" }, @@ -3005,9 +2843,6 @@ dependencies = [ { name = "faster-whisper" }, { name = "fpdf2" }, { name = "ftfy" }, - { name = "google-api-python-client" }, - { name = "google-auth-httplib2" }, - { name = "google-auth-oauthlib" }, { name = "google-cloud-storage" }, { name = "google-genai" }, { name = "googleapis-common-protos" }, @@ -3015,7 +2850,6 @@ dependencies = [ { name = "httpx", extra = ["brotli", "cli", "http2", "socks", "zstd"] }, { name = "itsdangerous" }, { name = "joserfc" }, - { name = "langchain" }, { name = "langchain-classic" }, { name = "langchain-community" }, { name = "langchain-text-splitters" }, @@ -3025,7 +2859,6 @@ dependencies = [ { name = "markdown" }, { name = "mcp" }, { name = "msoffcrypto-tool" }, - { name = "nltk" }, { name = "onnxruntime" }, { name = "openai" }, { name = "opencv-python-headless" }, @@ -3041,7 +2874,6 @@ dependencies = [ { name = "pydantic" }, { name = "pydub" }, { name = "pyjwt", extra = ["crypto"] }, - { name = "pymdown-extensions" }, { name = "pymysql" }, { name = "pypandoc" }, { name = "pypdf" }, @@ -3050,7 +2882,6 @@ dependencies = [ { name = "python-multipart" }, { name = "python-pptx" }, { name = "python-socketio" }, - { name = "pytube" }, { name = "pytz" }, { name = "pyxlsb" }, { name = "rank-bm25" }, @@ -3058,7 +2889,6 @@ dependencies = [ { name = "redis" }, { name = "regex" }, { name = "requests" }, - { name = "restrictedpython" }, { name = "sentence-transformers" }, { name = "sentencepiece" }, { name = "soundfile" }, @@ -3084,7 +2914,6 @@ all = [ { name = "playwright" }, { name = "psycopg2-binary" }, { name = "pymilvus" }, - { name = "pymongo" }, { name = "qdrant-client" }, { name = "unstructured" }, { name = "weaviate-client" }, @@ -3116,10 +2945,8 @@ requires-dist = [ { name = "aiosqlite", specifier = "==0.22.1" }, { name = "alembic", specifier = "==1.18.4" }, { name = "anthropic", specifier = "==0.86.0" }, - { name = "apscheduler", specifier = "==3.11.2" }, { name = "argon2-cffi", specifier = "==25.1.0" }, { name = "asgiref", specifier = "==3.11.1" }, - { name = "async-timeout", specifier = "==5.0.1" }, { name = "authlib", specifier = "==1.7.2" }, { name = "azure-ai-documentintelligence", specifier = "==1.0.2" }, { name = "azure-identity", specifier = "==1.25.3" }, @@ -3144,9 +2971,6 @@ requires-dist = [ { name = "faster-whisper", specifier = "==1.2.1" }, { name = "fpdf2", specifier = "==2.8.7" }, { name = "ftfy", specifier = "==6.3.1" }, - { name = "google-api-python-client", specifier = "==2.197.0" }, - { name = "google-auth-httplib2", specifier = "==0.4.0" }, - { name = "google-auth-oauthlib", specifier = "==1.4.0" }, { name = "google-cloud-storage", specifier = "==3.9.0" }, { name = "google-genai", specifier = "==1.66.0" }, { name = "googleapis-common-protos", specifier = "==1.75.0" }, @@ -3154,7 +2978,6 @@ requires-dist = [ { name = "httpx", extras = ["brotli", "cli", "http2", "socks", "zstd"], specifier = "==0.28.1" }, { name = "itsdangerous", specifier = "==2.2.0" }, { name = "joserfc", specifier = "==1.7.4" }, - { name = "langchain", specifier = "==1.2.10" }, { name = "langchain-classic", specifier = "==1.0.7" }, { name = "langchain-community", specifier = "==0.4.2" }, { name = "langchain-text-splitters", specifier = "==1.1.2" }, @@ -3165,7 +2988,6 @@ requires-dist = [ { name = "markdown", specifier = "==3.10.2" }, { name = "mcp", specifier = "==1.27.2" }, { name = "msoffcrypto-tool", specifier = "==6.0.0" }, - { name = "nltk", specifier = "==3.9.4" }, { name = "onnxruntime", specifier = "==1.26.0" }, { name = "openai", specifier = "==2.29.0" }, { name = "opencv-python-headless", specifier = "==4.13.0.92" }, @@ -3188,9 +3010,7 @@ requires-dist = [ { name = "pydantic", specifier = "==2.13.4" }, { name = "pydub", specifier = "==0.25.1" }, { name = "pyjwt", extras = ["crypto"], specifier = "==2.13.0" }, - { name = "pymdown-extensions", specifier = "==10.21.3" }, { name = "pymilvus", marker = "extra == 'all'", specifier = "==2.6.14" }, - { name = "pymongo", marker = "extra == 'all'", specifier = "==4.17.0" }, { name = "pymysql", specifier = "==1.2.0" }, { name = "pypandoc", specifier = "==1.17" }, { name = "pypdf", specifier = "==6.7.5" }, @@ -3199,7 +3019,6 @@ requires-dist = [ { name = "python-multipart", specifier = "==0.0.32" }, { name = "python-pptx", specifier = "==1.0.2" }, { name = "python-socketio", specifier = "==5.16.2" }, - { name = "pytube", specifier = "==15.0.0" }, { name = "pytz", specifier = "==2026.2" }, { name = "pyxlsb", specifier = "==1.0.10" }, { name = "qdrant-client", marker = "extra == 'all'", specifier = "==1.18.0" }, @@ -3208,7 +3027,6 @@ requires-dist = [ { name = "redis", specifier = "==8.0.1" }, { name = "regex", specifier = "==2026.5.9" }, { name = "requests", specifier = "==2.34.2" }, - { name = "restrictedpython", specifier = "==8.2" }, { name = "sentence-transformers", specifier = "==5.5.1" }, { name = "sentencepiece", specifier = "==0.2.1" }, { name = "soundfile", specifier = "==0.13.1" }, @@ -3470,32 +3288,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, ] -[[package]] -name = "ormsgpack" -version = "1.12.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/0c/f1761e21486942ab9bb6feaebc610fa074f7c5e496e6962dea5873348077/ormsgpack-1.12.2.tar.gz", hash = "sha256:944a2233640273bee67521795a73cf1e959538e0dfb7ac635505010455e53b33", size = 39031, upload-time = "2026-01-18T20:55:28.023Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/08/8b68f24b18e69d92238aa8f258218e6dfeacf4381d9d07ab8df303f524a9/ormsgpack-1.12.2-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bd5f4bf04c37888e864f08e740c5a573c4017f6fd6e99fa944c5c935fabf2dd9", size = 378266, upload-time = "2026-01-18T20:55:59.876Z" }, - { url = "https://files.pythonhosted.org/packages/0d/24/29fc13044ecb7c153523ae0a1972269fcd613650d1fa1a9cec1044c6b666/ormsgpack-1.12.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34d5b28b3570e9fed9a5a76528fc7230c3c76333bc214798958e58e9b79cc18a", size = 203035, upload-time = "2026-01-18T20:55:30.59Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c2/00169fb25dd8f9213f5e8a549dfb73e4d592009ebc85fbbcd3e1dcac575b/ormsgpack-1.12.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3708693412c28f3538fb5a65da93787b6bbab3484f6bc6e935bfb77a62400ae5", size = 210539, upload-time = "2026-01-18T20:55:48.569Z" }, - { url = "https://files.pythonhosted.org/packages/1b/33/543627f323ff3c73091f51d6a20db28a1a33531af30873ea90c5ac95a9b5/ormsgpack-1.12.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43013a3f3e2e902e1d05e72c0f1aeb5bedbb8e09240b51e26792a3c89267e181", size = 212401, upload-time = "2026-01-18T20:56:10.101Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5d/f70e2c3da414f46186659d24745483757bcc9adccb481a6eb93e2b729301/ormsgpack-1.12.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7c8b1667a72cbba74f0ae7ecf3105a5e01304620ed14528b2cb4320679d2869b", size = 387082, upload-time = "2026-01-18T20:56:12.047Z" }, - { url = "https://files.pythonhosted.org/packages/c0/d6/06e8dc920c7903e051f30934d874d4afccc9bb1c09dcaf0bc03a7de4b343/ormsgpack-1.12.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:df6961442140193e517303d0b5d7bc2e20e69a879c2d774316125350c4a76b92", size = 482346, upload-time = "2026-01-18T20:56:05.152Z" }, - { url = "https://files.pythonhosted.org/packages/66/c4/f337ac0905eed9c393ef990c54565cd33644918e0a8031fe48c098c71dbf/ormsgpack-1.12.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c6a4c34ddef109647c769d69be65fa1de7a6022b02ad45546a69b3216573eb4a", size = 425181, upload-time = "2026-01-18T20:55:37.83Z" }, - { url = "https://files.pythonhosted.org/packages/78/29/6d5758fabef3babdf4bbbc453738cc7de9cd3334e4c38dd5737e27b85653/ormsgpack-1.12.2-cp311-cp311-win_amd64.whl", hash = "sha256:73670ed0375ecc303858e3613f407628dd1fca18fe6ac57b7b7ce66cc7bb006c", size = 117182, upload-time = "2026-01-18T20:55:31.472Z" }, - { url = "https://files.pythonhosted.org/packages/c4/57/17a15549233c37e7fd054c48fe9207492e06b026dbd872b826a0b5f833b6/ormsgpack-1.12.2-cp311-cp311-win_arm64.whl", hash = "sha256:c2be829954434e33601ae5da328cccce3266b098927ca7a30246a0baec2ce7bd", size = 111464, upload-time = "2026-01-18T20:55:38.811Z" }, - { url = "https://files.pythonhosted.org/packages/4c/36/16c4b1921c308a92cef3bf6663226ae283395aa0ff6e154f925c32e91ff5/ormsgpack-1.12.2-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7a29d09b64b9694b588ff2f80e9826bdceb3a2b91523c5beae1fab27d5c940e7", size = 378618, upload-time = "2026-01-18T20:55:50.835Z" }, - { url = "https://files.pythonhosted.org/packages/c0/68/468de634079615abf66ed13bb5c34ff71da237213f29294363beeeca5306/ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b39e629fd2e1c5b2f46f99778450b59454d1f901bc507963168985e79f09c5d", size = 203186, upload-time = "2026-01-18T20:56:11.163Z" }, - { url = "https://files.pythonhosted.org/packages/73/a9/d756e01961442688b7939bacd87ce13bfad7d26ce24f910f6028178b2cc8/ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:958dcb270d30a7cb633a45ee62b9444433fa571a752d2ca484efdac07480876e", size = 210738, upload-time = "2026-01-18T20:56:09.181Z" }, - { url = "https://files.pythonhosted.org/packages/7b/ba/795b1036888542c9113269a3f5690ab53dd2258c6fb17676ac4bd44fcf94/ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d379d72b6c5e964851c77cfedfb386e474adee4fd39791c2c5d9efb53505cc", size = 212569, upload-time = "2026-01-18T20:56:06.135Z" }, - { url = "https://files.pythonhosted.org/packages/6c/aa/bff73c57497b9e0cba8837c7e4bcab584b1a6dbc91a5dd5526784a5030c8/ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8463a3fc5f09832e67bdb0e2fda6d518dc4281b133166146a67f54c08496442e", size = 387166, upload-time = "2026-01-18T20:55:36.738Z" }, - { url = "https://files.pythonhosted.org/packages/d3/cf/f8283cba44bcb7b14f97b6274d449db276b3a86589bdb363169b51bc12de/ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:eddffb77eff0bad4e67547d67a130604e7e2dfbb7b0cde0796045be4090f35c6", size = 482498, upload-time = "2026-01-18T20:55:29.626Z" }, - { url = "https://files.pythonhosted.org/packages/05/be/71e37b852d723dfcbe952ad04178c030df60d6b78eba26bfd14c9a40575e/ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcd55e5f6ba0dbce624942adf9f152062135f991a0126064889f68eb850de0dd", size = 425518, upload-time = "2026-01-18T20:55:49.556Z" }, - { url = "https://files.pythonhosted.org/packages/7a/0c/9803aa883d18c7ef197213cd2cbf73ba76472a11fe100fb7dab2884edf48/ormsgpack-1.12.2-cp312-cp312-win_amd64.whl", hash = "sha256:d024b40828f1dde5654faebd0d824f9cc29ad46891f626272dd5bfd7af2333a4", size = 117462, upload-time = "2026-01-18T20:55:47.726Z" }, - { url = "https://files.pythonhosted.org/packages/c8/9e/029e898298b2cc662f10d7a15652a53e3b525b1e7f07e21fef8536a09bb8/ormsgpack-1.12.2-cp312-cp312-win_arm64.whl", hash = "sha256:da538c542bac7d1c8f3f2a937863dba36f013108ce63e55745941dda4b75dbb6", size = 111559, upload-time = "2026-01-18T20:55:54.273Z" }, -] - [[package]] name = "overrides" version = "7.7.0" @@ -4239,19 +4031,6 @@ crypto = [ { name = "cryptography" }, ] -[[package]] -name = "pymdown-extensions" -version = "10.21.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/26/d1015444da4d952a1ca487a236b522eb979766f0295a0bd0c5fc089989a9/pymdown_extensions-10.21.3.tar.gz", hash = "sha256:72cfcf55f07aea0d4af2c4f11dd4e52466ddfb1bb819673146398e0bd3a77354", size = 854140, upload-time = "2026-05-13T12:57:32.267Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/85/545a951eecc270fcd688288c600017e2050a1aacb56c711d208586d3e470/pymdown_extensions-10.21.3-py3-none-any.whl", hash = "sha256:d7a5d08014fc571e80ca21dd6f854e31f94c489800350564d55d15b3c41e76b6", size = 269002, upload-time = "2026-05-13T12:57:30.296Z" }, -] - [[package]] name = "pymilvus" version = "2.6.14" @@ -4271,37 +4050,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/0e/2807bf330edb41537caabf483fa4128e429b8fa5bcfa9bf67effcafa722e/pymilvus-2.6.14-py3-none-any.whl", hash = "sha256:01694978c2bbbf4bcc5ef29dc788e480245154a38eae32da7184959ea4b501fb", size = 340221, upload-time = "2026-05-15T14:11:45.052Z" }, ] -[[package]] -name = "pymongo" -version = "4.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dnspython" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ca/64/50be6fbac9c79fe2e4c17401a467da2d8764d82833d83cec325afe5cab32/pymongo-4.17.0.tar.gz", hash = "sha256:70ffa08ba641468cc068cf46c06b34f01a8ce3489f6411309fcb5ceabe6b2fc0", size = 2523370, upload-time = "2026-04-20T16:39:53.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/e2/336d86f221cf1b56b2ed9330d4a3b98f9f38f0b37829ae9a9184617d5419/pymongo-4.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4141e6c6a339789b2974efa00ecd9409101672d77a0e3ee2cc3839eedf8ec4df", size = 874668, upload-time = "2026-04-20T16:37:41.39Z" }, - { url = "https://files.pythonhosted.org/packages/34/8e/75d3c6c935d187ab59c61e9c15d9aab3f274b563eaf1706e8cae5f508dec/pymongo-4.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e68c76b84e0c132d9dbf9307f12ff8185702328187a87b9aca8c941303873433", size = 875294, upload-time = "2026-04-20T16:37:43.432Z" }, - { url = "https://files.pythonhosted.org/packages/5f/ec/62e855744489dbcd54fd778aae4d80fa4c4819e8fb228ca0cf6f21a03997/pymongo-4.17.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ba2195d4f386f839a52a23ea1cfd60ffaaba78a3d7841db51b7e433001139918", size = 1496233, upload-time = "2026-04-20T16:37:45.518Z" }, - { url = "https://files.pythonhosted.org/packages/82/e8/93e4e5e5ce8fdf8929dabeefe24aafa5ce046028eed0dfa8eeb936e72c49/pymongo-4.17.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8446ff4bfcb6ec2a2e50998c860986a1e992136f998b7f53e7a717fb8aa5a0b9", size = 1522927, upload-time = "2026-04-20T16:37:47.492Z" }, - { url = "https://files.pythonhosted.org/packages/f7/ca/425dc1d21e0f17bdea0072fc463f662f7fa06d2852af52975c9eced3c07c/pymongo-4.17.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2a0d5ac205728c86e0a02192f1aa5f865b0d7d51f8df6101c01a69a7fc620d72", size = 1583468, upload-time = "2026-04-20T16:37:49.221Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9d/f08b07eeffda1a43c1759f0fa625e88ae12360996eb56d42aad832fa7dff/pymongo-4.17.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:485c8a8eaa4c739f00a331fc73757898ee7c092c214a79e63866ff76aaf282ff", size = 1572787, upload-time = "2026-04-20T16:37:51.061Z" }, - { url = "https://files.pythonhosted.org/packages/e9/c2/6855a07aafa7b894929af23675b6fb9634800ce43122b76a62f6eeb8da2a/pymongo-4.17.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2dfcc795f5b9fedbe179a11fdf6051581479d196582a3fe819a92a00e9b9969", size = 1526184, upload-time = "2026-04-20T16:37:53.358Z" }, - { url = "https://files.pythonhosted.org/packages/4e/05/c952bac7db71c1942ea3559fcd308b49754cc5004b455935fb4000d1f37b/pymongo-4.17.0-cp311-cp311-win32.whl", hash = "sha256:c2292144505fb12156b981bd440f3dc994a883da06ac726c0c8692ccdbc1c510", size = 852621, upload-time = "2026-04-20T16:37:55.28Z" }, - { url = "https://files.pythonhosted.org/packages/11/c0/c04da9f4c0c6252404598f4e394b862a58a9e866822a70ae261c8a018fdf/pymongo-4.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:2e190827834fce70ecdf9d46796c6dbc0ce08ea87dc2ff5bc6f3f5579b605cb9", size = 867852, upload-time = "2026-04-20T16:37:57.233Z" }, - { url = "https://files.pythonhosted.org/packages/1d/b2/c7b4870fbeef471e947d3e014676f5910d02e0197074d692ebcf24ec049a/pymongo-4.17.0-cp311-cp311-win_arm64.whl", hash = "sha256:a8f9c40a09bb7d4b9fc8b1da65ecf6efa79bda5cb2756f39d9b6940fac1d19ae", size = 855019, upload-time = "2026-04-20T16:37:58.983Z" }, - { url = "https://files.pythonhosted.org/packages/98/90/60bcb508840135d5ee46b51b1a950f548338aa8145a8366dbe6639ae51ac/pymongo-4.17.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53ffa94b2340dbf6b055e09a0090618c60482c158ecfc9565642fc996bf0944", size = 930529, upload-time = "2026-04-20T16:38:00.936Z" }, - { url = "https://files.pythonhosted.org/packages/a6/e9/313840f1e52c6dfac47f704428cbfbce59956ebe7633bffc92b03f74f0ad/pymongo-4.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6fe0de9d0f6791abce3471230b32b4817bf89d27b1182b6a550e1ec0fa72aa9a", size = 930665, upload-time = "2026-04-20T16:38:02.915Z" }, - { url = "https://files.pythonhosted.org/packages/78/35/9d3565ea45b1606f635c1e2cd2563c28d66caafdc50f7ad7d979fcd1b363/pymongo-4.17.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e537e95514dae1aaa718f481ec03151a0f0394bcd05f1322896d8fc1330cb729", size = 1762369, upload-time = "2026-04-20T16:38:05.375Z" }, - { url = "https://files.pythonhosted.org/packages/95/ee/149b0d4b1a11c38bff6f14c23d5814c9b0843fd6dc38ad40596bdb1a62d2/pymongo-4.17.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37a8385c29881b43eab31f584100fa0eaddedd5607adf010147ba1810118be90", size = 1798044, upload-time = "2026-04-20T16:38:07.195Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d4/4cee4a7b8d8f6f0550ef6cd2fea42455c5ed619a220cb6ba4fb40d6a5bc8/pymongo-4.17.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3ee3d241ed77a4fc99ce3cff3b289c3ebce37f61fdd7349d3592c23b82c8784", size = 1878567, upload-time = "2026-04-20T16:38:09.121Z" }, - { url = "https://files.pythonhosted.org/packages/45/ef/7fe366c84952619ee2f69973566c214775e083dd4df465751912153e4b72/pymongo-4.17.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9eb5d63a3c518cb0804ed678f5e2b875af032d89a7cf57a57360322cf6a4d222", size = 1864881, upload-time = "2026-04-20T16:38:10.896Z" }, - { url = "https://files.pythonhosted.org/packages/2f/35/b577d82c6d1be7aee7ac7e249bc86f7847998345042e5f8360de238e177b/pymongo-4.17.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e97e03fa13327c87e3fdc5656acd01e71817f0c1dc3221cd8f30de136bf4ec3", size = 1800349, upload-time = "2026-04-20T16:38:13.589Z" }, - { url = "https://files.pythonhosted.org/packages/b8/69/dafcf04f66e130ddd91aeb92e7a692480eda46dcd04ec1dbe82c06619e10/pymongo-4.17.0-cp312-cp312-win32.whl", hash = "sha256:6877214bff5f06f6884a9fc8d9016a4a7a5f51f537f5c51ac3a576f93e7dfb32", size = 900518, upload-time = "2026-04-20T16:38:15.541Z" }, - { url = "https://files.pythonhosted.org/packages/11/35/5c9262a459f988b4eb2605f70815240b77a0d4131136c4326d18f1822b89/pymongo-4.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:9828485f72f63c7d802e0ec41f71906f633c2692621ab3af55ca990186b091b1", size = 920335, upload-time = "2026-04-20T16:38:17.665Z" }, - { url = "https://files.pythonhosted.org/packages/8d/da/e9c7265ee176faccf4e52c4797837e794d93569a1046f6b19a4acc36e5ad/pymongo-4.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:1195370a77baf003b59b10e91ecc4706297197f0dd9d29c840cc556dc08f7cee", size = 903289, upload-time = "2026-04-20T16:38:19.33Z" }, -] - [[package]] name = "pymysql" version = "1.2.0" @@ -4320,15 +4068,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/86/e2ffa604eacfbec3f430b1d850e7e04c4101eca1a5828f9ae54bf51dfba4/pypandoc-1.17-py3-none-any.whl", hash = "sha256:01fdbffa61edb9f8e82e8faad6954efcb7b6f8f0634aead4d89e322a00225a67", size = 23554, upload-time = "2026-03-14T22:38:46.007Z" }, ] -[[package]] -name = "pyparsing" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, -] - [[package]] name = "pypdf" version = "6.7.5" @@ -4557,15 +4296,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] -[[package]] -name = "pytube" -version = "15.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/e7/16fec46c8d255c4bbc4b185d89c91dc92cdb802836570d8004d0db169c91/pytube-15.0.0.tar.gz", hash = "sha256:076052efe76f390dfa24b1194ff821d4e86c17d41cb5562f3a276a8bcbfc9d1d", size = 67229, upload-time = "2023-05-07T19:39:01.903Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/64/bcf8632ed2b7a36bbf84a0544885ffa1d0b4bcf25cc0903dba66ec5fdad9/pytube-15.0.0-py3-none-any.whl", hash = "sha256:07b9904749e213485780d7eb606e5e5b8e4341aa4dccf699160876da00e12d78", size = 57594, upload-time = "2023-05-07T19:38:59.191Z" }, -] - [[package]] name = "pytz" version = "2026.2" @@ -4816,15 +4546,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] -[[package]] -name = "restrictedpython" -version = "8.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/97/04/7314639eab9edd57b5c8f6e158a558f5a5a65453c37a51fb8f9ecc1d112e/restrictedpython-8.2.tar.gz", hash = "sha256:b835c2ff87d71b76f6d8b129096cc5a1e893bf01839ab802ba47a4129f1bdfe4", size = 449323, upload-time = "2026-05-29T06:27:57.285Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/59/b2778bfafcc37fdfd87c02702bbfc5753c58c28c7eb82999b7b49ec6e0e7/restrictedpython-8.2-py3-none-any.whl", hash = "sha256:6ff69e2fe06674bfe88943f17f2ce8353a9bc743ee37ea7bec38ff8b4851afd1", size = 27176, upload-time = "2026-05-29T06:27:55.651Z" }, -] - [[package]] name = "rich" version = "13.9.4" @@ -5636,18 +5357,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] -[[package]] -name = "tzlocal" -version = "5.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/55/15e2340963d2bfedcc6042da3911438fd336f8ae96b65bdbe3a29766da0c/tzlocal-5.4.3.tar.gz", hash = "sha256:3a8c9bc18cf47e1dcde252ea0e6a72a6cde320a400b6ac6db1f1f8cccd553c00", size = 30873, upload-time = "2026-06-17T04:17:41.764Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/28/fc144409c71569e928585f8f3c629d80d1ca3ef40175e9222f01588f98c9/tzlocal-5.4.3-py3-none-any.whl", hash = "sha256:24ce97bb58e2a973f7640ec2553ab4e6f6d5a0d0d1aa9dc43bca21d89e1feb82", size = 18039, upload-time = "2026-06-17T04:17:40.027Z" }, -] - [[package]] name = "ujson" version = "5.13.0" @@ -5749,15 +5458,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/80/fbf02ec3c566a3e383a5649385096834a2a981832f1432c3a8797b29185a/unstructured_client-0.42.12-py3-none-any.whl", hash = "sha256:fe6f217066a0c308ba7213185524506dbfc3bb9d35df0ab79549291e9728a012", size = 220154, upload-time = "2026-03-25T20:24:20.288Z" }, ] -[[package]] -name = "uritemplate" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, -] - [[package]] name = "urllib3" version = "2.7.0"