scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-09-07 08:27:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

chore: drop nltk, unused at the pinned versions (#29725)

Browse source 
The main and CUDA Docker images get about 21 MB smaller (the nltk package, the punkt_tab data and its zip); slim images, which never downloaded the data, about 6 MB.

nltk was in the image for unstructured, which used it to tokenize documents. The Dockerfile download was added for airgapped containers failing on the missing punkt_tab data (#21150; the same request in #16260), the same lookup failed on first use in other setups (#17594, #4642), and the download in start.sh and start_windows.bat came with the Playwright web loader mode and sits in that branch.

unstructured 0.22.31, the pinned version, has no nltk references at all and tokenizes with spaCy, nothing else installed requires nltk outside transformers' testing and dev extras, and nothing in the backend imports it, so the pin and both downloads go together.

One user-visible consequence: a tool or function that imports nltk inside the container stops working unless it declares nltk in its frontmatter requirements. On an offline instance the package, and any nltk data such as punkt_tab, have to be installed into the image instead.

Part of #29721.
This commit is contained in:
Classic298 2026-09-06 22:39:05 +02:00 • committed by GitHub
parent 55a6198e26
commit ca9ec06c7e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 0 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -148,7 +148,6 @@ RUN set -e; \
python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ.get('AUXILIARY_EMBEDDING_MODEL', 'TaylorAI/bge-micro-v2'), device='cpu')"; \
python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \
python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \
python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')"; \
else \
pip3 install 'torch<=2.9.1' torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir; \
uv pip install --system -r requirements.txt --no-cache-dir; \
@ -157,7 +156,6 @@ RUN set -e; \
python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ.get('AUXILIARY_EMBEDDING_MODEL', 'TaylorAI/bge-micro-v2'), device='cpu')"; \
python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \
python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \
python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')"; \
fi; \
fi; \
mkdir -p /app/backend/data; chown -R $UID:$GID /app/backend/data/; \

1
backend/requirements.txt
View file

@ -78,7 +78,6 @@ python-pptx==1.0.2
msoffcrypto-tool==6.0.0
unstructured==0.22.31
nltk==3.9.4
Markdown==3.10.2
beautifulsoup4==4.14.3
lxml==6.1.1

1
backend/start.sh
View file

@ -24,7 +24,6 @@ if [[ "${WEB_LOADER_ENGINE,,}" == "playwright" ]]; then
playwright install chromium
playwright install-deps chromium
fi
python -c "import nltk; nltk.download('punkt_tab')"
fi
# ── Secret key setup ─────────────────────────────────────────────────────────

2
backend/start_windows.bat
View file

@ -13,8 +13,6 @@ IF /I "%WEB_LOADER_ENGINE%" == "playwright" (
playwright install chromium
playwright install-deps chromium
)
python -c "import nltk; nltk.download('punkt_tab')"
)
SET "KEY_FILE=.webui_secret_key"

1
pyproject.toml
View file

@ -85,7 +85,6 @@ dependencies = [
"python-docx==1.2.0",
"python-pptx==1.0.2",
"msoffcrypto-tool==6.0.0",
"nltk==3.9.4",
"Markdown==3.10.2",
"beautifulsoup4==4.14.3",
"lxml==6.1.1",

17
uv.lock generated
View file

@ -2690,21 +2690,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" },
]
[[package]]
name = "nltk"
version = "3.9.4"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
{ name = "joblib" },
{ name = "regex" },
{ name = "tqdm" },
]
sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" },
]
[[package]]
name = "numba"
version = "0.65.1"
@ -3025,7 +3010,6 @@ dependencies = [
{ name = "markdown" },
{ name = "mcp" },
{ name = "msoffcrypto-tool" },
{ name = "nltk" },
{ name = "onnxruntime" },
{ name = "openai" },
{ name = "opencv-python-headless" },
@ -3165,7 +3149,6 @@ requires-dist = [
{ name = "markdown", specifier = "==3.10.2" },
{ name = "mcp", specifier = "==1.27.2" },
{ name = "msoffcrypto-tool", specifier = "==6.0.0" },
{ name = "nltk", specifier = "==3.9.4" },
{ name = "onnxruntime", specifier = "==1.26.0" },
{ name = "openai", specifier = "==2.29.0" },
{ name = "opencv-python-headless", specifier = "==4.13.0.92" },