From ca9ec06c7e65f795cc950ed497ca8b93dd3169c0 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:39:05 +0200 Subject: [PATCH] chore: drop nltk, unused at the pinned versions (#29725) The main and CUDA Docker images get about 21 MB smaller (the nltk package, the punkt_tab data and its zip); slim images, which never downloaded the data, about 6 MB. nltk was in the image for unstructured, which used it to tokenize documents. The Dockerfile download was added for airgapped containers failing on the missing punkt_tab data (#21150; the same request in #16260), the same lookup failed on first use in other setups (#17594, #4642), and the download in start.sh and start_windows.bat came with the Playwright web loader mode and sits in that branch. unstructured 0.22.31, the pinned version, has no nltk references at all and tokenizes with spaCy, nothing else installed requires nltk outside transformers' testing and dev extras, and nothing in the backend imports it, so the pin and both downloads go together. One user-visible consequence: a tool or function that imports nltk inside the container stops working unless it declares nltk in its frontmatter requirements. On an offline instance the package, and any nltk data such as punkt_tab, have to be installed into the image instead. Part of #29721. --- Dockerfile | 2 -- backend/requirements.txt | 1 - backend/start.sh | 1 - backend/start_windows.bat | 2 -- pyproject.toml | 1 - uv.lock | 17 ----------------- 6 files changed, 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e5b033853..e5d7925e94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -148,7 +148,6 @@ RUN set -e; \ python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ.get('AUXILIARY_EMBEDDING_MODEL', 'TaylorAI/bge-micro-v2'), device='cpu')"; \ python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \ python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \ - python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')"; \ else \ pip3 install 'torch<=2.9.1' torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir; \ uv pip install --system -r requirements.txt --no-cache-dir; \ @@ -157,7 +156,6 @@ RUN set -e; \ python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ.get('AUXILIARY_EMBEDDING_MODEL', 'TaylorAI/bge-micro-v2'), device='cpu')"; \ python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \ python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \ - python -c "import nltk; nltk.download('punkt_tab', download_dir='/usr/local/share/nltk_data')"; \ fi; \ fi; \ mkdir -p /app/backend/data; chown -R $UID:$GID /app/backend/data/; \ diff --git a/backend/requirements.txt b/backend/requirements.txt index a8b091c37b..254425a988 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -78,7 +78,6 @@ python-pptx==1.0.2 msoffcrypto-tool==6.0.0 unstructured==0.22.31 -nltk==3.9.4 Markdown==3.10.2 beautifulsoup4==4.14.3 lxml==6.1.1 diff --git a/backend/start.sh b/backend/start.sh index 634f4bcc1b..b80e2fd67f 100755 --- a/backend/start.sh +++ b/backend/start.sh @@ -24,7 +24,6 @@ if [[ "${WEB_LOADER_ENGINE,,}" == "playwright" ]]; then playwright install chromium playwright install-deps chromium fi - python -c "import nltk; nltk.download('punkt_tab')" fi # ── Secret key setup ───────────────────────────────────────────────────────── diff --git a/backend/start_windows.bat b/backend/start_windows.bat index 4a57d34694..22aaaafc76 100644 --- a/backend/start_windows.bat +++ b/backend/start_windows.bat @@ -13,8 +13,6 @@ IF /I "%WEB_LOADER_ENGINE%" == "playwright" ( playwright install chromium playwright install-deps chromium ) - - python -c "import nltk; nltk.download('punkt_tab')" ) SET "KEY_FILE=.webui_secret_key" diff --git a/pyproject.toml b/pyproject.toml index 5b6b3e0749..ce1d71d436 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,6 @@ dependencies = [ "python-docx==1.2.0", "python-pptx==1.0.2", "msoffcrypto-tool==6.0.0", - "nltk==3.9.4", "Markdown==3.10.2", "beautifulsoup4==4.14.3", "lxml==6.1.1", diff --git a/uv.lock b/uv.lock index cff82b6167..62d9b8b266 100644 --- a/uv.lock +++ b/uv.lock @@ -2690,21 +2690,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, ] -[[package]] -name = "nltk" -version = "3.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, -] - [[package]] name = "numba" version = "0.65.1" @@ -3025,7 +3010,6 @@ dependencies = [ { name = "markdown" }, { name = "mcp" }, { name = "msoffcrypto-tool" }, - { name = "nltk" }, { name = "onnxruntime" }, { name = "openai" }, { name = "opencv-python-headless" }, @@ -3165,7 +3149,6 @@ requires-dist = [ { name = "markdown", specifier = "==3.10.2" }, { name = "mcp", specifier = "==1.27.2" }, { name = "msoffcrypto-tool", specifier = "==6.0.0" }, - { name = "nltk", specifier = "==3.9.4" }, { name = "onnxruntime", specifier = "==1.26.0" }, { name = "openai", specifier = "==2.29.0" }, { name = "opencv-python-headless", specifier = "==4.13.0.92" },