From 329fd9a6a60facf34ae57030df01d636d88e33a7 Mon Sep 17 00:00:00 2001
From: jinliyl <6469360+jinliyl@users.noreply.github.com>
Date: Fri, 17 Jul 2026 11:26:39 +0800
Subject: [PATCH] refactor(config): remove max_file_bytes limit from background
 jobs (#367)

- Removed max_file_bytes configuration from index_update_loop, resource_watch_loop, digest_watch_loop, and reindex jobs
- Updated default.yaml to reflect simplified job configurations without file size limits
- Removed corresponding test case that validated the 20 MiB limit behavior
- Simplified watch directories and suffixes to basic configurations
- Cleaned up unnecessary commented configurations in the YAML file
---
 reme/config/default.yaml         | 10 ++--------
 tests/unit/test_config_parser.py |  8 --------
 2 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/reme/config/default.yaml b/reme/config/default.yaml
index c810758b..8b65573b 100644
--- a/reme/config/default.yaml
+++ b/reme/config/default.yaml
@@ -4,11 +4,8 @@ service:
 jobs:
   index_update_loop:
     backend: background
-    max_file_bytes: 20971520
-    watch_dirs: [daily_dir, digest_dir, resource_dir]
-#    watch_dirs: [daily_dir, digest_dir]
-    watch_suffixes: [md, jsonl]
-#    watch_suffixes: [md]
+    watch_dirs: [daily_dir, digest_dir]
+    watch_suffixes: [md]
     steps:
       - backend: init_changes_step
         monitor_type: file_store
@@ -21,7 +18,6 @@ jobs:
 
   resource_watch_loop:
     backend: background
-    max_file_bytes: 20971520
     watch_dirs: [resource_dir]
     watch_suffixes: [md, txt, json, jsonl, csv, yaml, html]
     steps:
@@ -40,7 +36,6 @@ jobs:
 
   digest_watch_loop:
     backend: background
-    max_file_bytes: 20971520
     watch_dirs: [daily_dir, digest_dir]
     watch_suffixes: [md]
     steps:
@@ -282,7 +277,6 @@ jobs:
 
   reindex:
     backend: base
-    max_file_bytes: 20971520
     description: "wipe the file store and rebuild it from the existing files"
     watch_dirs: [daily_dir, digest_dir, resource_dir]
     watch_suffixes: [md, jsonl]
diff --git a/tests/unit/test_config_parser.py b/tests/unit/test_config_parser.py
index e056b433..9f1c39c1 100644
--- a/tests/unit/test_config_parser.py
+++ b/tests/unit/test_config_parser.py
@@ -61,14 +61,6 @@ def test_default_config_keeps_frontmatter_chunk_metadata_opt_in():
     ) in (None, [])
 
 
-def test_default_config_limits_background_and_reindex_file_processing():
-    """All built-in content-processing entry points use the 20 MiB limit."""
-    cfg = _load_config("default.yaml")
-
-    for job_name in ("index_update_loop", "resource_watch_loop", "digest_watch_loop", "reindex"):
-        assert cfg["jobs"][job_name]["max_file_bytes"] == 20 * 1024 * 1024
-
-
 def test_parse_args_rejects_non_key_value_extra_argument():
     """Extra CLI arguments must use key=value syntax."""
     with pytest.raises(ValueError, match="expected key=value"):