refactor(config): remove max_file_bytes limit from background jobs (#367)

- Removed max_file_bytes configuration from index_update_loop, resource_watch_loop, digest_watch_loop, and reindex jobs
- Updated default.yaml to reflect simplified job configurations without file size limits
- Removed corresponding test case that validated the 20 MiB limit behavior
- Simplified watch directories and suffixes to basic configurations
- Cleaned up unnecessary commented configurations in the YAML file

reme/config/default.yaml

@@ -4,11 +4,8 @@ service:
 jobs:
   index_update_loop:
     backend: background
-    max_file_bytes: 20971520
-    watch_dirs: [daily_dir, digest_dir, resource_dir]
-#    watch_dirs: [daily_dir, digest_dir]
-    watch_suffixes: [md, jsonl]
-#    watch_suffixes: [md]
+    watch_dirs: [daily_dir, digest_dir]
+    watch_suffixes: [md]
     steps:
       - backend: init_changes_step
         monitor_type: file_store
@@ -21,7 +18,6 @@ jobs:
 
   resource_watch_loop:
     backend: background
-    max_file_bytes: 20971520
     watch_dirs: [resource_dir]
     watch_suffixes: [md, txt, json, jsonl, csv, yaml, html]
     steps:
@@ -40,7 +36,6 @@ jobs:
 
   digest_watch_loop:
     backend: background
-    max_file_bytes: 20971520
     watch_dirs: [daily_dir, digest_dir]
     watch_suffixes: [md]
     steps:
@@ -282,7 +277,6 @@ jobs:
 
   reindex:
     backend: base
-    max_file_bytes: 20971520
     description: "wipe the file store and rebuild it from the existing files"
     watch_dirs: [daily_dir, digest_dir, resource_dir]
     watch_suffixes: [md, jsonl]

tests/unit/test_config_parser.py

@@ -61,14 +61,6 @@ def test_default_config_keeps_frontmatter_chunk_metadata_opt_in():
     ) in (None, [])
 
 
-def test_default_config_limits_background_and_reindex_file_processing():
-    """All built-in content-processing entry points use the 20 MiB limit."""
-    cfg = _load_config("default.yaml")
-
-    for job_name in ("index_update_loop", "resource_watch_loop", "digest_watch_loop", "reindex"):
-        assert cfg["jobs"][job_name]["max_file_bytes"] == 20 * 1024 * 1024
-
-
 def test_parse_args_rejects_non_key_value_extra_argument():
     """Extra CLI arguments must use key=value syntax."""
     with pytest.raises(ValueError, match="expected key=value"):