mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-02 07:13:50 +00:00
* feat(codex): add authentication mode support with thread-safe logging - Implement _CodexAuthConfig dataclass for resolved auth settings - Add auth_mode parameter with auto/api_key/oauth options - Separate API key and OAuth authentication flows - Force specific login method based on auth mode - Add explicit API key validation requirement - Serialize concurrent logger initialization in thread lock - Close logging handlers properly during cleanup - Update default config with auth_mode presets for codex and codex_oauth - Add comprehensive tests for authentication modes and concurrent logging * feat(file_store): implement background embedding backfill and keyword index repair - Add _after_embedding_backfill hook in FAISS local file store - Schedule startup embedding repair without delaying component readiness - Cancel and collect embedding backfill task during component shutdown - Log progress at fixed percentage boundaries for long-running operations - Process embedding backfill in configurable batch sizes with progress reporting - Rebuild keyword index in bounded batches with detailed mismatch diagnostics - Format stdlib logs consistently with QwenPaw console output using relative paths - Run embedding backfill as background task that doesn't block component startup - Add comprehensive tests for background embedding and keyword index repair scenarios * fix(file-store): repair graph-chunk consistency on load - Add _repair_graph_chunk_consistency method to detect and fix mismatched graph/chunk states - Clear torn graph/chunk state when missing or orphaned chunks are detected - Ensure keyword index sync handles empty chunks properly - Add comprehensive tests for graph-chunk consistency scenarios - Update test utilities to properly seed graph/chunk snapshots - Increment version to 0.4.1.2 * feat(file_io): enhance list step response format and add comprehensive logging - Format list output with bullet points for better readability - Add explicit "No files found" message when directory is empty - Include detailed timing information for file store startup phases - Add logging for chunk loading, graph consistency checks, and keyword indexing - Provide detailed metrics for embedding backfill operations - Add comprehensive test coverage for empty directory scenarios - Include batch processing statistics for embedding operations * feat(logger): add QwenPaw logging integration with forwarding mechanism - Introduce _ForwardToLoggerHandler to forward log records to target logger - Add qwenpaw logger integration that forwards ReMe logs to QwenPaw handlers - Maintain ReMe logger stability for modules that cache it at import time - Enable QwenPaw handlers to take effect without ReMe reconfiguration - Add comprehensive tests for stdlib forwarding to QwenPaw sinks - Support explicit REME_DISABLE_LOGURU=false to keep original Loguru backend - Preserve existing logging behavior when QwenPaw is not configured * fix(file_store): serialize concurrent FAISS dump operations to prevent corruption - Add asyncio lock to ensure only one FAISS dump operation runs at a time - Generate unique temporary filenames using UUID tokens for atomic replacement - Implement proper cleanup of temporary files in finally block - Add comprehensive test to verify concurrent dumps are serialized - Ensure atomic writes by replacing both index and idmap files together - Prevent partial state writes during concurrent access scenarios
215 lines
7.2 KiB
Python
215 lines
7.2 KiB
Python
"""Tests for logging configuration handoff during app startup."""
|
|
|
|
import concurrent.futures
|
|
import io
|
|
import logging
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from reme.application import Application
|
|
from reme.config.config_parser import resolve_app_config
|
|
from reme.utils import logger_utils
|
|
|
|
|
|
class DummyLogger:
|
|
"""Minimal logger used to capture initialization without touching sinks."""
|
|
|
|
def bind(self, **_kwargs):
|
|
"""No-op."""
|
|
return self
|
|
|
|
def info(self, *_args, **_kwargs):
|
|
"""No-op."""
|
|
return None
|
|
|
|
|
|
def test_stdlib_formatter_matches_qwenpaw_console_format(monkeypatch, tmp_path, capsys):
|
|
"""Stdlib logs should use QwenPaw's level/path/time/message layout."""
|
|
monkeypatch.chdir(tmp_path)
|
|
monkeypatch.setenv("REME_DISABLE_LOGURU", "true")
|
|
source_path = tmp_path / "src" / "qwenpaw" / "worker.py"
|
|
record = logging.LogRecord(
|
|
name="reme",
|
|
level=logging.INFO,
|
|
pathname=str(source_path),
|
|
lineno=42,
|
|
msg="Memory index loaded",
|
|
args=(),
|
|
exc_info=None,
|
|
)
|
|
record.created = 0
|
|
|
|
logger = logger_utils.get_logger(log_to_file=False, force_init=True)
|
|
logger.handle(record)
|
|
|
|
formatted_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
|
|
assert capsys.readouterr().out == (f"INFO src/qwenpaw/worker.py:42 | {formatted_time} | Memory index loaded\n")
|
|
logger_utils.get_logger(log_to_console=False, log_to_file=False, force_init=True)
|
|
|
|
|
|
def test_stdlib_forwards_screen_and_file_logs_to_qwenpaw(monkeypatch, tmp_path):
|
|
"""Embedded stdlib logging should reuse QwenPaw's active sinks."""
|
|
monkeypatch.chdir(tmp_path)
|
|
monkeypatch.setenv("REME_DISABLE_LOGURU", "true")
|
|
monkeypatch.setattr(logger_utils, "_logger", None)
|
|
|
|
qwenpaw_logger = logging.getLogger("qwenpaw")
|
|
original_handlers = list(qwenpaw_logger.handlers)
|
|
original_level = qwenpaw_logger.level
|
|
original_propagate = qwenpaw_logger.propagate
|
|
for handler in original_handlers:
|
|
qwenpaw_logger.removeHandler(handler)
|
|
|
|
console_stream = io.StringIO()
|
|
console_handler = logging.StreamHandler(console_stream)
|
|
file_path = tmp_path / "qwenpaw.log"
|
|
file_handler = logging.FileHandler(file_path, encoding="utf-8")
|
|
formatter = logging.Formatter("%(levelname)s | %(message)s")
|
|
console_handler.setFormatter(formatter)
|
|
file_handler.setFormatter(formatter)
|
|
qwenpaw_logger.addHandler(console_handler)
|
|
qwenpaw_logger.addHandler(file_handler)
|
|
qwenpaw_logger.setLevel(logging.INFO)
|
|
qwenpaw_logger.propagate = False
|
|
|
|
reme_logger = logging.getLogger("reme")
|
|
try:
|
|
logger = logger_utils.get_logger(
|
|
log_to_console=True,
|
|
log_to_file=True,
|
|
force_init=True,
|
|
)
|
|
logger.info("Memory index loaded")
|
|
file_handler.flush()
|
|
|
|
assert console_stream.getvalue() == "INFO | Memory index loaded\n"
|
|
assert file_path.read_text(encoding="utf-8") == "INFO | Memory index loaded\n"
|
|
assert not (tmp_path / "logs").exists()
|
|
assert len(reme_logger.handlers) == 1
|
|
assert reme_logger.handlers[0].target_name == "qwenpaw"
|
|
finally:
|
|
for handler in list(reme_logger.handlers):
|
|
reme_logger.removeHandler(handler)
|
|
handler.close()
|
|
qwenpaw_logger.removeHandler(console_handler)
|
|
qwenpaw_logger.removeHandler(file_handler)
|
|
console_handler.close()
|
|
file_handler.close()
|
|
for handler in original_handlers:
|
|
qwenpaw_logger.addHandler(handler)
|
|
qwenpaw_logger.setLevel(original_level)
|
|
qwenpaw_logger.propagate = original_propagate
|
|
|
|
|
|
def test_explicit_loguru_enable_keeps_original_backend(monkeypatch):
|
|
"""An explicit false value must continue to select Loguru unchanged."""
|
|
sentinel_logger = object()
|
|
calls = []
|
|
|
|
def fake_init(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return sentinel_logger
|
|
|
|
monkeypatch.setenv("REME_DISABLE_LOGURU", "false")
|
|
monkeypatch.setattr(logger_utils, "_logger", None)
|
|
monkeypatch.setattr(logger_utils, "_init_loguru", fake_init)
|
|
monkeypatch.setattr(
|
|
logger_utils,
|
|
"_init_stdlib",
|
|
lambda *_args, **_kwargs: pytest.fail("stdlib backend selected"),
|
|
)
|
|
|
|
result = logger_utils.get_logger(force_init=True)
|
|
|
|
assert result is sentinel_logger
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_resolve_app_config_does_not_create_file_logger(monkeypatch):
|
|
"""Config-loading messages should not create empty run log files."""
|
|
calls = []
|
|
|
|
def fake_get_logger(**kwargs):
|
|
calls.append(kwargs)
|
|
return DummyLogger()
|
|
|
|
monkeypatch.setattr("reme.utils.get_logger", fake_get_logger)
|
|
|
|
resolve_app_config(config="demo")
|
|
|
|
assert calls[0]["log_to_file"] is False
|
|
|
|
|
|
def test_application_reinitializes_logger_from_final_config(monkeypatch, tmp_path):
|
|
"""Application startup should install sinks from the resolved ApplicationConfig."""
|
|
calls = []
|
|
|
|
def fake_get_logger(**kwargs):
|
|
calls.append(kwargs)
|
|
return DummyLogger()
|
|
|
|
monkeypatch.setattr("reme.application.get_logger", fake_get_logger)
|
|
monkeypatch.setattr("reme.components.base_component.get_logger", lambda **_kwargs: DummyLogger())
|
|
monkeypatch.setattr(Application, "_init_service", lambda self: setattr(self.context, "service", None))
|
|
monkeypatch.setattr(Application, "_init_components", lambda self: None)
|
|
monkeypatch.setattr(Application, "_init_jobs", lambda self: None)
|
|
|
|
Application(
|
|
enable_logo=False,
|
|
log_to_console=False,
|
|
log_to_file=True,
|
|
workspace_dir=str(tmp_path / "workspace"),
|
|
service={"backend": "unused"},
|
|
)
|
|
|
|
assert calls[0] == {
|
|
"log_to_console": False,
|
|
"log_to_file": True,
|
|
"force_init": True,
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("use_loguru", [True, False])
|
|
def test_concurrent_force_init_is_serialized(monkeypatch, use_loguru):
|
|
"""Concurrent application startup must not overlap global logger resets."""
|
|
state_lock = threading.Lock()
|
|
active_initializers = 0
|
|
max_active_initializers = 0
|
|
initialization_count = 0
|
|
sentinel_logger = object()
|
|
|
|
def fake_init(*_args, **_kwargs):
|
|
nonlocal active_initializers
|
|
nonlocal max_active_initializers
|
|
nonlocal initialization_count
|
|
|
|
with state_lock:
|
|
active_initializers += 1
|
|
initialization_count += 1
|
|
max_active_initializers = max(
|
|
max_active_initializers,
|
|
active_initializers,
|
|
)
|
|
time.sleep(0.01)
|
|
with state_lock:
|
|
active_initializers -= 1
|
|
return sentinel_logger
|
|
|
|
monkeypatch.setattr(logger_utils, "_logger", None)
|
|
monkeypatch.setattr(logger_utils, "_enable_loguru", lambda: use_loguru)
|
|
monkeypatch.setattr(logger_utils, "_init_loguru", fake_init)
|
|
monkeypatch.setattr(logger_utils, "_init_stdlib", fake_init)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
|
|
results = list(
|
|
pool.map(
|
|
lambda _index: logger_utils.get_logger(force_init=True),
|
|
range(32),
|
|
),
|
|
)
|
|
|
|
assert results == [sentinel_logger] * 32
|
|
assert initialization_count == 32
|
|
assert max_active_initializers == 1
|