mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-19 00:01:33 +00:00
Some checks are pending
Pre-commit / run (ubuntu-latest) (push) Waiting to run
- Implement BaseComponent with async lifecycle and context management - Add ApplicationContext for managing component initialization and registry - Create Application class for orchestrating job execution and lifecycle - Add AS LLM components with OpenAI chat model wrapper - Implement AS LLM formatter components with OpenAI formatter - Add client implementations including base, HTTP and ReMe clients - Create embedding model base class with caching and batching support - Implement file store base class with vector and full-text search - Add file watcher components for monitoring file system changes - Create job components for executing workflows - Implement service components for exposing jobs via different protocols - Add configuration schema with ApplicationConfig and ComponentConfig - Include utility modules for case conversion, chunking, logging and similarity - Register component types and create component registry system
158 lines
4.5 KiB
Python
158 lines
4.5 KiB
Python
"""Parser for YAML config with CLI argument overrides."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
# Config files are looked up relative to this module's directory
|
|
_CONFIG_DIR = Path(__file__).parent
|
|
_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
|
|
|
|
|
|
# Pre-scan config directory: maps basename(without ext) -> Path
|
|
def _discover_configs() -> dict[str, Path]:
|
|
discovered: dict[str, Path] = {}
|
|
if _CONFIG_DIR.is_dir():
|
|
for p in _CONFIG_DIR.iterdir():
|
|
if p.is_file() and p.suffix in _SUPPORTED_EXTS:
|
|
discovered.setdefault(p.stem, p)
|
|
return discovered
|
|
|
|
|
|
_CONFIG_REGISTRY = _discover_configs()
|
|
|
|
|
|
def parse_dot_notation(dot_list: list[str]) -> dict:
|
|
"""Parse "key.subkey=value" strings into nested dict."""
|
|
result: dict = {}
|
|
for item in dot_list:
|
|
if "=" not in item:
|
|
raise ValueError(f"Invalid dot notation format (missing '='): {item}")
|
|
key_path, value_str = item.split("=", 1)
|
|
keys = key_path.split(".")
|
|
current = result
|
|
for key in keys[:-1]:
|
|
if key in current and not isinstance(current[key], dict):
|
|
raise ValueError(f"Cannot set nested key '{key_path}': '{key}' is already a value")
|
|
current = current.setdefault(key, {})
|
|
current[keys[-1]] = _convert_value(value_str)
|
|
return result
|
|
|
|
|
|
def _convert_value(value_str: str) -> Any:
|
|
"""Convert string to appropriate Python type.
|
|
|
|
Only converts "true"/"false" (case-insensitive) to boolean.
|
|
Use JSON format (e.g., '"yes"', '"no"') to preserve these as strings.
|
|
"""
|
|
s = value_str.strip()
|
|
|
|
# Null
|
|
if s.lower() in ("none", "null"):
|
|
return None
|
|
|
|
# Boolean: only accept strict true/false to avoid surprising conversions
|
|
lower = s.lower()
|
|
if lower == "true":
|
|
return True
|
|
if lower == "false":
|
|
return False
|
|
|
|
# Int
|
|
try:
|
|
return int(s)
|
|
except ValueError:
|
|
pass
|
|
|
|
# Float
|
|
try:
|
|
return float(s)
|
|
except ValueError:
|
|
pass
|
|
|
|
# JSON (list, dict, quoted strings)
|
|
try:
|
|
return json.loads(s)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
return s
|
|
|
|
|
|
def _load_yaml(name_or_path: str, encoding: str = "utf-8") -> dict:
|
|
"""Load YAML or JSON file.
|
|
|
|
First check if name_or_path matches a pre-discovered config (key in _CONFIG_REGISTRY).
|
|
If not, treat as a file path and load directly.
|
|
"""
|
|
# 1. Try pre-discovered configs first
|
|
if name_or_path in _CONFIG_REGISTRY:
|
|
return _read_config_file(_CONFIG_REGISTRY[name_or_path], encoding)
|
|
|
|
# 2. Treat as file path
|
|
p = Path(name_or_path)
|
|
if p.suffix in _SUPPORTED_EXTS:
|
|
if not p.exists():
|
|
raise FileNotFoundError(f"Config file not found: {p}")
|
|
return _read_config_file(p, encoding)
|
|
|
|
known = ", ".join(sorted(_CONFIG_REGISTRY)) if _CONFIG_REGISTRY else "none"
|
|
raise FileNotFoundError(f"Config file not found: {name_or_path}. Available: {known}")
|
|
|
|
|
|
def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
|
|
"""Read YAML or JSON file based on extension."""
|
|
with path.open(encoding=encoding) as f:
|
|
if path.suffix == ".json":
|
|
result = json.load(f)
|
|
return result if result is not None else {}
|
|
else:
|
|
result = yaml.safe_load(f)
|
|
return result if result is not None else {}
|
|
|
|
|
|
def _deep_merge(base: dict, update: dict) -> dict:
|
|
"""Recursively merge dicts."""
|
|
result = base.copy()
|
|
for k, v in update.items():
|
|
if k in result and isinstance(result[k], dict) and isinstance(v, dict):
|
|
result[k] = _deep_merge(result[k], v)
|
|
else:
|
|
result[k] = v
|
|
return result
|
|
|
|
|
|
def parse_args(*args, **kwargs) -> tuple[str, dict]:
|
|
"""Parse CLI args: first arg is action, rest are config overrides.
|
|
|
|
Usage: reme app config=paw.yaml service.name=test
|
|
Returns: (action, merged_config_dict)
|
|
"""
|
|
if not args:
|
|
raise ValueError("No arguments provided")
|
|
|
|
first = args[0].lstrip("-")
|
|
if "=" in first:
|
|
raise ValueError(f"First argument must be action, got: {args[0]}")
|
|
|
|
action = first
|
|
configs: list[dict] = []
|
|
|
|
for arg in args[1:]:
|
|
arg = arg.lstrip("-")
|
|
if arg.startswith("config="):
|
|
path = arg.split("=", 1)[1].strip()
|
|
if path:
|
|
configs.append(_load_yaml(path))
|
|
elif "=" in arg:
|
|
configs.append(parse_dot_notation([arg]))
|
|
|
|
configs.append(kwargs)
|
|
|
|
merged: dict = {}
|
|
for cfg in configs:
|
|
merged = _deep_merge(merged, cfg)
|
|
|
|
return action, merged
|