feat: add per-model concurrency limits
This commit is contained in:
1 parent
24eb629773
commit
aca089e8ef
11 files changed
+695
-25
No files matched your search
@@ -249,6 +249,26 @@ class ConcurrencyMonitorSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class ModelConcurrencySettings(BaseModel):
|
||||
"""Admin-configurable per-model concurrent Q&A limits."""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether per-model concurrency limiting is enabled.",
|
||||
)
|
||||
default_limit: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=10_000,
|
||||
description="Fallback max concurrent runs for models without an explicit override.",
|
||||
)
|
||||
model_limits: dict[str, int] = Field(
|
||||
default_factory=dict,
|
||||
description="Optional per-model overrides: model_name -> max concurrent runs.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SystemSettings(BaseModel):
|
||||
"""Top-level system settings document."""
|
||||
|
||||
@@ -262,6 +282,7 @@ class SystemSettings(BaseModel):
|
||||
skill_compression: SkillCompressionSettings = Field(default_factory=SkillCompressionSettings)
|
||||
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
|
||||
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
|
||||
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
@@ -305,6 +326,15 @@ def get_concurrency_monitor_settings() -> ConcurrencyMonitorSettings:
|
||||
return ConcurrencyMonitorSettings()
|
||||
|
||||
|
||||
def get_model_concurrency_settings() -> ModelConcurrencySettings:
|
||||
"""Convenience accessor for the model-concurrency block (safe defaults on error)."""
|
||||
try:
|
||||
return load_system_settings().model_concurrency
|
||||
except Exception:
|
||||
logger.debug("Failed to read model concurrency settings; using defaults", exc_info=True)
|
||||
return ModelConcurrencySettings()
|
||||
|
||||
|
||||
def save_system_settings(settings: SystemSettings) -> SystemSettings:
|
||||
"""Persist the settings document atomically, returning the saved value."""
|
||||
path = _settings_path()
|
||||
|
||||
Reference in new issue
Block a user