feat(skills): curator 健壮性加固 + 评审模型可配置 + 手动触发体验
调度健壮性:
- 技能进化总开关 skill_evolution.enabled 真正生效;CuratorConfig
与 CuratorSubConfig 默认 enabled 改为 false(显式 opt-in)
- 新增 per-user 运行锁(.curator_state 的 running/started_at),
防定时器与手动触发并发重复运行;6h stale 检测防进程残留死锁
- maybe_run_curator 改用 ThreadPoolExecutor 限并发(上限 5),
防首次部署所有用户同时到期的惊群
- 首次运行持续失败时按 6h 退避重试,不再每个 tick 烧 LLM 配额
- should_run_now 时间解析容忍 naive 时间串;损坏的 last_run_at
触发重跑而非永久死锁
- SkillStore 新增 list_published_names 批量接口,curator 一轮
只查一次 published,取代逐技能查库
LLM 评审超时:
- _run_llm_review 给 agent.invoke 套整体超时
(curator.review_timeout_seconds,默认 600s),超时中止本次评审
并在 curator 日志输出,curator 正常收尾、释放运行锁
- copy_context 把 user 上下文传入工作线程
模型可配置:
- 新增 curator.model_name;curator 评审与技能安全审核共用该模型,
未配置则回退主模型
- 移除独立的 moderation_model_name 配置
- 前端进化面板新增模型下拉选择器
手动触发体验:
- /api/curator/run 派发后台线程前先预检,返回 {triggered, reason},
不再无论开关状态都提示"已开始"
- 前端:总开关/curator 开关未开时禁用"立即执行"按钮,
被预检拦下时按原因(开关未开 / 已有任务在跑)给出准确提示
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5a25dd08f6
commit
39def1c20b
16 files changed
+448
-65
No files matched your search
@@ -21,6 +21,13 @@ import {
|
|||||||
CardTitle,
|
CardTitle,
|
||||||
} from "@/components/ui/card";
|
} from "@/components/ui/card";
|
||||||
import { Input } from "@/components/ui/input";
|
import { Input } from "@/components/ui/input";
|
||||||
|
import {
|
||||||
|
Select,
|
||||||
|
SelectContent,
|
||||||
|
SelectItem,
|
||||||
|
SelectTrigger,
|
||||||
|
SelectValue,
|
||||||
|
} from "@/components/ui/select";
|
||||||
import { Switch } from "@/components/ui/switch";
|
import { Switch } from "@/components/ui/switch";
|
||||||
import {
|
import {
|
||||||
useCuratorConfig,
|
useCuratorConfig,
|
||||||
@@ -31,6 +38,7 @@ import {
|
|||||||
} from "@/core/curator/hooks";
|
} from "@/core/curator/hooks";
|
||||||
import type { CuratorConfig } from "@/core/curator/types";
|
import type { CuratorConfig } from "@/core/curator/types";
|
||||||
import { useI18n } from "@/core/i18n/hooks";
|
import { useI18n } from "@/core/i18n/hooks";
|
||||||
|
import { useModels } from "@/core/models/hooks";
|
||||||
import { cn } from "@/lib/utils";
|
import { cn } from "@/lib/utils";
|
||||||
|
|
||||||
export function SkillEvolutionPanel() {
|
export function SkillEvolutionPanel() {
|
||||||
@@ -68,6 +76,7 @@ function StatusCard() {
|
|||||||
const { t } = useI18n();
|
const { t } = useI18n();
|
||||||
const tt = t.settings.skillEvolution.status;
|
const tt = t.settings.skillEvolution.status;
|
||||||
const { status, isLoading } = useCuratorStatus();
|
const { status, isLoading } = useCuratorStatus();
|
||||||
|
const { config } = useCuratorConfig();
|
||||||
const runCurator = useRunCurator();
|
const runCurator = useRunCurator();
|
||||||
|
|
||||||
const lastRunDisplay = useMemo(() => {
|
const lastRunDisplay = useMemo(() => {
|
||||||
@@ -79,10 +88,25 @@ function StatusCard() {
|
|||||||
}
|
}
|
||||||
}, [status?.last_run_at, tt.lastRunNever]);
|
}, [status?.last_run_at, tt.lastRunNever]);
|
||||||
|
|
||||||
|
// 总开关或 curator 子开关任一关闭时,手动触发跑不起来 —— 直接禁用按钮。
|
||||||
|
const runDisabled = !config || !config.enabled || !config.curator.enabled;
|
||||||
|
|
||||||
const handleRun = async () => {
|
const handleRun = async () => {
|
||||||
try {
|
try {
|
||||||
await runCurator.mutateAsync();
|
const res = await runCurator.mutateAsync();
|
||||||
toast.success(tt.runStarted);
|
if (res.triggered) {
|
||||||
|
toast.success(tt.runStarted);
|
||||||
|
} else {
|
||||||
|
// 后端预检拦下了(开关未开 / 已有任务在跑)—— 给出对应原因,而不是误报"已开始"。
|
||||||
|
const reasonMsg: Record<string, string> = {
|
||||||
|
evolution_disabled: tt.runBlockedEvolutionDisabled,
|
||||||
|
curator_disabled: tt.runBlockedCuratorDisabled,
|
||||||
|
already_running: tt.runBlockedAlreadyRunning,
|
||||||
|
};
|
||||||
|
toast.warning(
|
||||||
|
reasonMsg[res.reason ?? ""] ?? tt.runBlockedEvolutionDisabled,
|
||||||
|
);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
toast.error(err instanceof Error ? err.message : String(err));
|
toast.error(err instanceof Error ? err.message : String(err));
|
||||||
}
|
}
|
||||||
@@ -107,7 +131,8 @@ function StatusCard() {
|
|||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
onClick={handleRun}
|
onClick={handleRun}
|
||||||
disabled={runCurator.isPending}
|
disabled={runCurator.isPending || runDisabled}
|
||||||
|
title={runDisabled ? tt.runDisabledHint : undefined}
|
||||||
>
|
>
|
||||||
{runCurator.isPending ? (
|
{runCurator.isPending ? (
|
||||||
<Loader2Icon className="size-4 animate-spin" />
|
<Loader2Icon className="size-4 animate-spin" />
|
||||||
@@ -245,11 +270,15 @@ function StatRow({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Select 不允许空字符串 value,用此哨兵代表"用主模型"(对应 model_name=null)。 */
|
||||||
|
const DEFAULT_MODEL_VALUE = "__default__";
|
||||||
|
|
||||||
function ConfigCard() {
|
function ConfigCard() {
|
||||||
const { t } = useI18n();
|
const { t } = useI18n();
|
||||||
const tt = t.settings.skillEvolution.config;
|
const tt = t.settings.skillEvolution.config;
|
||||||
const { config, isLoading } = useCuratorConfig();
|
const { config, isLoading } = useCuratorConfig();
|
||||||
const updateConfig = useUpdateCuratorConfig();
|
const updateConfig = useUpdateCuratorConfig();
|
||||||
|
const { models } = useModels();
|
||||||
|
|
||||||
const [draft, setDraft] = useState<CuratorConfig | null>(null);
|
const [draft, setDraft] = useState<CuratorConfig | null>(null);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -378,22 +407,35 @@ function ConfigCard() {
|
|||||||
/>
|
/>
|
||||||
</FieldRow>
|
</FieldRow>
|
||||||
<FieldRow
|
<FieldRow
|
||||||
label={tt.moderationModelLabel}
|
label={tt.curatorModelLabel}
|
||||||
helper={tt.moderationModelHelper}
|
helper={tt.curatorModelHelper}
|
||||||
>
|
>
|
||||||
<Input
|
<Select
|
||||||
type="text"
|
value={draft.curator.model_name ?? DEFAULT_MODEL_VALUE}
|
||||||
className="max-w-[20rem]"
|
onValueChange={(v) =>
|
||||||
value={draft.moderation_model_name ?? ""}
|
|
||||||
placeholder=""
|
|
||||||
onChange={(e) =>
|
|
||||||
setDraft({
|
setDraft({
|
||||||
...draft,
|
...draft,
|
||||||
moderation_model_name:
|
curator: {
|
||||||
e.target.value.trim() === "" ? null : e.target.value,
|
...draft.curator,
|
||||||
|
model_name: v === DEFAULT_MODEL_VALUE ? null : v,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
/>
|
>
|
||||||
|
<SelectTrigger className="max-w-[20rem]">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value={DEFAULT_MODEL_VALUE}>
|
||||||
|
{tt.curatorModelDefault}
|
||||||
|
</SelectItem>
|
||||||
|
{models.map((m) => (
|
||||||
|
<SelectItem key={m.name} value={m.name}>
|
||||||
|
{m.display_name || m.name}
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
</FieldRow>
|
</FieldRow>
|
||||||
|
|
||||||
<div className="flex justify-end gap-2 pt-2">
|
<div className="flex justify-end gap-2 pt-2">
|
||||||
|
|||||||
@@ -28,11 +28,17 @@ export async function updateCuratorConfig(
|
|||||||
return response.json() as Promise<CuratorConfig>;
|
return response.json() as Promise<CuratorConfig>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runCurator(): Promise<{ triggered: boolean; message: string }> {
|
export interface RunCuratorResult {
|
||||||
|
triggered: boolean;
|
||||||
|
/** 未触发时的原因代码:evolution_disabled / curator_disabled / already_running。 */
|
||||||
|
reason: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runCurator(): Promise<RunCuratorResult> {
|
||||||
const response = await apiFetch(`${getBackendBaseURL()}/api/curator/run`, {
|
const response = await apiFetch(`${getBackendBaseURL()}/api/curator/run`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
});
|
});
|
||||||
return response.json();
|
return response.json() as Promise<RunCuratorResult>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getCuratorLogs(): Promise<CuratorLogs> {
|
export async function getCuratorLogs(): Promise<CuratorLogs> {
|
||||||
|
|||||||
@@ -3,11 +3,13 @@ export interface CuratorSubConfig {
|
|||||||
interval_hours: number;
|
interval_hours: number;
|
||||||
stale_after_days: number;
|
stale_after_days: number;
|
||||||
archive_after_days: number;
|
archive_after_days: number;
|
||||||
|
review_timeout_seconds: number;
|
||||||
|
/** curator 评审使用的模型名;null = 用主模型。 */
|
||||||
|
model_name: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CuratorConfig {
|
export interface CuratorConfig {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
moderation_model_name: string | null;
|
|
||||||
curator: CuratorSubConfig;
|
curator: CuratorSubConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -557,6 +557,12 @@ export const enUS: Translations = {
|
|||||||
total: "Total",
|
total: "Total",
|
||||||
runNow: "Run now",
|
runNow: "Run now",
|
||||||
runStarted: "Background run started",
|
runStarted: "Background run started",
|
||||||
|
runBlockedEvolutionDisabled:
|
||||||
|
"Skill evolution is off — enable it below to run the curator.",
|
||||||
|
runBlockedCuratorDisabled:
|
||||||
|
"The curator is off — enable it below to run it manually.",
|
||||||
|
runBlockedAlreadyRunning: "A curator run is already in progress.",
|
||||||
|
runDisabledHint: "Enable skill evolution and the curator below first.",
|
||||||
lastActions: "Last run",
|
lastActions: "Last run",
|
||||||
llmSuggested: "LLM suggested (self-report)",
|
llmSuggested: "LLM suggested (self-report)",
|
||||||
actuallyDid: "Actually done (tool calls)",
|
actuallyDid: "Actually done (tool calls)",
|
||||||
@@ -581,9 +587,10 @@ export const enUS: Translations = {
|
|||||||
archiveAfterDaysLabel: "Archive threshold (days)",
|
archiveAfterDaysLabel: "Archive threshold (days)",
|
||||||
archiveAfterDaysHelper:
|
archiveAfterDaysHelper:
|
||||||
"Stale skills inactive for this many additional days are archived.",
|
"Stale skills inactive for this many additional days are archived.",
|
||||||
moderationModelLabel: "Moderation model (optional)",
|
curatorModelLabel: "Curator review model",
|
||||||
moderationModelHelper:
|
curatorModelHelper:
|
||||||
"Leave blank to use the default chat model.",
|
"Model for the curator's LLM review and skill security moderation. Defaults to the primary model.",
|
||||||
|
curatorModelDefault: "Primary model (default)",
|
||||||
save: "Save changes",
|
save: "Save changes",
|
||||||
saveSuccess: "Configuration updated",
|
saveSuccess: "Configuration updated",
|
||||||
reset: "Reset",
|
reset: "Reset",
|
||||||
|
|||||||
@@ -482,6 +482,10 @@ export interface Translations {
|
|||||||
total: string;
|
total: string;
|
||||||
runNow: string;
|
runNow: string;
|
||||||
runStarted: string;
|
runStarted: string;
|
||||||
|
runBlockedEvolutionDisabled: string;
|
||||||
|
runBlockedCuratorDisabled: string;
|
||||||
|
runBlockedAlreadyRunning: string;
|
||||||
|
runDisabledHint: string;
|
||||||
lastActions: string;
|
lastActions: string;
|
||||||
llmSuggested: string;
|
llmSuggested: string;
|
||||||
actuallyDid: string;
|
actuallyDid: string;
|
||||||
@@ -500,8 +504,9 @@ export interface Translations {
|
|||||||
staleAfterDaysHelper: string;
|
staleAfterDaysHelper: string;
|
||||||
archiveAfterDaysLabel: string;
|
archiveAfterDaysLabel: string;
|
||||||
archiveAfterDaysHelper: string;
|
archiveAfterDaysHelper: string;
|
||||||
moderationModelLabel: string;
|
curatorModelLabel: string;
|
||||||
moderationModelHelper: string;
|
curatorModelHelper: string;
|
||||||
|
curatorModelDefault: string;
|
||||||
save: string;
|
save: string;
|
||||||
saveSuccess: string;
|
saveSuccess: string;
|
||||||
reset: string;
|
reset: string;
|
||||||
|
|||||||
@@ -533,6 +533,10 @@ export const zhCN: Translations = {
|
|||||||
total: "合计",
|
total: "合计",
|
||||||
runNow: "立即执行",
|
runNow: "立即执行",
|
||||||
runStarted: "已触发后台执行",
|
runStarted: "已触发后台执行",
|
||||||
|
runBlockedEvolutionDisabled: "自我进化未开启 —— 请先在下方打开开关。",
|
||||||
|
runBlockedCuratorDisabled: "curator 未开启 —— 请先在下方打开 curator 开关。",
|
||||||
|
runBlockedAlreadyRunning: "已有一次 curator 运行进行中。",
|
||||||
|
runDisabledHint: "请先在下方开启自我进化与 curator 开关。",
|
||||||
lastActions: "最近一次运行",
|
lastActions: "最近一次运行",
|
||||||
llmSuggested: "LLM 建议(自述)",
|
llmSuggested: "LLM 建议(自述)",
|
||||||
actuallyDid: "实际执行(工具调用)",
|
actuallyDid: "实际执行(工具调用)",
|
||||||
@@ -552,8 +556,9 @@ export const zhCN: Translations = {
|
|||||||
staleAfterDaysHelper: "技能长时间不活跃后被标记为陈旧。",
|
staleAfterDaysHelper: "技能长时间不活跃后被标记为陈旧。",
|
||||||
archiveAfterDaysLabel: "归档阈值(天)",
|
archiveAfterDaysLabel: "归档阈值(天)",
|
||||||
archiveAfterDaysHelper: "标记陈旧后再经过这么多天会被自动归档。",
|
archiveAfterDaysHelper: "标记陈旧后再经过这么多天会被自动归档。",
|
||||||
moderationModelLabel: "安全审核模型(可选)",
|
curatorModelLabel: "技能优化模型",
|
||||||
moderationModelHelper: "为空时使用默认主模型。",
|
curatorModelHelper: "用于 curator 技能评审和技能安全审核的模型,为空则使用主模型。",
|
||||||
|
curatorModelDefault: "主模型(默认)",
|
||||||
save: "保存修改",
|
save: "保存修改",
|
||||||
saveSuccess: "配置已更新",
|
saveSuccess: "配置已更新",
|
||||||
reset: "重置",
|
reset: "重置",
|
||||||
|
|||||||
@@ -302,6 +302,55 @@ Custom agent metadata lives in the `agents` ORM table (`deerflow.persistence.age
|
|||||||
- **Injection**: Enabled skills listed in agent system prompt with container paths
|
- **Injection**: Enabled skills listed in agent system prompt with container paths
|
||||||
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
|
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
|
||||||
|
|
||||||
|
#### Skill Curator (`packages/harness/deerflow/skills/curator.py`)
|
||||||
|
|
||||||
|
Background skill lifecycle manager. The Gateway `lifespan` starts a resident
|
||||||
|
scheduler task (`_curator_scheduler_loop` in `app/gateway/app.py`) that — after a
|
||||||
|
5-minute startup delay — calls `maybe_run_curator()` every hour. A cross-process
|
||||||
|
file lock (`.curator_scheduler.lock`) elects a single leader worker so the
|
||||||
|
scheduler runs once per deployment. Per run:
|
||||||
|
|
||||||
|
- **Master switch** — `skill_evolution.enabled` (default **`false`**) gates the
|
||||||
|
whole feature; `curator.enabled` (default **`false`**) gates the curator
|
||||||
|
specifically. Both must be `true` for the curator to act. `should_run_now()`
|
||||||
|
and `_curator_thread_target()` both enforce this — so manual `POST
|
||||||
|
/api/curator/run` also respects the master switch. Config is
|
||||||
|
`config.yaml.skill_evolution` deep-merged with per-user
|
||||||
|
`users/{user_id}/skill_evolution.json`.
|
||||||
|
- **Per-user run lock** — `.curator_state` carries `running` / `started_at`;
|
||||||
|
`_try_acquire_run_lock()` prevents the same user's curator from running
|
||||||
|
concurrently (scheduler vs. manual trigger, or double manual clicks). A lock
|
||||||
|
whose `started_at` is older than `_CURATOR_STALE_LOCK_HOURS` (6h) is treated as
|
||||||
|
process-crash residue and can be taken over.
|
||||||
|
- **Concurrency cap** — `maybe_run_curator()` collects all due users and hands
|
||||||
|
them to `_dispatch_curator_runs()`, which runs them through a
|
||||||
|
`ThreadPoolExecutor` capped at `_CURATOR_MAX_CONCURRENCY` (5) — avoids a
|
||||||
|
thundering herd of LLM calls when many users come due at once (e.g. first
|
||||||
|
deploy).
|
||||||
|
- **Failure back-off** — `_release_run_lock()` always stamps `last_attempt_at`.
|
||||||
|
When a first run keeps crashing (`last_run_at` never set), `should_run_now()`
|
||||||
|
retries on `_CURATOR_RETRY_HOURS` (6h) instead of every hourly tick.
|
||||||
|
- **`should_run_now()`** is the real gate: the hourly tick only checks "is the
|
||||||
|
per-user `interval_hours` (default 168h) elapsed". Timestamp parsing tolerates
|
||||||
|
naive (tz-less) strings — a corrupt `last_run_at` triggers a re-run rather than
|
||||||
|
a permanent deadlock.
|
||||||
|
- **Published-skill filter** — `SkillStore.list_published_names()` is a batch
|
||||||
|
query; the curator fetches the published-name set once per run instead of one
|
||||||
|
DB round-trip per candidate skill.
|
||||||
|
- **LLM review timeout** — `_run_llm_review()` runs `agent.invoke()` in a worker
|
||||||
|
thread bounded by `curator.review_timeout_seconds` (default 600s). The model's
|
||||||
|
per-request `request_timeout` can't bound a multi-turn React agent loop, so on
|
||||||
|
timeout the run is abandoned (a warning is written to `curator_log`, visible in
|
||||||
|
the Web UI) and the curator finishes cleanly + releases the run lock. The
|
||||||
|
worker context is copied via `contextvars.copy_context()` so tool calls still
|
||||||
|
resolve the correct `user_id`.
|
||||||
|
- **Review model** — `_run_llm_review()` uses `curator.model_name` (a
|
||||||
|
`config.yaml` `models[]` entry name) if set, else the primary model; it is
|
||||||
|
editable from the Web UI evolution panel. The skill security scanner
|
||||||
|
(`scan_skill_content` in `security_scanner.py`) reuses the same
|
||||||
|
`skill_evolution.curator.model_name` setting — review and moderation share one
|
||||||
|
model. There is no separate moderation-model config field.
|
||||||
|
|
||||||
### Model Factory (`packages/harness/deerflow/models/factory.py`)
|
### Model Factory (`packages/harness/deerflow/models/factory.py`)
|
||||||
|
|
||||||
- `create_chat_model(name, thinking_enabled)` instantiates LLM from config via reflection
|
- `create_chat_model(name, thinking_enabled)` instantiates LLM from config via reflection
|
||||||
|
|||||||
@@ -967,12 +967,21 @@ async def curator_status(request: Request) -> dict:
|
|||||||
@router.post("/curator/run", summary="Trigger Curator Run")
|
@router.post("/curator/run", summary="Trigger Curator Run")
|
||||||
async def run_curator(request: Request) -> dict:
|
async def run_curator(request: Request) -> dict:
|
||||||
"""Manually trigger a curator review pass for the authenticated user
|
"""Manually trigger a curator review pass for the authenticated user
|
||||||
(bypasses interval check, runs in a background thread)."""
|
(bypasses the interval check, runs in a background thread).
|
||||||
|
|
||||||
|
Pre-checks the master switch / curator switch / run lock so the response
|
||||||
|
accurately reflects whether a run actually started — instead of always
|
||||||
|
reporting success. Returns ``{"triggered": bool, "reason": str | None}``.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
import threading
|
import threading
|
||||||
from deerflow.skills.curator import _curator_thread_target
|
|
||||||
|
from deerflow.skills.curator import _curator_thread_target, manual_trigger_blocked_reason
|
||||||
|
|
||||||
user_id = _current_user_id(request)
|
user_id = _current_user_id(request)
|
||||||
|
reason = manual_trigger_blocked_reason(user_id)
|
||||||
|
if reason is not None:
|
||||||
|
return {"triggered": False, "reason": reason}
|
||||||
thread = threading.Thread(
|
thread = threading.Thread(
|
||||||
target=_curator_thread_target,
|
target=_curator_thread_target,
|
||||||
args=(user_id,),
|
args=(user_id,),
|
||||||
@@ -980,7 +989,7 @@ async def run_curator(request: Request) -> dict:
|
|||||||
name=f"skill-curator-manual-{user_id}",
|
name=f"skill-curator-manual-{user_id}",
|
||||||
)
|
)
|
||||||
thread.start()
|
thread.start()
|
||||||
return {"triggered": True, "message": "Curator run started in background"}
|
return {"triggered": True, "reason": None}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Failed to trigger curator: %s", e, exc_info=True)
|
logger.error("Failed to trigger curator: %s", e, exc_info=True)
|
||||||
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
|
raise HTTPException(status_code=500, detail=f"Failed to trigger curator: {str(e)}")
|
||||||
@@ -989,17 +998,19 @@ async def run_curator(request: Request) -> dict:
|
|||||||
class CuratorSubConfig(BaseModel):
|
class CuratorSubConfig(BaseModel):
|
||||||
"""Nested curator configuration mirroring CuratorConfig."""
|
"""Nested curator configuration mirroring CuratorConfig."""
|
||||||
|
|
||||||
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
|
# 与 CuratorConfig.enabled 保持一致:默认关闭,需显式 opt-in。
|
||||||
|
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||||
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
|
interval_hours: int = Field(default=168, ge=1, description="Hours between curator runs.")
|
||||||
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
|
stale_after_days: int = Field(default=30, ge=1, description="Days of inactivity before a skill is marked stale.")
|
||||||
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
|
archive_after_days: int = Field(default=90, ge=1, description="Days of inactivity before a stale skill is archived.")
|
||||||
|
review_timeout_seconds: int = Field(default=600, ge=1, description="Overall timeout (seconds) for the LLM review pass.")
|
||||||
|
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass; null = primary model.")
|
||||||
|
|
||||||
|
|
||||||
class CuratorConfigResponse(BaseModel):
|
class CuratorConfigResponse(BaseModel):
|
||||||
"""Response model exposing the effective skill_evolution config."""
|
"""Response model exposing the effective skill_evolution config."""
|
||||||
|
|
||||||
enabled: bool
|
enabled: bool
|
||||||
moderation_model_name: str | None
|
|
||||||
curator: CuratorSubConfig
|
curator: CuratorSubConfig
|
||||||
|
|
||||||
|
|
||||||
@@ -1011,7 +1022,6 @@ class CuratorConfigRequest(BaseModel):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
|
enabled: bool = Field(..., description="Whether the agent can create and modify skills under skills/custom.")
|
||||||
moderation_model_name: str | None = Field(default=None, description="Optional model name for skill security moderation.")
|
|
||||||
curator: CuratorSubConfig
|
curator: CuratorSubConfig
|
||||||
|
|
||||||
|
|
||||||
@@ -1035,12 +1045,13 @@ def _build_curator_config_response(evo) -> CuratorConfigResponse:
|
|||||||
"""
|
"""
|
||||||
return CuratorConfigResponse(
|
return CuratorConfigResponse(
|
||||||
enabled=evo.enabled,
|
enabled=evo.enabled,
|
||||||
moderation_model_name=evo.moderation_model_name,
|
|
||||||
curator=CuratorSubConfig(
|
curator=CuratorSubConfig(
|
||||||
enabled=evo.curator.enabled,
|
enabled=evo.curator.enabled,
|
||||||
interval_hours=evo.curator.interval_hours,
|
interval_hours=evo.curator.interval_hours,
|
||||||
stale_after_days=evo.curator.stale_after_days,
|
stale_after_days=evo.curator.stale_after_days,
|
||||||
archive_after_days=evo.curator.archive_after_days,
|
archive_after_days=evo.curator.archive_after_days,
|
||||||
|
review_timeout_seconds=evo.curator.review_timeout_seconds,
|
||||||
|
model_name=evo.curator.model_name,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
+8
-5
@@ -4,10 +4,17 @@ from pydantic import BaseModel, Field
|
|||||||
class CuratorConfig(BaseModel):
|
class CuratorConfig(BaseModel):
|
||||||
"""Configuration for the background skill curator."""
|
"""Configuration for the background skill curator."""
|
||||||
|
|
||||||
enabled: bool = Field(default=True, description="Whether the curator runs automatically.")
|
# 默认关闭:curator 仅在用户显式开启时才运行(配合 SkillEvolutionConfig.enabled
|
||||||
|
# 总开关,二者均为安全默认 False —— 未显式 opt-in 则整套技能进化不动作)。
|
||||||
|
enabled: bool = Field(default=False, description="Whether the curator runs automatically.")
|
||||||
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
interval_hours: int = Field(default=168, description="Hours between curator runs (default: 7 days).")
|
||||||
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
stale_after_days: int = Field(default=30, description="Days of inactivity before a skill is marked stale.")
|
||||||
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
archive_after_days: int = Field(default=90, description="Days of inactivity before a stale skill is archived.")
|
||||||
|
# LLM 评审整体超时:agent.invoke 内部可能多轮调 LLM,模型层的单次 request_timeout
|
||||||
|
# 兜不住整体;超过此秒数则中止本次评审,curator 正常收尾并释放运行锁。
|
||||||
|
review_timeout_seconds: int = Field(default=600, description="Overall timeout (seconds) for the LLM review pass.")
|
||||||
|
# curator LLM 评审使用的模型名(对应 config.yaml models[].name);为空则用主模型。
|
||||||
|
model_name: str | None = Field(default=None, description="Model name for the curator LLM review pass. Defaults to the primary chat model.")
|
||||||
|
|
||||||
|
|
||||||
class SkillEvolutionConfig(BaseModel):
|
class SkillEvolutionConfig(BaseModel):
|
||||||
@@ -17,10 +24,6 @@ class SkillEvolutionConfig(BaseModel):
|
|||||||
default=False,
|
default=False,
|
||||||
description="Whether the agent can create and modify skills under skills/custom.",
|
description="Whether the agent can create and modify skills under skills/custom.",
|
||||||
)
|
)
|
||||||
moderation_model_name: str | None = Field(
|
|
||||||
default=None,
|
|
||||||
description="Optional model name for skill security moderation. Defaults to the primary chat model.",
|
|
||||||
)
|
|
||||||
curator: CuratorConfig = Field(
|
curator: CuratorConfig = Field(
|
||||||
default_factory=CuratorConfig,
|
default_factory=CuratorConfig,
|
||||||
description="Configuration for the background skill curator.",
|
description="Configuration for the background skill curator.",
|
||||||
|
|||||||
@@ -67,3 +67,13 @@ class SkillStore(ABC):
|
|||||||
async def count_owned(self, user_id: str) -> int:
|
async def count_owned(self, user_id: str) -> int:
|
||||||
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
|
"""Return the number of skills owned by ``user_id`` (owner_user_id IS NOT NULL)."""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def list_published_names(self) -> set[str]:
|
||||||
|
"""Return the set of skill names with ``published = True`` (any owner).
|
||||||
|
|
||||||
|
Batch alternative to calling ``get_any`` per skill — the curator uses
|
||||||
|
this to filter out published skills in a single query instead of one
|
||||||
|
DB round-trip per candidate skill.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
+3
@@ -127,3 +127,6 @@ class MemorySkillStore(SkillStore):
|
|||||||
|
|
||||||
async def count_owned(self, user_id: str) -> int:
|
async def count_owned(self, user_id: str) -> int:
|
||||||
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
|
return sum(1 for s in self._skills.values() if s.get("owner_user_id") == user_id)
|
||||||
|
|
||||||
|
async def list_published_names(self) -> set[str]:
|
||||||
|
return {name for name, record in self._skills.items() if record.get("published")}
|
||||||
@@ -142,3 +142,9 @@ class SkillRepository(SkillStore):
|
|||||||
stmt = select(func.count()).select_from(SkillRow).where(SkillRow.owner_user_id == user_id)
|
stmt = select(func.count()).select_from(SkillRow).where(SkillRow.owner_user_id == user_id)
|
||||||
async with self._sf() as session:
|
async with self._sf() as session:
|
||||||
return await session.scalar(stmt) or 0
|
return await session.scalar(stmt) or 0
|
||||||
|
|
||||||
|
async def list_published_names(self) -> set[str]:
|
||||||
|
stmt = select(SkillRow.name).where(SkillRow.published.is_(True))
|
||||||
|
async with self._sf() as session:
|
||||||
|
result = await session.execute(stmt)
|
||||||
|
return {row[0] for row in result.all()}
|
||||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
|
import contextvars
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
@@ -25,6 +26,10 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
_CURATOR_STATE_FILENAME = ".curator_state"
|
_CURATOR_STATE_FILENAME = ".curator_state"
|
||||||
|
|
||||||
|
# 周期调度一轮里同时执行的 curator 数量上限。到期用户多时(尤其首次部署所有用户一起
|
||||||
|
# 到期)用线程池排队消费,避免一次性起 N 个线程齐调 LLM 打爆 provider 速率限制。
|
||||||
|
_CURATOR_MAX_CONCURRENCY = 5
|
||||||
|
|
||||||
_T = TypeVar("_T")
|
_T = TypeVar("_T")
|
||||||
|
|
||||||
|
|
||||||
@@ -119,6 +124,81 @@ def _write_curator_state(state: dict, user_id: str | None) -> None:
|
|||||||
logger.debug("Failed to write curator state", exc_info=True)
|
logger.debug("Failed to write curator state", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_state_time(value: str | None) -> datetime | None:
|
||||||
|
"""解析 ``.curator_state`` 里的 ISO 时间字符串,返回 aware datetime(UTC)。
|
||||||
|
|
||||||
|
naive(无时区)字符串按 UTC 补全 —— 与 ``apply_automatic_transitions`` 的处理
|
||||||
|
一致,避免 naive 时间串与 ``datetime.now(UTC)`` 直接比较时抛异常。无法解析时
|
||||||
|
返回 None,由调用方决定降级行为。
|
||||||
|
"""
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(value)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
dt = dt.replace(tzinfo=UTC)
|
||||||
|
return dt
|
||||||
|
|
||||||
|
|
||||||
|
# Curator 运行锁:防止同一用户的 curator 并发执行(定时器 vs 手动触发、连点手动触发)。
|
||||||
|
# 锁状态写进 .curator_state 的 running/started_at 字段,跨 worker 可见(手动请求可能
|
||||||
|
# 落到非调度 worker)。started_at 超过下面的阈值视为进程残留,允许接管以免永久死锁。
|
||||||
|
_CURATOR_STALE_LOCK_HOURS = 6
|
||||||
|
# 上一轮"尝试"过但未成功(last_run_at 始终为空)时的重试间隔 —— 比正常 interval_hours
|
||||||
|
# 短得多,但也不至于每个调度 tick(1h)都重试、每次都烧一份 LLM 配额。
|
||||||
|
_CURATOR_RETRY_HOURS = 6
|
||||||
|
# 进程内串行化运行锁的"读-改-写",保证同进程内的抢锁是原子的(跨进程的极小竞态窗口
|
||||||
|
# 可接受 —— curator 触发频率极低)。
|
||||||
|
_run_lock_guard = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_lock_active(state: dict, now: datetime) -> bool:
|
||||||
|
"""判断 ``.curator_state`` 里的运行锁当前是否有效(running 且未过期)。
|
||||||
|
|
||||||
|
``started_at`` 缺失或不可解析、或距今已超过 ``_CURATOR_STALE_LOCK_HOURS`` 时,
|
||||||
|
视为进程残留留下的死锁,返回 False 以便后续触发能接管。
|
||||||
|
"""
|
||||||
|
if not state.get("running"):
|
||||||
|
return False
|
||||||
|
started_dt = _parse_state_time(state.get("started_at"))
|
||||||
|
if started_dt is None:
|
||||||
|
return False
|
||||||
|
return now < started_dt + timedelta(hours=_CURATOR_STALE_LOCK_HOURS)
|
||||||
|
|
||||||
|
|
||||||
|
def _try_acquire_run_lock(user_id: str, now: datetime) -> bool:
|
||||||
|
"""尝试为 ``user_id`` 抢占 curator 运行锁。
|
||||||
|
|
||||||
|
抢到锁(已把 running=True / started_at 写回)返回 True;已有未过期的运行时返回
|
||||||
|
False。读-改-写在 ``_run_lock_guard`` 下进行,保证同进程内原子。
|
||||||
|
"""
|
||||||
|
with _run_lock_guard:
|
||||||
|
state = _read_curator_state(user_id)
|
||||||
|
if _run_lock_active(state, now):
|
||||||
|
return False
|
||||||
|
state["running"] = True
|
||||||
|
state["started_at"] = now.isoformat()
|
||||||
|
_write_curator_state(state, user_id)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _release_run_lock(user_id: str) -> None:
|
||||||
|
"""释放 curator 运行锁并记录本次尝试时间。无论运行成功或异常都应调用。
|
||||||
|
|
||||||
|
除了把 running 置回 False,还写入 ``last_attempt_at`` —— 用于失败退避:首次运行
|
||||||
|
若一直崩溃,``last_run_at`` 永远为空,``should_run_now`` 改用
|
||||||
|
``last_attempt_at + _CURATOR_RETRY_HOURS`` 判定,避免每个 tick 都重试。
|
||||||
|
成功跑完后 ``last_run_at`` 已被写入,``should_run_now`` 不再看 ``last_attempt_at``。
|
||||||
|
"""
|
||||||
|
with _run_lock_guard:
|
||||||
|
state = _read_curator_state(user_id)
|
||||||
|
state["running"] = False
|
||||||
|
state["last_attempt_at"] = datetime.now(UTC).isoformat()
|
||||||
|
_write_curator_state(state, user_id)
|
||||||
|
|
||||||
|
|
||||||
def _get_curator_config(user_id: str | None = None):
|
def _get_curator_config(user_id: str | None = None):
|
||||||
"""Return the curator config object for ``user_id``.
|
"""Return the curator config object for ``user_id``.
|
||||||
|
|
||||||
@@ -135,6 +215,21 @@ def _get_curator_config(user_id: str | None = None):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _skill_evolution_enabled(user_id: str | None = None) -> bool:
|
||||||
|
"""返回技能进化总开关 ``skill_evolution.enabled``(已合并 per-user 覆盖)。
|
||||||
|
|
||||||
|
这是整套技能进化功能的总闸:关闭时 curator 完全不动作。任何读取异常都按
|
||||||
|
"关闭"处理(fail-safe)—— 宁可不跑,也不在配置不可读时偷偷运行。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from deerflow.config.skill_evolution_runtime import load_skill_evolution_config_for_user
|
||||||
|
|
||||||
|
skill_evo = load_skill_evolution_config_for_user(user_id)
|
||||||
|
return bool(getattr(skill_evo, "enabled", False))
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
|
def _list_user_owned_skill_names(user_id: str) -> set[str] | None:
|
||||||
"""Return skill names owned by ``user_id`` (used to filter transitions).
|
"""Return skill names owned by ``user_id`` (used to filter transitions).
|
||||||
|
|
||||||
@@ -231,12 +326,39 @@ def _is_skill_published(skill_name: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _list_published_skill_names() -> set[str]:
|
||||||
|
"""一次性返回所有 ``published=True`` 的技能名集合。
|
||||||
|
|
||||||
|
取代"每个技能调一次 ``_is_skill_published``"的逐条查库 —— curator 一轮里会遍历
|
||||||
|
所有候选技能,逐条查会产生 ≈2N 次独立 event loop + DB 往返。出错时返回空集
|
||||||
|
(fail-open:与 ``_is_skill_published`` 出错返回 False 语义一致,即 DB 不可用时
|
||||||
|
所有技能都视为未发布,回退到 curator 的 legacy 行为)。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from deerflow.persistence.engine import get_session_factory
|
||||||
|
from deerflow.persistence.skills import make_skill_store
|
||||||
|
|
||||||
|
factory = get_session_factory()
|
||||||
|
if factory is None:
|
||||||
|
return set()
|
||||||
|
store = make_skill_store(factory)
|
||||||
|
|
||||||
|
async def _run() -> set[str]:
|
||||||
|
return await store.list_published_names()
|
||||||
|
|
||||||
|
return _run_async_blocking(_run())
|
||||||
|
except Exception:
|
||||||
|
logger.debug("_list_published_skill_names failed", exc_info=True)
|
||||||
|
return set()
|
||||||
|
|
||||||
|
|
||||||
def _eligible_for_curation(
|
def _eligible_for_curation(
|
||||||
name: str,
|
name: str,
|
||||||
entry: dict,
|
entry: dict,
|
||||||
*,
|
*,
|
||||||
owned_set: set[str] | None,
|
owned_set: set[str] | None,
|
||||||
referenced_set: set[str],
|
referenced_set: set[str],
|
||||||
|
published_set: set[str],
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Three-condition gate for whether curator may touch a skill.
|
"""Three-condition gate for whether curator may touch a skill.
|
||||||
|
|
||||||
@@ -244,11 +366,15 @@ def _eligible_for_curation(
|
|||||||
1. source == "agent" (caller is expected to have already filtered, but
|
1. source == "agent" (caller is expected to have already filtered, but
|
||||||
we double-check here for safety)
|
we double-check here for safety)
|
||||||
2. NOT referenced by any agent's skills config
|
2. NOT referenced by any agent's skills config
|
||||||
3. NOT published (queried via SkillStore)
|
3. NOT published
|
||||||
|
|
||||||
``owned_set`` is the set of skill names owned by the running user. When
|
``owned_set`` is the set of skill names owned by the running user. When
|
||||||
None, the legacy "process every agent-created skill" fallback applies
|
None, the legacy "process every agent-created skill" fallback applies
|
||||||
for ownership only (the other two conditions still hold).
|
for ownership only (the other two conditions still hold).
|
||||||
|
|
||||||
|
``published_set`` is the set of published skill names, pre-fetched once by
|
||||||
|
the caller via ``_list_published_skill_names()`` — avoids a per-skill DB
|
||||||
|
round-trip.
|
||||||
"""
|
"""
|
||||||
if entry.get("source") != "agent":
|
if entry.get("source") != "agent":
|
||||||
return False
|
return False
|
||||||
@@ -256,7 +382,7 @@ def _eligible_for_curation(
|
|||||||
return False
|
return False
|
||||||
if name in referenced_set:
|
if name in referenced_set:
|
||||||
return False
|
return False
|
||||||
if _is_skill_published(name):
|
if name in published_set:
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -264,21 +390,52 @@ def _eligible_for_curation(
|
|||||||
def should_run_now(user_id: str) -> bool:
|
def should_run_now(user_id: str) -> bool:
|
||||||
"""Return True if the curator interval has elapsed for ``user_id`` and it
|
"""Return True if the curator interval has elapsed for ``user_id`` and it
|
||||||
is not paused."""
|
is not paused."""
|
||||||
|
# 总开关:技能进化整体关闭时,curator 一律不跑。
|
||||||
|
if not _skill_evolution_enabled(user_id):
|
||||||
|
return False
|
||||||
curator_cfg = _get_curator_config(user_id)
|
curator_cfg = _get_curator_config(user_id)
|
||||||
if curator_cfg is None or not getattr(curator_cfg, "enabled", True):
|
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
|
||||||
return False
|
return False
|
||||||
state = _read_curator_state(user_id)
|
state = _read_curator_state(user_id)
|
||||||
if state.get("paused"):
|
if state.get("paused"):
|
||||||
return False
|
return False
|
||||||
|
# 运行锁:已有 curator 在跑(且锁未过期)时不再触发新的一轮。
|
||||||
|
if _run_lock_active(state, datetime.now(UTC)):
|
||||||
|
return False
|
||||||
last_run = state.get("last_run_at")
|
last_run = state.get("last_run_at")
|
||||||
if not last_run:
|
if not last_run:
|
||||||
return True
|
# 从未成功跑完过。若此前已尝试过(last_attempt_at 存在),按较短的重试间隔
|
||||||
|
# 退避 —— 避免首次运行持续失败时每个 tick(1h)都重试、每次烧一份 LLM 配额。
|
||||||
|
attempt_dt = _parse_state_time(state.get("last_attempt_at"))
|
||||||
|
if attempt_dt is None:
|
||||||
|
return True
|
||||||
|
return datetime.now(UTC) >= attempt_dt + timedelta(hours=_CURATOR_RETRY_HOURS)
|
||||||
interval_hours = getattr(curator_cfg, "interval_hours", 168)
|
interval_hours = getattr(curator_cfg, "interval_hours", 168)
|
||||||
try:
|
last_dt = _parse_state_time(last_run)
|
||||||
last_dt = datetime.fromisoformat(last_run)
|
if last_dt is None:
|
||||||
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
|
# 时间串损坏 —— 视同"从未跑过",允许重跑并重写正确时间戳,而不是像旧逻辑那样
|
||||||
except Exception:
|
# except 返回 False 导致该用户永久不再触发。
|
||||||
return False
|
return True
|
||||||
|
return datetime.now(UTC) >= last_dt + timedelta(hours=interval_hours)
|
||||||
|
|
||||||
|
|
||||||
|
def manual_trigger_blocked_reason(user_id: str) -> str | None:
|
||||||
|
"""手动触发 curator 的预检。返回 None 表示可以跑;否则返回阻塞原因代码:
|
||||||
|
``evolution_disabled`` / ``curator_disabled`` / ``already_running``。
|
||||||
|
|
||||||
|
供 ``POST /api/curator/run`` 在派发后台线程**前**同步判定 —— 后台线程
|
||||||
|
(``_curator_thread_target``)本身也会再查一遍,这里只是为了让前端拿到准确反馈,
|
||||||
|
而不是无论开关状态如何都提示"已开始"。
|
||||||
|
"""
|
||||||
|
if not _skill_evolution_enabled(user_id):
|
||||||
|
return "evolution_disabled"
|
||||||
|
curator_cfg = _get_curator_config(user_id)
|
||||||
|
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
|
||||||
|
return "curator_disabled"
|
||||||
|
state = _read_curator_state(user_id)
|
||||||
|
if _run_lock_active(state, datetime.now(UTC)):
|
||||||
|
return "already_running"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
|
def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> dict:
|
||||||
@@ -303,6 +460,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
|
|||||||
|
|
||||||
owned = _list_user_owned_skill_names(user_id)
|
owned = _list_user_owned_skill_names(user_id)
|
||||||
referenced = _collect_referenced_skill_names()
|
referenced = _collect_referenced_skill_names()
|
||||||
|
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
|
||||||
raw_entries = agent_created_entries()
|
raw_entries = agent_created_entries()
|
||||||
|
|
||||||
skipped_referenced = 0
|
skipped_referenced = 0
|
||||||
@@ -314,7 +472,7 @@ def apply_automatic_transitions(user_id: str, now: datetime | None = None) -> di
|
|||||||
if name in referenced:
|
if name in referenced:
|
||||||
skipped_referenced += 1
|
skipped_referenced += 1
|
||||||
continue
|
continue
|
||||||
if _is_skill_published(name):
|
if name in published:
|
||||||
skipped_published += 1
|
skipped_published += 1
|
||||||
continue
|
continue
|
||||||
eligible[name] = entry
|
eligible[name] = entry
|
||||||
@@ -377,9 +535,10 @@ def _render_candidate_list(user_id: str) -> str:
|
|||||||
|
|
||||||
owned = _list_user_owned_skill_names(user_id)
|
owned = _list_user_owned_skill_names(user_id)
|
||||||
referenced = _collect_referenced_skill_names()
|
referenced = _collect_referenced_skill_names()
|
||||||
|
published = _list_published_skill_names() # 一次性查出,避免逐技能查库
|
||||||
entries: dict[str, dict] = {}
|
entries: dict[str, dict] = {}
|
||||||
for k, v in all_entries().items():
|
for k, v in all_entries().items():
|
||||||
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced):
|
if not _eligible_for_curation(k, v, owned_set=owned, referenced_set=referenced, published_set=published):
|
||||||
continue
|
continue
|
||||||
if v.get("state") == STATE_ARCHIVED:
|
if v.get("state") == STATE_ARCHIVED:
|
||||||
continue
|
continue
|
||||||
@@ -666,7 +825,11 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dic
|
|||||||
from deerflow.models import create_chat_model
|
from deerflow.models import create_chat_model
|
||||||
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
from deerflow.tools.builtins.skill_tools import skill_list_tool, skill_view_tool
|
||||||
|
|
||||||
model = create_chat_model()
|
curator_cfg = _get_curator_config(user_id)
|
||||||
|
# curator 评审用的模型:per-user 可配 curator.model_name(对应 config.yaml
|
||||||
|
# models[].name);为空则用主模型。
|
||||||
|
model_name = getattr(curator_cfg, "model_name", None) if curator_cfg else None
|
||||||
|
model = create_chat_model(name=model_name)
|
||||||
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
|
# 用 curator 受限版 skill_manage(屏蔽 delete),不用通用版 —— 见 _make_curator_skill_manage_tool。
|
||||||
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
|
tools = [skill_list_tool, skill_view_tool, _make_curator_skill_manage_tool(), _make_skill_archive_tool(user_id)]
|
||||||
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
|
# Reuse the lead-agent error middleware: when the LLM mis-calls a tool
|
||||||
@@ -677,9 +840,36 @@ def _run_llm_review(candidate_list: str, *, user_id: str) -> tuple[str, list[dic
|
|||||||
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
|
# middleware kwarg — use LangChain 1.x's ``create_agent`` which does.
|
||||||
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
|
agent = create_agent(model=model, tools=tools, middleware=[ToolErrorHandlingMiddleware()])
|
||||||
|
|
||||||
curator_log.record("info", "LLM 评审:开始调用 React Agent", user_id=user_id)
|
timeout_seconds = getattr(curator_cfg, "review_timeout_seconds", 600) if curator_cfg else 600
|
||||||
|
curator_log.record(
|
||||||
|
"info",
|
||||||
|
f"LLM 评审:开始调用 React Agent(模型 {model_name or '主模型'},整体超时 {timeout_seconds}s)",
|
||||||
|
user_id=user_id,
|
||||||
|
)
|
||||||
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
|
prompt = _CURATOR_REVIEW_PROMPT + f"\n\nSkill candidates for this pass:\n{candidate_list}"
|
||||||
result = agent.invoke({"messages": [HumanMessage(content=prompt)]})
|
|
||||||
|
# 给整个 React Agent 评审套一个总超时:agent.invoke 内部可能多轮调 LLM,模型层的
|
||||||
|
# 单次 request_timeout 兜不住整体。超时后本 curator 线程能正常收尾、释放运行锁,
|
||||||
|
# 而不是无限期挂起。注意:Python 无法强杀线程 —— 超时后那个 agent.invoke 线程会
|
||||||
|
# 成为孤儿(最终被模型层 request_timeout 或自然返回所终结),但不再拖住 curator。
|
||||||
|
# copy_context():把 current_user 等 ContextVar 传进工作线程,否则工具调用会把
|
||||||
|
# user_id 误解析为 DEFAULT_USER_ID。
|
||||||
|
ctx = contextvars.copy_context()
|
||||||
|
pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="curator-llm")
|
||||||
|
future = pool.submit(ctx.run, agent.invoke, {"messages": [HumanMessage(content=prompt)]})
|
||||||
|
try:
|
||||||
|
result = future.result(timeout=timeout_seconds)
|
||||||
|
except concurrent.futures.TimeoutError:
|
||||||
|
curator_log.record(
|
||||||
|
"warning",
|
||||||
|
f"LLM 评审超时(超过 {timeout_seconds}s),已中止本次评审",
|
||||||
|
user_id=user_id,
|
||||||
|
)
|
||||||
|
logger.warning("Curator[%s] LLM review timed out after %ss", user_id, timeout_seconds)
|
||||||
|
return "", []
|
||||||
|
finally:
|
||||||
|
# wait=False:超时时不阻塞等待挂死的 invoke 线程。
|
||||||
|
pool.shutdown(wait=False)
|
||||||
|
|
||||||
messages = result.get("messages", [])
|
messages = result.get("messages", [])
|
||||||
tool_actions = _extract_tool_actions(messages)
|
tool_actions = _extract_tool_actions(messages)
|
||||||
@@ -876,6 +1066,23 @@ def _curator_thread_target(user_id: str) -> None:
|
|||||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||||
from deerflow.skills import curator_log
|
from deerflow.skills import curator_log
|
||||||
|
|
||||||
|
# 总开关 + curator 子开关:任一关闭则直接跳过。手动触发 /api/curator/run 也经由
|
||||||
|
# 本函数,因此手动触发同样受总开关与 curator.enabled 约束。
|
||||||
|
if not _skill_evolution_enabled(user_id):
|
||||||
|
logger.info("Curator[%s]: 技能进化总开关关闭,跳过", user_id)
|
||||||
|
return
|
||||||
|
curator_cfg = _get_curator_config(user_id)
|
||||||
|
if curator_cfg is None or not getattr(curator_cfg, "enabled", False):
|
||||||
|
logger.info("Curator[%s]: curator 未启用,跳过", user_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 运行锁:防止同一用户的 curator 并发(定时器 vs 手动触发、连点手动触发)。抢不到
|
||||||
|
# 锁说明已有一轮在跑,直接跳过本次触发。
|
||||||
|
if not _try_acquire_run_lock(user_id, datetime.now(UTC)):
|
||||||
|
logger.info("Curator[%s]: 已有 curator 正在运行,跳过本次触发", user_id)
|
||||||
|
curator_log.record("info", "已有 curator 正在运行,跳过本次触发", user_id=user_id)
|
||||||
|
return
|
||||||
|
|
||||||
# Bind the per-thread user context for the entire curator run so every
|
# Bind the per-thread user context for the entire curator run so every
|
||||||
# downstream tool / repository call resolves the correct user_id.
|
# downstream tool / repository call resolves the correct user_id.
|
||||||
token = set_current_user(_CuratorUser(user_id))
|
token = set_current_user(_CuratorUser(user_id))
|
||||||
@@ -886,6 +1093,8 @@ def _curator_thread_target(user_id: str) -> None:
|
|||||||
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
|
curator_log.record("error", f"策展后台任务失败:{exc}", user_id=user_id)
|
||||||
finally:
|
finally:
|
||||||
reset_current_user(token)
|
reset_current_user(token)
|
||||||
|
# 无论成功或异常,必定释放运行锁。
|
||||||
|
_release_run_lock(user_id)
|
||||||
|
|
||||||
|
|
||||||
async def maybe_run_curator() -> None:
|
async def maybe_run_curator() -> None:
|
||||||
@@ -911,20 +1120,42 @@ async def maybe_run_curator() -> None:
|
|||||||
logger.exception("Curator: failed to enumerate users at startup")
|
logger.exception("Curator: failed to enumerate users at startup")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
due_users: list[str] = []
|
||||||
for user_id in user_ids:
|
for user_id in user_ids:
|
||||||
try:
|
try:
|
||||||
if not should_run_now(user_id):
|
if should_run_now(user_id):
|
||||||
continue
|
due_users.append(user_id)
|
||||||
logger.info("Curator[%s]: interval elapsed, spawning background thread", user_id)
|
|
||||||
thread = threading.Thread(
|
|
||||||
target=_curator_thread_target,
|
|
||||||
args=(user_id,),
|
|
||||||
daemon=True,
|
|
||||||
name=f"skill-curator-{user_id}",
|
|
||||||
)
|
|
||||||
thread.start()
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Curator: failed to spawn for user %s", user_id)
|
logger.exception("Curator: should_run_now check failed for user %s", user_id)
|
||||||
|
_dispatch_curator_runs(due_users)
|
||||||
|
|
||||||
|
|
||||||
|
def _dispatch_curator_runs(due_users: list[str]) -> None:
|
||||||
|
"""用线程池(并发上限 ``_CURATOR_MAX_CONCURRENCY``)派发到期用户的 curator 执行。
|
||||||
|
|
||||||
|
到期用户排队消费,而非一次性全部起线程齐调 LLM —— 防首次部署所有用户同时到期时
|
||||||
|
的惊群。``shutdown(wait=False)``:提交后立即返回,线程池在所有任务跑完后自行回收。
|
||||||
|
抽成独立函数便于对并发上限做单元测试。
|
||||||
|
"""
|
||||||
|
if not due_users:
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Curator: %d 个用户到期,以并发上限 %d 派发执行",
|
||||||
|
len(due_users),
|
||||||
|
_CURATOR_MAX_CONCURRENCY,
|
||||||
|
)
|
||||||
|
executor = concurrent.futures.ThreadPoolExecutor(
|
||||||
|
max_workers=_CURATOR_MAX_CONCURRENCY,
|
||||||
|
thread_name_prefix="skill-curator",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
for user_id in due_users:
|
||||||
|
executor.submit(_curator_thread_target, user_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Curator: failed to submit curator tasks")
|
||||||
|
finally:
|
||||||
|
executor.shutdown(wait=False)
|
||||||
|
|
||||||
|
|
||||||
def get_curator_status(user_id: str) -> dict:
|
def get_curator_status(user_id: str) -> dict:
|
||||||
|
|||||||
+4
-2
@@ -50,8 +50,10 @@ async def scan_skill_content(content: str, *, executable: bool = False, location
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
config = app_config or get_app_config()
|
config = app_config or get_app_config()
|
||||||
model_name = config.skill_evolution.moderation_model_name
|
# 安全审核复用 curator 的评审模型配置(skill_evolution.curator.model_name):
|
||||||
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config) if model_name else create_chat_model(thinking_enabled=False, app_config=config)
|
# 二者用同一个模型。未配置时 model_name 为 None,create_chat_model 回退主模型。
|
||||||
|
model_name = config.skill_evolution.curator.model_name
|
||||||
|
model = create_chat_model(name=model_name, thinking_enabled=False, app_config=config)
|
||||||
response = await model.ainvoke(
|
response = await model.ainvoke(
|
||||||
[
|
[
|
||||||
{"role": "system", "content": rubric},
|
{"role": "system", "content": rubric},
|
||||||
|
|||||||
@@ -25,12 +25,12 @@ def test_load_overrides_returns_none_when_missing(overrides_path):
|
|||||||
def test_save_and_load_overrides_roundtrip(overrides_path):
|
def test_save_and_load_overrides_roundtrip(overrides_path):
|
||||||
payload = {
|
payload = {
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
"moderation_model_name": "moderation-mini",
|
|
||||||
"curator": {
|
"curator": {
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
"interval_hours": 12,
|
"interval_hours": 12,
|
||||||
"stale_after_days": 7,
|
"stale_after_days": 7,
|
||||||
"archive_after_days": 30,
|
"archive_after_days": 30,
|
||||||
|
"model_name": "curator-mini",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
written = skill_evolution_runtime.save_overrides(payload)
|
written = skill_evolution_runtime.save_overrides(payload)
|
||||||
|
|||||||
@@ -501,7 +501,8 @@ def test_curator_skips_published_skill(isolated_paths, monkeypatch) -> None:
|
|||||||
|
|
||||||
monkeypatch.setattr(curator, "_list_user_owned_skill_names", lambda uid: {"public-tool"})
|
monkeypatch.setattr(curator, "_list_user_owned_skill_names", lambda uid: {"public-tool"})
|
||||||
monkeypatch.setattr(curator, "_collect_referenced_skill_names", lambda: set())
|
monkeypatch.setattr(curator, "_collect_referenced_skill_names", lambda: set())
|
||||||
monkeypatch.setattr(curator, "_is_skill_published", lambda name: name == "public-tool")
|
# curator 现在用批量接口一次性查 published 集合,而非逐技能 _is_skill_published。
|
||||||
|
monkeypatch.setattr(curator, "_list_published_skill_names", lambda: {"public-tool"})
|
||||||
_isolated_skills_root(isolated_paths, monkeypatch)
|
_isolated_skills_root(isolated_paths, monkeypatch)
|
||||||
_seed_old_agent_skill("public-tool")
|
_seed_old_agent_skill("public-tool")
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user