diff --git a/frontend-web/src/components/workspace/agents/agent-module-sidebar.tsx b/frontend-web/src/components/workspace/agents/agent-module-sidebar.tsx index d0712ccb..b9e48c2a 100644 --- a/frontend-web/src/components/workspace/agents/agent-module-sidebar.tsx +++ b/frontend-web/src/components/workspace/agents/agent-module-sidebar.tsx @@ -211,6 +211,13 @@ export function AgentModuleSidebar({ { + try { + sessionStorage.setItem(`agent-module:${moduleKey}:last-agent`, agent.id); + } catch { + // Ignore storage failures; route navigation is the source of truth. + } + }} className={cn( "group relative flex w-full items-start gap-2.5 rounded-xl border px-2.5 py-2 transition-colors", isActive diff --git a/frontend-web/src/core/auth/yuser-login.ts b/frontend-web/src/core/auth/yuser-login.ts index a43405bb..5787963d 100644 --- a/frontend-web/src/core/auth/yuser-login.ts +++ b/frontend-web/src/core/auth/yuser-login.ts @@ -84,6 +84,7 @@ function writeCachedYUserInfo(yUserId: string, info: Record): v function buildExternalUserInfo( yUserId: string, result: Record, + access_token: string | null | undefined, previous: Record, cached: Record, ): Record { @@ -101,6 +102,12 @@ function buildExternalUserInfo( return next; } + const queryAccessToken = normalizeExternalAccessToken(access_token); + if (queryAccessToken) { + next.accessToken = queryAccessToken; + return next; + } + const previousYUserId = readStoredYUserId(previous); const previousAccessToken = previousYUserId === yUserId ? normalizeExternalAccessToken(previous.accessToken) : ""; @@ -121,6 +128,7 @@ function buildExternalUserInfo( export async function loginByYUserId( yUserId: string, token?: string | null, + access_token?: string | null, ): Promise { if (token) { authStorageSet(TOOLS_TOKEN_STORAGE_KEY, token); @@ -134,12 +142,11 @@ export async function loginByYUserId( // consumer 返回 { accessToken, gateDepartName, typeName, userName }。 // 一并存入路由参数 yUserId，使「应用管理」里 source=yUserId 的路由参数能取到（与 // source=userName/gateDepartName/typeName 直接取接口值统一走 userInfo[source]）。 - // 注意:接口也会返回 accessToken,但 XK/PG 及 source=accessToken 要用路由透传的 - // access_token,故登录流程随后会用 persistAccessToken 覆盖 userInfo.accessToken。 + // 注意:接口返回的 accessToken 优先；登录 URL 的 access_token 只在接口值为空时兜底。 const result = res.data ?? {}; const previous = readExternalUserInfo(); const cached = readCachedYUserInfo(yUserId); - const next = buildExternalUserInfo(yUserId, result, previous, cached); + const next = buildExternalUserInfo(yUserId, result, access_token, previous, cached); authStorageSet(EXTERNAL_USER_INFO_STORAGE_KEY, JSON.stringify(next)); writeCachedYUserInfo(yUserId, next); } @@ -149,7 +156,7 @@ export function persistToolsToken(token: string): void { } /** - * 把内网登录路由参数 access_token 合并写入 userInfo.accessToken（覆盖接口返回的同名值）。 + * 把内网登录路由参数 access_token 合并写入 userInfo.accessToken。 * 据此拼参的有：XK/PG 中任务 iframe（XK 用 accessToken、PG 用 access_token，二者均读 * external-links.getAccessToken() = userInfo.accessToken）；以及「应用管理」里 * source=accessToken 的轻应用路由参数（resolveRouteParamValue 取 userInfo.accessToken）。 diff --git a/frontend-web/src/pages/LoginPage.tsx b/frontend-web/src/pages/LoginPage.tsx index 7ca382fb..75dc8a44 100644 --- a/frontend-web/src/pages/LoginPage.tsx +++ b/frontend-web/src/pages/LoginPage.tsx @@ -75,7 +75,7 @@ export default function LoginPage() { const yUserId = searchParams.get('yUserId')?.trim() ?? ''; const toolToken = searchParams.get('token')?.trim() ?? ''; // 内网登录透传的访问令牌:存入 userInfo.accessToken,供 XK/PG 中任务 iframe 拼参 - const accessToken = searchParams.get('access_token')?.trim() ?? ''; + const access_token = searchParams.get('access_token')?.trim() ?? ''; // 用户名直登口令门:开启后端开关时,口令从 ?password= 透传 const gatePassword = searchParams.get('password') ?? ''; // token 换登录:?authToken= 传上游 token(与 tools_token 的 ?token= 区分开) @@ -182,7 +182,7 @@ export default function LoginPage() { if (onlineTicket) { deerflowUsername = await resolveUsernameByOnlineTicket(onlineTicket); } else if (yUserId) { - await loginByYUserId(yUserId, toolToken || null); + await loginByYUserId(yUserId, toolToken || null, access_token || null); } else if (toolToken) { persistToolsToken(toolToken); } @@ -212,9 +212,11 @@ export default function LoginPage() { } // 标准登录把用户名写入 userInfo，供顶栏用户区显示（已有则不覆盖） setStoredUserName(loginUserName); - // 内网透传的 access_token 合并进 userInfo（在 loginByYUserId 覆盖之后）， - // XK/PG 中任务 iframe 据此取 accessToken - persistAccessToken(accessToken); + // 非 yUserId 登录仍可直接持久化 access_token；yUserId 登录已在 loginByYUserId 内按 + // “接口 accessToken 优先，URL access_token 仅兜底”的规则处理。 + if (!yUserId) { + persistAccessToken(access_token); + } // 嵌入模式不写 sidebarEnabled（localStorage 的「简洁/全量」偏好会跨标签污染正常页）； // 去 chrome 由按标签页的嵌入会话（isEmbedActive）驱动，见 PageLayout。 if (!isEmbed) persistSidebarEnabled(sidebarEnabled); @@ -273,7 +275,7 @@ export default function LoginPage() { sidebarEnabled, yUserId, toolToken, - accessToken, + access_token, onlineTicket, authToken, gatePassword, diff --git a/frontend-web/src/shims/next-navigation.ts b/frontend-web/src/shims/next-navigation.ts index b3bf4941..7ab31dde 100644 --- a/frontend-web/src/shims/next-navigation.ts +++ b/frontend-web/src/shims/next-navigation.ts @@ -8,21 +8,21 @@ import { import { RouteBaseContext } from "@/core/route-base"; -export function useRouter() { - const navigate = useNavigate(); - const base = useContext(RouteBaseContext); - - return useMemo( - () => ({ - push: (href: string) => - navigate(href.replace("/page/workspace", base)), - replace: (href: string) => - navigate(href.replace("/page/workspace", base), { replace: true }), - back: () => navigate(-1), - forward: () => navigate(1), - refresh: () => window.location.reload(), - prefetch: async () => {}, - }), +export function useRouter() { + const navigate = useNavigate(); + const base = useContext(RouteBaseContext); + const resolveHref = (href: string) => + href.startsWith(base) ? href : href.replace("/page/workspace", base); + + return useMemo( + () => ({ + push: (href: string) => navigate(resolveHref(href)), + replace: (href: string) => navigate(resolveHref(href), { replace: true }), + back: () => navigate(-1), + forward: () => navigate(1), + refresh: () => window.location.reload(), + prefetch: async () => {}, + }), [navigate, base], ); } diff --git a/skills-data/xdfx-full-flow-skill-guide.md b/skills-data/xdfx-full-flow-skill-guide.md index 88dd1805..9621f0c3 100644 --- a/skills-data/xdfx-full-flow-skill-guide.md +++ b/skills-data/xdfx-full-flow-skill-guide.md @@ -6,10 +6,10 @@ 1. 受众目标分析，选出最优 3 个受众并入库。 2. 关键传播节点分析，跨受众去重后选出最优 3 个节点并入库。 -3. 受众脆弱点分析并入库。 -4. 关键传播节点脆弱点分析并入库。 -5. 受众策略分析并入库。 -6. 关键传播节点策略分析并入库。 +3. 从已入库受众中默认选前 2 个继续，每个生成最多 3 个脆弱点并入库。 +4. 从已入库关键传播节点中默认选前 2 个继续，每个生成最多 3 个脆弱点并入库。 +5. 对上述选中的每个受众默认生成最多 3 个策略并入库。 +6. 对上述选中的每个关键传播节点默认生成最多 3 个策略并入库。 7. 同步前端步骤状态，保证原 `item-aiceh` 页面步骤条可继续识别。 ## 技能位置 @@ -41,9 +41,9 @@ skills-data/xdfx-single-agent-full-flow/scripts/xdfx_flow.py } ``` -默认会自动处理登录态：当命令没有传 `Authorization`，或某个接口返回 401 时，脚本会用固定账号 `admin`、口令 `ch@user` 调用 `POST /api/v1/auth/login/username?password=ch%40user`，请求体为 `{"username":"admin"}`。拿到 `access_token` 后写入 `Authorization: Bearer ...` 并重试原接口一次。 +默认会自动处理登录态：脚本启动时先 `import scripts/xdfx_auth.py`，用固定账号 `admin`、口令 `ch@user` 调用 `POST /api/v1/auth/login/username?password=ch%40user`，请求体为 `{"username":"admin"}`。拿到 `access_token` 后写入内部 `Authorization: Bearer ...`，再调用第一步业务接口；不要通过任何命令行参数传 token。 -默认关闭 HTTPS 证书校验，用于兼容 UAT/内网自签证书；如果部署环境证书链正常，可以追加 `--verify-ssl`。如果已经有可用 token，也可以继续追加 `--auth-token TOKEN` 或 `--header "Authorization: Bearer TOKEN"`，脚本会优先使用传入 token。 +默认关闭 HTTPS 证书校验，用于兼容 UAT/内网自签证书；如果部署环境证书链正常，可以追加 `--verify-ssl`。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次。 ## 执行命令 @@ -60,30 +60,44 @@ python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-fl `--mode full` 只作为无人值守或排障兜底，不作为默认执行方式；外层工具通常会等长命令结束后才返回输出，无法保证真实逐步反馈。逐步命令共享同一个 `xdfx-flow-state.json`，不会影响 id 传递、断点续跑和最终入库。 +如果需要一条命令完成逐步编排，可以使用包装脚本。它内部仍按 step1-step6 调用 `xdfx_flow.py`，并逐行转发反馈： + +```bash +python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md +``` + 每完成一步都会输出一行： ```text -STEP_FEEDBACK {...} +步骤反馈 {...} ``` -调用技能的智能体必须在每个 step 命令结束后读取 `STEP_FEEDBACK`，给用户一次反馈，再执行下一条 step 命令。 +调用技能的智能体必须在每个 step 命令结束后读取 `步骤反馈`，给用户一次反馈，再执行下一条 step 命令。 每次执行具体操作或调用接口前，脚本会输出： ```text -OP_FEEDBACK {...} +操作反馈 {...} ``` -调用技能的智能体必须把 `operation`、`method` 和 `url` 转述给用户，例如： +调用技能的智能体必须把 `操作`、`请求方式` 和 `接口地址` 转述给用户，例如： ```text 正在保存受众目标，调用 POST https://.../api/task/task-action-design。 ``` +每次保存接口成功拿到 `id` 后，脚本会输出： + +```text +入库反馈 {...} +``` + +调用技能的智能体必须把 `入库对象类型`、`入库对象名称` 和 `返回ID` 转述给用户，例如“已入库受众目标：xxx，designId=123”或“已入库受众脆弱点：xxx，id=456”。 + 如果报错，脚本会立即输出： ```text -ERROR_FEEDBACK {...} +错误反馈 {...} ``` 调用技能的智能体必须立刻告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 @@ -91,10 +105,10 @@ ERROR_FEEDBACK {...} 如果模型 JSON 经过自动提取和修复仍无法解析，脚本会输出： ```text -JSON_PARSE_REVIEW {...} +JSON解析复核 {...} ``` -调用技能的智能体必须查看 `responseExcerpt`，判断是否能识别真正 JSON；无法确认时重跑当前步骤或重新调用对应模型接口。 +调用技能的智能体必须查看 `响应片段`，判断是否能识别真正 JSON；无法确认时重跑当前步骤或重新调用对应模型接口。 只检查脚本与参数、不调用真实接口： @@ -130,6 +144,8 @@ python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-fl - `task-action-design-strategy-relation` - `task/task-action/{actionId}` 的 `step` 字段 +策略信息形式/模态会按前端 `modelType` 字典保存为 value 逗号串，例如模型输出“图片,视频”时，脚本先查 `sys/sys-dict-data` 再写入 `task-action-message.modelType`，不会直接保存中文文字。 + ## 输出文件 执行后查看： @@ -139,27 +155,29 @@ xdfx-flow-state.json xdfx-flow-summary.md ``` -`xdfx-flow-state.json` 保存断点状态、接口返回 id、模型解析结果与错误明细。 +`xdfx-flow-state.json` 保存断点状态、接口返回 ID、模型解析结果与错误明细。 `xdfx-flow-summary.md` 是给用户看的执行摘要。 ## 每步反馈 -脚本会输出三类反馈行： +脚本会输出五类反馈行： -- `OP_FEEDBACK`：当前正在做什么、调用哪个接口。 -- `ERROR_FEEDBACK`：当前错误的阶段、接口、状态码、请求摘要和响应摘要。 -- `JSON_PARSE_REVIEW`：模型返回内容无法稳定解析为 JSON，需要执行智能体复核是否重试。 -- `STEP_FEEDBACK`：当前步骤最终状态和累计入库数量。 +- `操作反馈`：当前正在做什么、调用哪个接口。 +- `入库反馈`：当前已入库的对象类型、名称和返回 ID。 +- `错误反馈`：当前错误的阶段、接口、状态码、请求摘要和响应摘要。 +- `JSON解析复核`：模型返回内容无法稳定解析为 JSON，需要执行智能体复核是否重试。 +- `步骤反馈`：当前步骤最终状态和累计入库数量。 -`STEP_FEEDBACK` 中包含： +`步骤反馈` 中包含： -- `step`：步骤编号。 -- `label`：步骤名称。 -- `status`：`ok`、`partial` 或 `skipped`。 -- `counts`：当前累计入库数量。 -- `newErrors`：本步骤新增错误数。 -- `totalErrors`：累计错误数。 +- `步骤`：步骤编号。 +- `步骤名称`：步骤名称。 +- `状态`：`成功`、`部分完成` 或 `已跳过`。 +- `执行状态`：中文执行状态。 +- `累计入库数量`：当前累计入库数量。 +- `本步骤新增错误数`：本步骤新增错误数。 +- `累计错误数`：累计错误数。 智能体反馈用户时建议使用这种格式： @@ -167,20 +185,23 @@ xdfx-flow-summary.md Step1 受众目标分析完成：当前已入库受众 X 个，本步骤新增错误 Y 个。 ``` -如果 `status=partial`，要补一句“该步骤部分完成”，并说明稍后最终汇总会列出错误明细。 +如果 `状态` 为 `部分完成`，要补一句“该步骤部分完成”，并说明稍后最终汇总会列出错误明细。 -如果 `status=skipped`，要说明跳过原因。 +如果 `状态` 为 `已跳过`，要说明跳过原因。 ## 容错规则 - 单个受众、账号、传播节点、脆弱点或策略失败，不中断整个流程。 - 保存接口必须返回 `id`，否则不会进入下游步骤。 +- 受众目标默认入库 3 个，关键传播节点默认入库 3 个。 +- 下游脆弱点/策略默认只选择前 2 个受众和前 2 个传播节点继续，可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。 +- 每个被选中的对象默认最多生成 3 个脆弱点、3 个策略。 - 模型流式 `answer` 会自动拼接。 - 流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时，会自动再调用一次；仍失败才写入 `errors`。 - 模型输出会自动去掉 ``，从前后混杂文本、多个 JSON 代码块、无换行流式文本、连续 JSON 对象、非工整 JSON 中提取多个候选，并按当前步骤所需字段选择最像的 JSON。 - 模型 JSON 会尝试修复中文引号、全角冒号/逗号、尾随逗号、未加引号的键、Python 风格 `True/False/None`。 - 模型 JSON 解析失败会自动重试 1 次。 -- 每一步完成都会写入 state/summary 并输出 `STEP_FEEDBACK`。 +- 每一步完成都会写入 state/summary 并输出 `步骤反馈`。 - 接口失败会写入 `errors`，包含阶段、URL、状态码、请求摘要、响应正文和异常类型。 ## 回复用户规则 @@ -190,3 +211,4 @@ Step1 受众目标分析完成：当前已入库受众 X 个，本步骤新增 如果 `errors` 为空，可以说明“全流程完成”。 如果 `errors` 非空，只能说明“部分完成”，并列出失败阶段、接口 URL、状态码和响应摘要。 + diff --git a/skills-data/xdfx-single-agent-full-flow/SKILL.md b/skills-data/xdfx-single-agent-full-flow/SKILL.md index 3c8a2656..7d3676d9 100644 --- a/skills-data/xdfx-single-agent-full-flow/SKILL.md +++ b/skills-data/xdfx-single-agent-full-flow/SKILL.md @@ -26,20 +26,20 @@ scripts/xdfx_flow.py } ``` -默认会在缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 调用： +鉴权不能通过命令行传递 token。脚本会在启动阶段先 `import scripts/xdfx_auth.py`，用 `admin / ch@user` 获取访问令牌，并在第一步业务接口调用前写入请求头。 + +默认调用： ```text POST /api/v1/auth/login/username?password=ch%40user body: {"username":"admin"} ``` -拿到 `access_token` 后会写入 `Authorization: Bearer ...` 并重试原接口一次。登录接口和 401 重试都会输出 `OP_FEEDBACK` / `ERROR_FEEDBACK`，但反馈中的 `password`、`token` 会脱敏。默认关闭 HTTPS 证书校验，适配 UAT/内网自签证书；如果环境证书链正常，可以追加 `--verify-ssl`。 - -如果已经有可用登录态，也可以在命令里追加 `--auth-token TOKEN` 或 `--header "Authorization: Bearer TOKEN"`；此时脚本优先使用传入的 token。 +拿到 `access_token` 后会写入内部 `Authorization: Bearer ...`。不要通过任何命令行参数传 token；脚本会忽略外部 Authorization，统一由 `xdfx_auth.py` 获取和刷新。登录接口和 401 重试都会输出 `操作反馈` / `错误反馈`，但反馈中的 `password`、`token` 会脱敏。默认关闭 HTTPS 证书校验，适配 UAT/内网自签证书；如果环境证书链正常，可以追加 `--verify-ssl`。 ## 执行命令 -从本 Skill 目录逐步执行。虽然这是“一个技能做完整流程”的版本，但为了给用户真实的逐步反馈，智能体必须按 step1-step6 分六个命令运行，每个命令结束后先把 `STEP_FEEDBACK` / 新增错误反馈给用户，再继续下一步： +从本 Skill 目录逐步执行。虽然这是“一个技能做完整流程”的版本，但为了给用户真实的逐步反馈，智能体必须按 step1-step6 分六个命令运行，每个命令结束后先把 `步骤反馈` / 新增错误反馈给用户，再继续下一步： ```bash python scripts/xdfx_flow.py --mode step1 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md @@ -52,13 +52,29 @@ python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-fl `--mode full` 仅作为无人值守或本地排障兜底，不作为默认执行方式；因为外层工具通常会等长命令结束才返回输出，无法保证用户看到实时反馈。逐步命令共享同一个 `xdfx-flow-state.json`，id 传递和断点续跑不受影响。 -每一步完成后都会输出一行 `STEP_FEEDBACK {...}`。智能体必须在每个 step 命令结束后给用户一次简短反馈，说明该步骤状态、已入库数量和新增错误数，然后再执行下一步。 +如果确实想用一条命令编排完整流程，优先使用轻量包装脚本，而不是 `--mode full`。它会在内部逐个启动 step 命令并转发每一行反馈： -脚本每次调用接口前还会输出 `OP_FEEDBACK {...}`，包含当前操作、HTTP 方法和 URL。智能体必须用简短中文告诉用户“正在做什么、调用哪个接口”。 +```bash +python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md +``` -脚本出现错误时会立即输出 `ERROR_FEEDBACK {...}`，包含阶段、接口 URL、状态码、请求摘要、响应摘要和异常类型。智能体必须立刻告诉用户错误细节，不能等到最后才说。 +每一步完成后都会输出一行 `步骤反馈 {...}`。智能体必须在每个 step 命令结束后给用户一次简短反馈，说明该步骤状态、已入库数量和新增错误数，然后再执行下一步。 -如果模型返回内容经过多候选 JSON 提取和修复后仍无法解析，脚本会输出 `JSON_PARSE_REVIEW {...}`。智能体必须查看其中的 `responseExcerpt`，判断是否能识别出可用 JSON；无法确认时，重跑当前步骤或重新调用对应接口。 +脚本每次调用接口前还会输出 `操作反馈 {...}`，包含当前操作、HTTP 方法和 URL。智能体必须用简短中文告诉用户“正在做什么、调用哪个接口”。 + +每次保存接口成功拿到返回 ID 后会输出 `入库反馈 {...}`，包含 `入库对象类型`、`入库对象名称`、`返回ID` 和 `接口路径`。智能体必须把它转述成“已入库 xxx 受众 / xxx 脆弱点 / xxx 策略”等明确日志。 + +脚本出现错误时会立即输出 `错误反馈 {...}`，包含阶段、接口 URL、状态码、请求摘要、响应摘要和异常类型。智能体必须立刻告诉用户错误细节，不能等到最后才说。 + +如果模型返回内容经过多候选 JSON 提取和修复后仍无法解析，脚本会输出 `JSON解析复核 {...}`。智能体必须查看其中的 `响应片段`，判断是否能识别出可用 JSON；无法确认时，重跑当前步骤或重新调用对应接口。 + +## 失败重试规则 + +- 任意步骤出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前失败的同一条 `--mode stepN` 命令。 +- 重试前必须告诉用户失败阶段、接口 URL、状态码、响应摘要和“正在重试当前步骤”，不能静默重试。 +- 当前步骤未重试成功前，不允许继续执行下一步；不要因为某一步失败就跳过进入后续分析或入库。 +- 同一步重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 +- 只有用户明确要求忽略该失败，才可以继续后续步骤；回复中必须说明这是用户确认后的带风险继续。 只做参数和脚本检查、不调用接口： @@ -68,36 +84,44 @@ python scripts/xdfx_flow.py --mode full --input xdfx-input.json --dry-run ## 脚本保证 -- 受众只取最优 3 个继续。 -- 关键传播节点全局去重后只取最优 3 个继续。 +- 受众目标默认入库最优 3 个，可用 `--max-audiences` 调整。 +- 关键传播节点全局去重后默认入库最优 3 个，可用 `--max-pathways` 调整。 +- 下游脆弱点/策略默认只从已入库结果中选择前 2 个受众、前 2 个传播节点继续，可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。 +- 每个被选中的受众/传播节点默认生成最多 3 个脆弱点，可用 `--max-vulns` 调整。 +- 每个被选中的受众/传播节点默认生成最多 3 个策略，可用 `--max-strategies` 调整。 - 每个保存接口必须拿到返回 `id` 才进入下游。 - 保存后会同步前端 `task/task-action.step` 步骤状态；如只想入库不更新步骤条，追加 `--no-step-status`。 - 策略保存会同步写入 `task-action-channel`、`base-narrate`、`task-action-message`、`task-action-design-strategy-relation`，兼容策略表格/编排页面读取。 +- 策略 `信息形式/模态` 入库前会读取 `sys/sys-dict-data` 的 `modelType` 字典，把模型返回的“图片,视频/文本,图片”等文字转成前端需要的 value 逗号串；字典读取失败时使用内置兜底映射。 +- Step1 的受众目标抽取结果里，`用户编号列表` / `编号id` / `id` 必须对应上一轮 `zhanghao` 账号数组的索引值；脚本会按这个索引依次找到账号并保存受众账号关系。 - 模型流式 `{"answer":"..."}` 会自动拼接。 - 流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时，会自动再调用一次；仍失败才写入 `errors`。 - 模型输出会自动去掉 ``，从前后混杂文本、多个 JSON 代码块、无换行流式文本、连续 JSON 对象、非工整 JSON 中提取多个候选，并按当前步骤所需字段选择最像的 JSON。 - 模型 JSON 会尝试修复中文引号、全角冒号/逗号、尾随逗号、未加引号的键、Python 风格 `True/False/None`。 - 模型 JSON 解析失败会自动重试 1 次。 -- 单个受众、账号、节点或策略失败不会拖垮全流程。 -- 每完成一步都会输出 `STEP_FEEDBACK`，包含 `step`、`label`、`status`、累计入库计数和新增错误数。 -- 每次接口调用前都会输出 `OP_FEEDBACK`，说明当前操作和接口 URL。 -- 每个错误都会立即输出 `ERROR_FEEDBACK`，说明失败阶段、接口、状态码、请求摘要和响应摘要。 +- 单个受众、账号、节点或策略失败会被脚本记录；智能体仍必须按“失败重试规则”先重跑当前步骤，不能直接跳到下一步。 +- 每完成一步都会输出 `步骤反馈`，包含 `步骤`、`步骤名称`、`状态`、`累计入库数量` 和 `本步骤新增错误数`。 +- 每次接口调用前都会输出 `操作反馈`，说明当前操作和接口 URL。 +- 每次保存成功都会输出 `入库反馈`，说明已入库对象类型、名称和返回 ID。 +- 每个错误都会立即输出 `错误反馈`，说明失败阶段、接口、状态码、请求摘要和响应摘要。 - 所有接口错误都会写入 `xdfx-flow-state.json.errors`，包含阶段、URL、状态码、请求摘要、响应正文和异常类型。 ## 回复用户 -执行过程中，每出现一行 `STEP_FEEDBACK`，都要立刻向用户反馈一次： +执行过程中，每出现一行 `步骤反馈`，都要立刻向用户反馈一次： - 当前完成的步骤名称。 -- 状态：`ok`、`partial` 或 `skipped`。 +- 状态：`成功`、`部分完成` 或 `已跳过`。 - 当前累计入库数量。 - 本步骤新增错误数。 -每出现一行 `OP_FEEDBACK`，都要简短告诉用户当前正在执行的操作和接口，例如“正在保存受众目标，调用 POST /api/task/task-action-design”。 +每出现一行 `操作反馈`，都要简短告诉用户当前正在执行的操作和接口，例如“正在保存受众目标，调用 POST /api/task/task-action-design”。 -每出现一行 `ERROR_FEEDBACK`，都要立刻告诉用户失败阶段、接口 URL、状态码和响应摘要。 +每出现一行 `入库反馈`，都要简短告诉用户当前已入库的对象，例如“已入库受众目标：xxx，designId=123”或“已入库受众脆弱点：xxx，id=456”。 -每出现一行 `JSON_PARSE_REVIEW`，都要查看 `responseExcerpt`。如果能明显识别出目标 JSON，说明需要人工复核；如果不能识别，重跑当前 step 或重新调用对应模型接口，并告诉用户“模型 JSON 不工整，正在复核并重试”。 +每出现一行 `错误反馈`，都要立刻告诉用户失败阶段、接口 URL、状态码和响应摘要。 + +每出现一行 `JSON解析复核`，都要查看 `响应片段`。如果能明显识别出目标 JSON，说明需要人工复核；如果不能识别，重跑当前 step 或重新调用对应模型接口，并告诉用户“模型 JSON 不工整，正在复核并重试”。 全部执行结束后再读取 `xdfx-flow-summary.md` 和命令输出，向用户说明： @@ -109,3 +133,4 @@ python scripts/xdfx_flow.py --mode full --input xdfx-input.json --dry-run - 失败明细 只要 `errors` 不为空，就说“部分完成”，不要说“全部完成”。 + diff --git a/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_auth.py b/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_flow.py b/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_flow.py +++ b/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_run_full_steps.py b/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_run_full_steps.py new file mode 100644 index 00000000..d6468b51 --- /dev/null +++ b/skills-data/xdfx-single-agent-full-flow/scripts/xdfx_run_full_steps.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Run the xdfx flow one step at a time. + +The heavy business logic intentionally stays in xdfx_flow.py. This wrapper only +keeps "full flow" orchestration readable and preserves per-step feedback: every +subprocess line is flushed immediately, so OP/SAVE/ERROR/STEP feedback remains +visible to the calling agent. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +STEPS = ("step1", "step2", "step3", "step4", "step5", "step6") + +STEP_LABELS = { + "step1": "受众目标分析与入库", + "step2": "关键传播节点分析与入库", + "step3": "受众脆弱点分析与入库", + "step4": "传播节点脆弱点分析与入库", + "step5": "受众策略分析与入库", + "step6": "传播节点策略分析与入库", +} + + +def build_parser() -> argparse.ArgumentParser: + """Parse only wrapper-owned flags; pass all other flags to xdfx_flow.py.""" + parser = argparse.ArgumentParser( + description="按六个步骤运行 xdfx_flow.py，并实时转发中文反馈日志。", + add_help=True, + ) + parser.add_argument("--from-step", choices=STEPS, default="step1") + parser.add_argument("--to-step", choices=STEPS, default="step6") + parser.add_argument( + "--continue-on-error", + action="store_true", + help="某一步返回非零退出码时仍继续执行后续步骤。", + ) + return parser + + +def selected_steps(first: str, last: str) -> list[str]: + """Return an inclusive step slice for resume/partial reruns.""" + start = STEPS.index(first) + end = STEPS.index(last) + if start > end: + raise SystemExit("起始步骤必须小于或等于结束步骤") + return list(STEPS[start : end + 1]) + + +def stream_command(step: str, cmd: list[str]) -> int: + """Run one step and forward stdout/stderr lines as they are produced.""" + print(f"运行步骤：{STEP_LABELS.get(step, step)}；执行命令：" + " ".join(cmd), flush=True) + env = os.environ.copy() + env["PYTHONIOENCODING"] = "utf-8" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + env=env, + ) + assert process.stdout is not None + for line in process.stdout: + sys.stdout.buffer.write(line.encode("utf-8", errors="replace")) + sys.stdout.buffer.flush() + return process.wait() + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args, passthrough = parser.parse_known_args(argv) + flow_script = Path(__file__).with_name("xdfx_flow.py") + exit_code = 0 + for step in selected_steps(args.from_step, args.to_step): + # Keep all business flags unchanged so the wrapper behaves like a thin + # streaming version of repeated xdfx_flow.py --mode stepN calls. + cmd = [sys.executable, str(flow_script), "--mode", step, *passthrough] + code = stream_command(step, cmd) + if code: + exit_code = code + if not args.continue_on_error: + return code + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills-data/xdfx-split-controller/SKILL.md b/skills-data/xdfx-split-controller/SKILL.md index f9444d5b..391463cb 100644 --- a/skills-data/xdfx-split-controller/SKILL.md +++ b/skills-data/xdfx-split-controller/SKILL.md @@ -28,7 +28,7 @@ scripts/xdfx_flow.py ## 推荐执行顺序 -默认会自动处理登录态：当命令没有传 `Authorization`，或某个接口返回 401 时，脚本会用固定账号 `admin`、口令 `ch@user` 调用 `POST /api/v1/auth/login/username?password=ch%40user`，请求体为 `{"username":"admin"}`。拿到 `access_token` 后写入 `Authorization: Bearer ...` 并重试原接口一次。默认关闭 HTTPS 证书校验以兼容 UAT/内网自签证书；如果环境证书链正常，可以追加 `--verify-ssl`。如果已有可用 token，可追加 `--auth-token TOKEN` 或 `--header "Authorization: Bearer TOKEN"`。 +默认会自动处理登录态：脚本启动时先 `import scripts/xdfx_auth.py`，用固定账号 `admin`、口令 `ch@user` 调用 `POST /api/v1/auth/login/username?password=ch%40user`，请求体为 `{"username":"admin"}`。拿到 `access_token` 后写入内部 `Authorization: Bearer ...`，再调用第一步业务接口；不要通过任何命令行参数传 token。401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次。默认关闭 HTTPS 证书校验以兼容 UAT/内网自签证书；如果环境证书链正常，可以追加 `--verify-ssl`。 总控按顺序执行下面 6 条命令，所有步骤共用同一个 `xdfx-flow-state.json`： @@ -41,13 +41,23 @@ python scripts/xdfx_flow.py --mode step5 --input xdfx-input.json --state xdfx-fl python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` -每条命令结束都会输出一行 `STEP_FEEDBACK {...}`。总控必须在每个 step 命令结束后立即反馈用户，再继续下一步。 +每条命令结束都会输出一行 `步骤反馈 {...}`。总控必须在每个 step 命令结束后立即反馈用户，再继续下一步。 -每次接口调用前会输出 `OP_FEEDBACK {...}`。总控必须告诉用户当前正在做什么、调用哪个接口。 +每次接口调用前会输出 `操作反馈 {...}`。总控必须告诉用户当前正在做什么、调用哪个接口。 -如果接口或模型处理报错，会立即输出 `ERROR_FEEDBACK {...}`。总控必须立刻把失败阶段、URL、状态码、请求摘要和响应摘要告诉用户。 +每次保存接口成功拿到返回 ID 后会输出 `入库反馈 {...}`。总控必须告诉用户已入库的对象类型、名称和返回 ID，例如“已入库关键传播节点：xxx，pathwayAccountId=123”。 -如果模型返回文本经过自动修复仍无法解析，会输出 `JSON_PARSE_REVIEW {...}`。总控必须查看响应片段，判断是否需要重跑当前 step 或重新调用对应接口。 +如果接口或模型处理报错，会立即输出 `错误反馈 {...}`。总控必须立刻把失败阶段、URL、状态码、请求摘要和响应摘要告诉用户。 + +如果模型返回文本经过自动修复仍无法解析，会输出 `JSON解析复核 {...}`。总控必须查看响应片段，判断是否需要重跑当前 step 或重新调用对应接口。 + +## 失败重试规则 + +- 任意 step 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，总控必须先重跑当前失败的同一条 `--mode stepN` 命令。 +- 重试前必须告诉用户失败阶段、接口 URL、状态码、响应摘要和“正在重试当前步骤”，不能静默重试。 +- 当前 step 未重试成功前，不允许继续执行下一 step；不要因为某一步失败就跳过进入后续分析或入库。 +- 同一 step 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 +- 只有用户明确要求忽略该失败，才可以继续后续 step；回复中必须说明这是用户确认后的带风险继续。 如果只想一条命令跑完，也可以使用下面命令做无人值守兜底，但不建议作为默认方式，因为外层工具通常会等命令结束后才返回输出，无法保证用户看到实时反馈： @@ -55,21 +65,32 @@ python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-fl python scripts/xdfx_flow.py --mode full --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` +更推荐的一条命令方式是使用包装脚本；它内部仍按 step1-step6 分步调用，并逐行转发反馈： + +```bash +python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md +``` + ## 断点续跑 - Step1 成功后会写入 `topAudiences`。 +- Step1 的受众目标抽取结果里，`用户编号列表` / `编号id` / `id` 必须对应上一轮 `zhanghao` 账号数组的索引值；脚本会按这个索引依次找到账号并保存受众账号关系。 - Step2 成功后会写入 `topPathways`。 -- Step3/Step4 会写入两类脆弱点。 -- Step5/Step6 会写入两类策略。 +- Step1 默认入库最优 3 个受众；Step2 默认入库最优 3 个传播节点。 +- Step3/Step4 默认只选择前 2 个已入库受众/传播节点继续，每个生成最多 3 个脆弱点；可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。 +- Step5/Step6 默认对上述选中的每个受众/传播节点生成最多 3 个策略。 - 各步骤会同步前端 `task/task-action.step`；需要禁用时每条命令追加 `--no-step-status`。 - 策略步骤会同步写入 channel、narrate、message、策略与受众/传播节点关系表，保证原前端策略表格/编排页可继续读取。 -- 每个步骤完成都会输出 `STEP_FEEDBACK`，总控必须逐步转述给用户。 -- 每次操作和接口调用都会输出 `OP_FEEDBACK`，总控必须逐条转述当前操作。 -- 每次错误都会输出 `ERROR_FEEDBACK`，总控必须立即报告错误细节。 -- 每次 JSON 彻底解析失败都会输出 `JSON_PARSE_REVIEW`，总控必须复核响应片段并判断是否重试当前 step。 +- 策略步骤保存 `信息形式/模态` 时，必须使用脚本读取 `sys/sys-dict-data` 的 `modelType` 字典并把文字转成 value 逗号串，不能把“图片,视频”等文字直接写入 `task-action-message.modelType`。 +- 每个步骤完成都会输出 `步骤反馈`，总控必须逐步转述给用户。 +- 每次操作和接口调用都会输出 `操作反馈`，总控必须逐条转述当前操作。 +- 每次保存成功都会输出 `入库反馈`，总控必须逐条转述入库对象。 +- 每次错误都会输出 `错误反馈`，总控必须立即报告错误细节。 +- 每次 JSON 彻底解析失败都会输出 `JSON解析复核`，总控必须复核响应片段并判断是否重试当前 step。 - 每个步骤里的流式模型接口第一次空响应、返回 `y模型没有响应`/`模型没有响应` 或超时类字符串时，会自动再调用一次；仍失败才写入 `errors`。 -- 某一步失败后，修正问题再重跑同一个 `--mode stepN`。 +- 某一步失败后，必须先重跑同一个 `--mode stepN`；重试仍失败再等待用户处理或确认，不能自动跳到后续步骤。 ## 汇报规则 -执行过程中按 `OP_FEEDBACK` 告诉用户当前操作和接口；按 `ERROR_FEEDBACK` 立即报告错误；每个 step 结束后再根据 `STEP_FEEDBACK` 回复用户一次。全部步骤结束后，读取 `xdfx-flow-summary.md` 和 `xdfx-flow-state.json.errors` 再做最终汇总。只要存在错误，就明确说“部分完成”，并列出失败阶段、接口 URL、状态码和响应摘要。 +执行过程中按 `操作反馈` 告诉用户当前操作和接口；按 `入库反馈` 告诉用户已入库什么对象和返回 ID；按 `错误反馈` 立即报告错误；每个步骤结束后再根据 `步骤反馈` 回复用户一次。全部步骤结束后，读取 `xdfx-flow-summary.md` 和 `xdfx-flow-state.json.errors` 再做最终汇总。只要存在错误，就明确说“部分完成”，并列出失败阶段、接口 URL、状态码和响应摘要。 + diff --git a/skills-data/xdfx-split-controller/scripts/xdfx_auth.py b/skills-data/xdfx-split-controller/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-split-controller/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-split-controller/scripts/xdfx_flow.py b/skills-data/xdfx-split-controller/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-split-controller/scripts/xdfx_flow.py +++ b/skills-data/xdfx-split-controller/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-split-controller/scripts/xdfx_run_full_steps.py b/skills-data/xdfx-split-controller/scripts/xdfx_run_full_steps.py new file mode 100644 index 00000000..d6468b51 --- /dev/null +++ b/skills-data/xdfx-split-controller/scripts/xdfx_run_full_steps.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Run the xdfx flow one step at a time. + +The heavy business logic intentionally stays in xdfx_flow.py. This wrapper only +keeps "full flow" orchestration readable and preserves per-step feedback: every +subprocess line is flushed immediately, so OP/SAVE/ERROR/STEP feedback remains +visible to the calling agent. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +STEPS = ("step1", "step2", "step3", "step4", "step5", "step6") + +STEP_LABELS = { + "step1": "受众目标分析与入库", + "step2": "关键传播节点分析与入库", + "step3": "受众脆弱点分析与入库", + "step4": "传播节点脆弱点分析与入库", + "step5": "受众策略分析与入库", + "step6": "传播节点策略分析与入库", +} + + +def build_parser() -> argparse.ArgumentParser: + """Parse only wrapper-owned flags; pass all other flags to xdfx_flow.py.""" + parser = argparse.ArgumentParser( + description="按六个步骤运行 xdfx_flow.py，并实时转发中文反馈日志。", + add_help=True, + ) + parser.add_argument("--from-step", choices=STEPS, default="step1") + parser.add_argument("--to-step", choices=STEPS, default="step6") + parser.add_argument( + "--continue-on-error", + action="store_true", + help="某一步返回非零退出码时仍继续执行后续步骤。", + ) + return parser + + +def selected_steps(first: str, last: str) -> list[str]: + """Return an inclusive step slice for resume/partial reruns.""" + start = STEPS.index(first) + end = STEPS.index(last) + if start > end: + raise SystemExit("起始步骤必须小于或等于结束步骤") + return list(STEPS[start : end + 1]) + + +def stream_command(step: str, cmd: list[str]) -> int: + """Run one step and forward stdout/stderr lines as they are produced.""" + print(f"运行步骤：{STEP_LABELS.get(step, step)}；执行命令：" + " ".join(cmd), flush=True) + env = os.environ.copy() + env["PYTHONIOENCODING"] = "utf-8" + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + env=env, + ) + assert process.stdout is not None + for line in process.stdout: + sys.stdout.buffer.write(line.encode("utf-8", errors="replace")) + sys.stdout.buffer.flush() + return process.wait() + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args, passthrough = parser.parse_known_args(argv) + flow_script = Path(__file__).with_name("xdfx_flow.py") + exit_code = 0 + for step in selected_steps(args.from_step, args.to_step): + # Keep all business flags unchanged so the wrapper behaves like a thin + # streaming version of repeated xdfx_flow.py --mode stepN calls. + cmd = [sys.executable, str(flow_script), "--mode", step, *passthrough] + code = stream_command(step, cmd) + if code: + exit_code = code + if not args.continue_on_error: + return code + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills-data/xdfx-split-flow-skill-guide.md b/skills-data/xdfx-split-flow-skill-guide.md index f40c8834..305c43a2 100644 --- a/skills-data/xdfx-split-flow-skill-guide.md +++ b/skills-data/xdfx-split-flow-skill-guide.md @@ -45,12 +45,12 @@ scripts/xdfx_flow.py xdfx-flow-state.json ``` -默认会自动处理登录态：缺少 `Authorization` 或接口返回 401 时，脚本会用固定账号 `admin`、口令 `ch@user` 调用 `POST /api/v1/auth/login/username?password=ch%40user`，请求体为 `{"username":"admin"}`，拿到 `access_token` 后写入 `Authorization: Bearer ...` 并重试原接口一次。默认关闭 HTTPS 证书校验，用于兼容 UAT/内网自签证书；需要严格校验证书时追加 `--verify-ssl`。已有 token 时可追加 `--auth-token TOKEN` 或 `--header "Authorization: Bearer TOKEN"`。 +默认会自动处理登录态：脚本启动时先 `import scripts/xdfx_auth.py`，用固定账号 `admin`、口令 `ch@user` 调用 `POST /api/v1/auth/login/username?password=ch%40user`，请求体为 `{"username":"admin"}`，拿到 `access_token` 后写入内部 `Authorization: Bearer ...`，再调用第一步业务接口；不要通过任何命令行参数传 token。默认关闭 HTTPS 证书校验，用于兼容 UAT/内网自签证书；需要严格校验证书时追加 `--verify-ssl`。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次。 每一步命令结束都会输出一行： ```text -STEP_FEEDBACK {...} +步骤反馈 {...} ``` 总控智能体必须在每一步结束后立即把这行反馈转述给用户，再执行下一步。 @@ -58,15 +58,23 @@ STEP_FEEDBACK {...} 每次执行具体操作或调用接口前，脚本会输出： ```text -OP_FEEDBACK {...} +操作反馈 {...} ``` -总控智能体必须把 `operation`、`method` 和 `url` 转述给用户。 +总控智能体必须把 `操作`、`请求方式` 和 `接口地址` 转述给用户。 + +每次保存接口成功拿到 `id` 后，脚本会输出： + +```text +入库反馈 {...} +``` + +总控智能体必须把 `入库对象类型`、`入库对象名称` 和 `返回ID` 转述给用户，例如“已入库受众目标：xxx，designId=123”或“已入库策略：xxx，id=456”。 如果报错，脚本会立即输出： ```text -ERROR_FEEDBACK {...} +错误反馈 {...} ``` 总控智能体必须立刻告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 @@ -74,12 +82,12 @@ ERROR_FEEDBACK {...} 如果模型 JSON 经过自动提取和修复仍无法解析，脚本会输出： ```text -JSON_PARSE_REVIEW {...} +JSON解析复核 {...} ``` -总控智能体必须查看 `responseExcerpt`，判断是否能识别真正 JSON；无法确认时重跑当前 step 或重新调用对应模型接口。 +总控智能体必须查看 `响应片段`，判断是否能识别真正 JSON；无法确认时重跑当前步骤或重新调用对应模型接口。 -每一步中的流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时，脚本会自动再调用一次；如果第二次仍失败，才写入 `errors` 并在该步骤反馈里体现为 `partial`。 +每一步中的流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时，脚本会自动再调用一次；如果第二次仍失败，才写入 `errors` 并在该步骤反馈里体现为 `部分完成`。 ## 推荐执行顺序 @@ -96,23 +104,39 @@ python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-fl 如果某一步失败，修复接口、鉴权或输入后，只重跑对应 `stepN` 即可。 +如果需要一条命令完成分步编排，可以在 `xdfx-split-controller` 目录使用包装脚本；它内部仍按 step1-step6 调用执行器，并逐行转发反馈： + +```bash +python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md +``` + ## 每步反馈格式 -脚本会输出三类反馈行： +脚本会输出五类反馈行： -- `OP_FEEDBACK`：当前正在做什么、调用哪个接口。 -- `ERROR_FEEDBACK`：当前错误的阶段、接口、状态码、请求摘要和响应摘要。 -- `JSON_PARSE_REVIEW`：模型返回内容无法稳定解析为 JSON，需要执行智能体复核是否重试。 -- `STEP_FEEDBACK`：当前步骤最终状态和累计入库数量。 +- `操作反馈`：当前正在做什么、调用哪个接口。 +- `入库反馈`：当前已入库对象的类型、名称和返回 ID。 +- `错误反馈`：当前错误的阶段、接口、状态码、请求摘要和响应摘要。 +- `JSON解析复核`：模型返回内容无法稳定解析为 JSON，需要执行智能体复核是否重试。 +- `步骤反馈`：当前步骤最终状态和累计入库数量。 -`STEP_FEEDBACK` 中包含： +`步骤反馈` 中包含： -- `step`：步骤编号。 -- `label`：步骤名称。 -- `status`：`ok`、`partial` 或 `skipped`。 -- `counts`：当前累计入库数量。 -- `newErrors`：本步骤新增错误数。 -- `totalErrors`：累计错误数。 +- `步骤`：步骤编号。 +- `步骤名称`：步骤名称。 +- `状态`：`成功`、`部分完成` 或 `已跳过`。 +- `执行状态`：中文执行状态。 +- `累计入库数量`：当前累计入库数量。 +- `本步骤新增错误数`：本步骤新增错误数。 +- `累计错误数`：累计错误数。 + +默认数量规则： + +- Step1 入库最优 3 个受众。 +- Step2 入库最优 3 个传播节点。 +- Step3/Step4 只从已入库结果中默认选择前 2 个受众/传播节点继续，每个生成最多 3 个脆弱点；可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。 +- Step5/Step6 对上述选中的每个受众/传播节点默认生成最多 3 个策略。 +- Step5/Step6 的策略模态必须按前端 `modelType` 字典保存为 value 逗号串；脚本会自动把“图片,视频/文本,图片”等文字转成 `task-action-message.modelType` 需要的值。 总控反馈用户时建议使用这种格式： @@ -120,9 +144,9 @@ python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-fl Step2 关键传播节点分析完成：当前已入库传播节点 X 个，本步骤新增错误 Y 个。 ``` -如果 `status=partial`，说明该步骤部分完成，并继续执行后续可执行步骤。 +如果 `状态` 为 `部分完成`，说明该步骤部分完成，并继续执行后续可执行步骤。 -如果 `status=skipped`，说明跳过原因。 +如果 `状态` 为 `已跳过`，说明跳过原因。 ## 各步骤职责 @@ -254,7 +278,7 @@ xdfx-flow-summary.md xdfx-flow-state.json ``` -优先依据本步骤的 `STEP_FEEDBACK` 给即时反馈；最终汇总时再读取完整 summary 和 errors。 +优先依据本步骤的 `步骤反馈` 给即时反馈；最终汇总时再读取完整 summary 和 errors。 如果 `errors` 非空，必须告诉用户： @@ -266,3 +290,4 @@ xdfx-flow-state.json - 哪些数据已成功入库，哪些数据未完成。 只要存在错误，就说“部分完成”，不要说“全部完成”。 + diff --git a/skills-data/xdfx-step1-audience/SKILL.md b/skills-data/xdfx-step1-audience/SKILL.md index 0496ce2d..5b9f7474 100644 --- a/skills-data/xdfx-step1-audience/SKILL.md +++ b/skills-data/xdfx-step1-audience/SKILL.md @@ -24,7 +24,7 @@ python scripts/xdfx_flow.py --mode step1 --input xdfx-input.json --state xdfx-fl } ``` -鉴权和证书由脚本兜底处理：缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 登录获取 `access_token` 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 +鉴权和证书由脚本兜底处理：脚本启动时先 `import scripts/xdfx_auth.py`，自动用 `admin / ch@user` 获取 `access_token`，并在第一次业务接口调用前写入内部 Authorization；不要通过命令行传 token。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 本步骤由脚本完成： @@ -32,7 +32,8 @@ python scripts/xdfx_flow.py --mode step1 --input xdfx-input.json --state xdfx-fl - 自动解析流式 answer 与候选账号。 - 流式模型接口首次空响应、模型无响应或超时类返回会自动再调用一次。 - 抽取结构化受众。 -- 选择最优 3 个受众。 +- 抽取结果中的 `用户编号列表` / `编号id` / `id` 必须对应上一轮 `zhanghao` 账号数组的索引值；脚本按索引依次找到账号并保存。 +- 选择并入库最优 3 个受众。 - 保存受众目标。 - 保存受众目标账号。 - 同步前端步骤状态 `step-analysis-audience` 与 `step-analysis-keypoint`。 @@ -40,12 +41,22 @@ python scripts/xdfx_flow.py --mode step1 --input xdfx-input.json --state xdfx-fl 执行后检查 `xdfx-flow-summary.md`。如果 `errors` 非空，向用户报告失败接口、状态码、响应正文摘要。 +## 失败重试规则 + +- 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前 Step1 命令。 +- 重试前必须告诉用户失败阶段、接口 URL、状态码、响应摘要和“正在重试 Step1”，不能静默重试。 +- Step1 未重试成功前，不允许继续 Step2；不要因为 Step1 失败就跳过受众分析。 +- Step1 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 + ## 步骤反馈 -命令结束会输出 `STEP_FEEDBACK {...}`。必须先根据这行反馈告诉用户 Step1 的完成状态、当前受众入库数量和新增错误数，再决定是否继续下一步。 +命令结束会输出 `步骤反馈 {...}`。必须先根据这行反馈告诉用户 Step1 的完成状态、当前受众入库数量和新增错误数，再决定是否继续下一步。 -执行过程中会输出 `OP_FEEDBACK {...}`。每出现一次都要告诉用户当前操作和接口 URL。 +执行过程中会输出 `操作反馈 {...}`。每出现一次都要告诉用户当前操作和接口 URL。 -如果出现 `ERROR_FEEDBACK {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 +保存成功会输出 `入库反馈 {...}`。每出现一次都要告诉用户已入库的受众、受众账号和返回 ID。 + +如果出现 `错误反馈 {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 + +如果出现 `JSON解析复核 {...}`，必须查看 `响应片段`，判断是否能识别受众 JSON；无法确认时重跑 Step1。 -如果出现 `JSON_PARSE_REVIEW {...}`，必须查看 `responseExcerpt`，判断是否能识别受众 JSON；无法确认时重跑 Step1。 diff --git a/skills-data/xdfx-step1-audience/scripts/xdfx_auth.py b/skills-data/xdfx-step1-audience/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-step1-audience/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-step1-audience/scripts/xdfx_flow.py b/skills-data/xdfx-step1-audience/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-step1-audience/scripts/xdfx_flow.py +++ b/skills-data/xdfx-step1-audience/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-step2-pathway/SKILL.md b/skills-data/xdfx-step2-pathway/SKILL.md index 0dbc754c..9c396ebb 100644 --- a/skills-data/xdfx-step2-pathway/SKILL.md +++ b/skills-data/xdfx-step2-pathway/SKILL.md @@ -13,27 +13,37 @@ description: xdfx 拆分流程 step2，基于受众分析关键传播节点， python scripts/xdfx_flow.py --mode step2 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` -鉴权和证书由脚本兜底处理：缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 登录获取 `access_token` 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 +鉴权和证书由脚本兜底处理：脚本启动时先 `import scripts/xdfx_auth.py`，自动用 `admin / ch@user` 获取 `access_token`，并在第一次业务接口调用前写入内部 Authorization；不要通过命令行传 token。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 本步骤由脚本完成： - 对已入库受众逐个分析关键传播节点。 - 自动解析候选账号和传播节点 JSON。 - 流式模型接口首次空响应、模型无响应或超时类返回会自动再调用一次。 -- 跨受众合并去重，选择全局最优 3 个传播节点。 +- 跨受众合并去重，选择并入库全局最优 3 个传播节点。 - 保存传播节点。 - 保存传播节点与受众关系。 - 同步前端步骤状态 `step-analysis-keypoint` 与 `step-analysis-weakpoint`。 - 把 `pathwayAccountId`、关联 `designIds`、错误明细写入 state。 -关系保存失败不阻断后续流程，但必须报告给用户。 +关系保存失败会写入错误；智能体必须按失败重试规则先重跑 Step2，不能直接跳到 Step3。 + +## 失败重试规则 + +- 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前 Step2 命令。 +- 重试前必须告诉用户失败阶段、接口 URL、状态码、响应摘要和“正在重试 Step2”，不能静默重试。 +- Step2 未重试成功前，不允许继续 Step3；不要因为 Step2 失败就跳过关键传播节点分析。 +- Step2 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 ## 步骤反馈 -命令结束会输出 `STEP_FEEDBACK {...}`。必须先根据这行反馈告诉用户 Step2 的完成状态、当前传播节点入库数量和新增错误数，再决定是否继续下一步。 +命令结束会输出 `步骤反馈 {...}`。必须先根据这行反馈告诉用户 Step2 的完成状态、当前传播节点入库数量和新增错误数，再决定是否继续下一步。 -执行过程中会输出 `OP_FEEDBACK {...}`。每出现一次都要告诉用户当前操作和接口 URL。 +执行过程中会输出 `操作反馈 {...}`。每出现一次都要告诉用户当前操作和接口 URL。 -如果出现 `ERROR_FEEDBACK {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 +保存成功会输出 `入库反馈 {...}`。每出现一次都要告诉用户已入库的传播节点和返回 ID。 + +如果出现 `错误反馈 {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 + +如果出现 `JSON解析复核 {...}`，必须查看 `响应片段`，判断是否能识别传播节点 JSON；无法确认时重跑 Step2。 -如果出现 `JSON_PARSE_REVIEW {...}`，必须查看 `responseExcerpt`，判断是否能识别传播节点 JSON；无法确认时重跑 Step2。 diff --git a/skills-data/xdfx-step2-pathway/scripts/xdfx_auth.py b/skills-data/xdfx-step2-pathway/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-step2-pathway/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-step2-pathway/scripts/xdfx_flow.py b/skills-data/xdfx-step2-pathway/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-step2-pathway/scripts/xdfx_flow.py +++ b/skills-data/xdfx-step2-pathway/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-step3-audience-vulns/SKILL.md b/skills-data/xdfx-step3-audience-vulns/SKILL.md index ca21e94d..552b6e5d 100644 --- a/skills-data/xdfx-step3-audience-vulns/SKILL.md +++ b/skills-data/xdfx-step3-audience-vulns/SKILL.md @@ -13,26 +13,37 @@ description: xdfx 拆分流程 step3，为已入库受众抽取并保存最多3 python scripts/xdfx_flow.py --mode step3 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` -鉴权和证书由脚本兜底处理：缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 登录获取 `access_token` 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 +鉴权和证书由脚本兜底处理：脚本启动时先 `import scripts/xdfx_auth.py`，自动用 `admin / ch@user` 获取 `access_token`，并在第一次业务接口调用前写入内部 Authorization；不要通过命令行传 token。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 本步骤由脚本完成： -- 对每个已入库受众抽取最多 3 个脆弱点。 +- 从已入库受众中默认选择前 2 个继续分析；可用 `--downstream-audiences 1` 调成只处理 1 个。 +- 对每个被选中的受众抽取并入库最多 3 个脆弱点。 - 流式模型接口首次空响应、模型无响应或超时类返回会自动再调用一次。 - 自动修复和解析模型 JSON。 - 调用受众脆弱点保存接口。 - 只有保存成功并返回 `id` 的脆弱点才进入后续策略阶段。 - 同步前端步骤状态 `step-analysis-weakpoint`。 -- 单个受众失败不影响其他受众。 +- 单个受众失败会写入错误；智能体必须按失败重试规则先重跑 Step3，不能直接跳到 Step4/Step5。 完成后查看 `audienceVulns` 和 `errors`。 +## 失败重试规则 + +- 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前 Step3 命令。 +- 重试前必须告诉用户失败阶段、接口 URL、状态码、响应摘要和“正在重试 Step3”，不能静默重试。 +- Step3 未重试成功前，不允许继续 Step4/Step5；不要因为 Step3 失败就跳过受众脆弱点分析。 +- Step3 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 + ## 步骤反馈 -命令结束会输出 `STEP_FEEDBACK {...}`。必须先根据这行反馈告诉用户 Step3 的完成状态、当前受众脆弱点数量和新增错误数，再决定是否继续下一步。 +命令结束会输出 `步骤反馈 {...}`。必须先根据这行反馈告诉用户 Step3 的完成状态、当前受众脆弱点数量和新增错误数，再决定是否继续下一步。 -执行过程中会输出 `OP_FEEDBACK {...}`。每出现一次都要告诉用户当前操作和接口 URL。 +执行过程中会输出 `操作反馈 {...}`。每出现一次都要告诉用户当前操作和接口 URL。 -如果出现 `ERROR_FEEDBACK {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 +保存成功会输出 `入库反馈 {...}`。每出现一次都要告诉用户已入库的受众脆弱点和返回 ID。 + +如果出现 `错误反馈 {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 + +如果出现 `JSON解析复核 {...}`，必须查看 `响应片段`，判断是否能识别受众脆弱点 JSON；无法确认时重跑 Step3。 -如果出现 `JSON_PARSE_REVIEW {...}`，必须查看 `responseExcerpt`，判断是否能识别受众脆弱点 JSON；无法确认时重跑 Step3。 diff --git a/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_auth.py b/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_flow.py b/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_flow.py +++ b/skills-data/xdfx-step3-audience-vulns/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-step4-pathway-vulns/SKILL.md b/skills-data/xdfx-step4-pathway-vulns/SKILL.md index 98374914..f845c8fe 100644 --- a/skills-data/xdfx-step4-pathway-vulns/SKILL.md +++ b/skills-data/xdfx-step4-pathway-vulns/SKILL.md @@ -13,26 +13,37 @@ description: xdfx 拆分流程 step4，为已入库关键传播节点抽取并 python scripts/xdfx_flow.py --mode step4 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` -鉴权和证书由脚本兜底处理：缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 登录获取 `access_token` 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 +鉴权和证书由脚本兜底处理：脚本启动时先 `import scripts/xdfx_auth.py`，自动用 `admin / ch@user` 获取 `access_token`，并在第一次业务接口调用前写入内部 Authorization；不要通过命令行传 token。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 本步骤由脚本完成： -- 对每个已入库关键传播节点抽取最多 3 个脆弱点。 +- 从已入库关键传播节点中默认选择前 2 个继续分析；可用 `--downstream-pathways 1` 调成只处理 1 个。 +- 对每个被选中的关键传播节点抽取并入库最多 3 个脆弱点。 - 流式模型接口首次空响应、模型无响应或超时类返回会自动再调用一次。 - 自动修复和解析模型 JSON。 - 调用传播节点脆弱点保存接口。 - 只有保存成功并返回 `id` 的脆弱点才进入后续策略阶段。 - 同步前端步骤状态 `step-analysis-weakpoint`。 -- 单个节点失败不影响其他节点。 +- 单个节点失败会写入错误；智能体必须按失败重试规则先重跑 Step4，不能直接跳到 Step5/Step6。 完成后查看 `pathwayVulns` 和 `errors`。 +## 失败重试规则 + +- 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前 Step4 命令。 +- 重试前必须告诉用户失败阶段、接口 URL、状态码、响应摘要和“正在重试 Step4”，不能静默重试。 +- Step4 未重试成功前，不允许继续 Step5/Step6；不要因为 Step4 失败就跳过传播节点脆弱点分析。 +- Step4 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 + ## 步骤反馈 -命令结束会输出 `STEP_FEEDBACK {...}`。必须先根据这行反馈告诉用户 Step4 的完成状态、当前传播节点脆弱点数量和新增错误数，再决定是否继续下一步。 +命令结束会输出 `步骤反馈 {...}`。必须先根据这行反馈告诉用户 Step4 的完成状态、当前传播节点脆弱点数量和新增错误数，再决定是否继续下一步。 -执行过程中会输出 `OP_FEEDBACK {...}`。每出现一次都要告诉用户当前操作和接口 URL。 +执行过程中会输出 `操作反馈 {...}`。每出现一次都要告诉用户当前操作和接口 URL。 -如果出现 `ERROR_FEEDBACK {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 +保存成功会输出 `入库反馈 {...}`。每出现一次都要告诉用户已入库的传播节点脆弱点和返回 ID。 + +如果出现 `错误反馈 {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 + +如果出现 `JSON解析复核 {...}`，必须查看 `响应片段`，判断是否能识别传播节点脆弱点 JSON；无法确认时重跑 Step4。 -如果出现 `JSON_PARSE_REVIEW {...}`，必须查看 `responseExcerpt`，判断是否能识别传播节点脆弱点 JSON；无法确认时重跑 Step4。 diff --git a/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_auth.py b/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_flow.py b/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_flow.py +++ b/skills-data/xdfx-step4-pathway-vulns/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-step5-audience-strategy/SKILL.md b/skills-data/xdfx-step5-audience-strategy/SKILL.md index b0f8de85..9bba2a5a 100644 --- a/skills-data/xdfx-step5-audience-strategy/SKILL.md +++ b/skills-data/xdfx-step5-audience-strategy/SKILL.md @@ -13,27 +13,39 @@ description: xdfx 拆分流程 step5，基于受众目标与受众脆弱点生 python scripts/xdfx_flow.py --mode step5 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` -鉴权和证书由脚本兜底处理：缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 登录获取 `access_token` 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 +鉴权和证书由脚本兜底处理：脚本启动时先 `import scripts/xdfx_auth.py`，自动用 `admin / ch@user` 获取 `access_token`，并在第一次业务接口调用前写入内部 Authorization；不要通过命令行传 token。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 本步骤由脚本完成： -- 按受众读取已入库脆弱点。 +- 从已入库受众中默认选择前 2 个继续生成策略；可用 `--downstream-audiences 1` 调成只处理 1 个。 +- 对每个被选中的受众读取已入库脆弱点并默认生成最多 3 个策略。 - 调用策略生成模型接口。 - 流式模型接口首次空响应、模型无响应或超时类返回会自动再调用一次。 - 自动解析 `发布人设N / 账号类型N / 信息标题N` 等字段。 - 调用策略保存接口。 - 同步写入 `task-action-channel`、`base-narrate`、`task-action-message`、`task-action-design-strategy-relation`。 +- 保存 `task-action-message.modelType` 前会读取前端 `modelType` 字典并把“图片,视频/文本,图片”等模态文字转成 value 逗号串，避免策略表格回显为空。 - 同步前端步骤状态 `step-analysis-strategy` 与 `step-analysis-strategy-gantt`。 - 保存成功后写入 `audienceStrategies`。 -如果某个受众没有可用脆弱点，脚本会跳过并在 `errors` 中记录。 +如果某个受众没有可用脆弱点，脚本会记录错误；智能体必须先回到 Step3 或重跑 Step5 修复当前策略环节，不能直接跳过。 + +## 失败重试规则 + +- 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前 Step5 命令。 +- 如果失败原因是缺少受众脆弱点，先重跑 Step3 生成脆弱点，再重跑 Step5；重试前必须告诉用户原因和重试动作。 +- Step5 未重试成功前，不允许继续 Step6 或做最终汇总；不要因为 Step5 失败就跳过受众策略分析。 +- Step5 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 ## 步骤反馈 -命令结束会输出 `STEP_FEEDBACK {...}`。必须先根据这行反馈告诉用户 Step5 的完成状态、当前受众策略数量和新增错误数，再决定是否继续下一步。 +命令结束会输出 `步骤反馈 {...}`。必须先根据这行反馈告诉用户 Step5 的完成状态、当前受众策略数量和新增错误数，再决定是否继续下一步。 -执行过程中会输出 `OP_FEEDBACK {...}`。每出现一次都要告诉用户当前操作和接口 URL。 +执行过程中会输出 `操作反馈 {...}`。每出现一次都要告诉用户当前操作和接口 URL。 -如果出现 `ERROR_FEEDBACK {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 +保存成功会输出 `入库反馈 {...}`。每出现一次都要告诉用户已入库的策略、策略子表和返回 ID。 + +如果出现 `错误反馈 {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 + +如果出现 `JSON解析复核 {...}`，必须查看 `响应片段`，判断是否能识别受众策略 JSON；无法确认时重跑 Step5。 -如果出现 `JSON_PARSE_REVIEW {...}`，必须查看 `responseExcerpt`，判断是否能识别受众策略 JSON；无法确认时重跑 Step5。 diff --git a/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_auth.py b/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_flow.py b/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_flow.py +++ b/skills-data/xdfx-step5-audience-strategy/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/skills-data/xdfx-step6-pathway-strategy/SKILL.md b/skills-data/xdfx-step6-pathway-strategy/SKILL.md index a185e631..54492394 100644 --- a/skills-data/xdfx-step6-pathway-strategy/SKILL.md +++ b/skills-data/xdfx-step6-pathway-strategy/SKILL.md @@ -13,27 +13,39 @@ description: xdfx 拆分流程 step6，基于关键传播节点与节点脆弱 python scripts/xdfx_flow.py --mode step6 --input xdfx-input.json --state xdfx-flow-state.json --summary xdfx-flow-summary.md ``` -鉴权和证书由脚本兜底处理：缺少 `Authorization` 或接口返回 401 时，自动用 `admin / ch@user` 登录获取 `access_token` 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 +鉴权和证书由脚本兜底处理：脚本启动时先 `import scripts/xdfx_auth.py`，自动用 `admin / ch@user` 获取 `access_token`，并在第一次业务接口调用前写入内部 Authorization；不要通过命令行传 token。接口返回 401 时仍由 `xdfx_auth.py` 刷新 token 并重试原接口一次；HTTPS 默认不校验证书，需校验证书时追加 `--verify-ssl`。 本步骤由脚本完成： -- 按关键传播节点读取已入库脆弱点。 +- 从已入库关键传播节点中默认选择前 2 个继续生成策略；可用 `--downstream-pathways 1` 调成只处理 1 个。 +- 对每个被选中的关键传播节点读取已入库脆弱点并默认生成最多 3 个策略。 - 调用策略生成模型接口。 - 流式模型接口首次空响应、模型无响应或超时类返回会自动再调用一次。 - 自动解析 `发布人设N / 账号类型N / 信息标题N` 等字段。 - 调用策略保存接口。 - 同步写入 `task-action-channel`、`base-narrate`、`task-action-message`、`task-action-design-strategy-relation`。 +- 保存 `task-action-message.modelType` 前会读取前端 `modelType` 字典并把“图片,视频/文本,图片”等模态文字转成 value 逗号串，避免策略表格回显为空。 - 同步前端步骤状态 `step-analysis-strategy` 与 `step-analysis-strategy-gantt`。 - 保存成功后写入 `pathwayStrategies`。 -如果某个传播节点没有可用脆弱点，脚本会跳过并在 `errors` 中记录。 +如果某个传播节点没有可用脆弱点，脚本会记录错误；智能体必须先回到 Step4 或重跑 Step6 修复当前策略环节，不能直接跳过。 + +## 失败重试规则 + +- 出现 `错误反馈`、`JSON解析复核`，或 `步骤反馈` 中 `状态` 为 `部分完成` / `已跳过` / `本步骤新增错误数` 大于 0 时，必须先重跑当前 Step6 命令。 +- 如果失败原因是缺少传播节点脆弱点，先重跑 Step4 生成脆弱点，再重跑 Step6；重试前必须告诉用户原因和重试动作。 +- Step6 未重试成功前，不允许做最终成功汇总；不要因为 Step6 失败就跳过传播节点策略分析。 +- Step6 重试后仍失败时，停止自动推进，读取 `xdfx-flow-state.json.errors` 汇报失败细节，并等待用户处理或确认。 ## 步骤反馈 -命令结束会输出 `STEP_FEEDBACK {...}`。必须先根据这行反馈告诉用户 Step6 的完成状态、当前传播节点策略数量和新增错误数，再做最终汇总。 +命令结束会输出 `步骤反馈 {...}`。必须先根据这行反馈告诉用户 Step6 的完成状态、当前传播节点策略数量和新增错误数，再做最终汇总。 -执行过程中会输出 `OP_FEEDBACK {...}`。每出现一次都要告诉用户当前操作和接口 URL。 +执行过程中会输出 `操作反馈 {...}`。每出现一次都要告诉用户当前操作和接口 URL。 -如果出现 `ERROR_FEEDBACK {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 +保存成功会输出 `入库反馈 {...}`。每出现一次都要告诉用户已入库的策略、策略子表和返回 ID。 + +如果出现 `错误反馈 {...}`，必须立即告诉用户失败阶段、接口 URL、状态码、请求摘要和响应摘要。 + +如果出现 `JSON解析复核 {...}`，必须查看 `响应片段`，判断是否能识别传播节点策略 JSON；无法确认时重跑 Step6。 -如果出现 `JSON_PARSE_REVIEW {...}`，必须查看 `responseExcerpt`，判断是否能识别传播节点策略 JSON；无法确认时重跑 Step6。 diff --git a/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_auth.py b/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_auth.py new file mode 100644 index 00000000..fe46f809 --- /dev/null +++ b/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_auth.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Authentication helper for xdfx skill scripts. + +The caller should not pass tokens through command-line arguments. This module +owns the fixed admin login flow and returns a Bearer token before business +interfaces are called. +""" + +from __future__ import annotations + +import json +import re +import ssl +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass +class AuthResult: + token: str + url: str + status: int | None + raw: str + + +class AuthError(RuntimeError): + def __init__( + self, + message: str, + *, + url: str, + status: int | None = None, + response: str = "", + exception: BaseException | None = None, + ) -> None: + super().__init__(message) + self.url = url + self.status = status + self.response = response + self.exception = exception + + +def auth_header_value(token: str) -> str: + token = token.strip() + return token if token.lower().startswith("bearer ") else f"Bearer {token}" + + +def redact_url(url: str) -> str: + return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) + + +def ssl_context(*, verify_ssl: bool) -> ssl.SSLContext | None: + if verify_ssl: + return None + return ssl._create_unverified_context() + + +def build_login_url(api_base: str, path: str, password: str) -> str: + base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else api_base.rstrip("/") + "/" + path.lstrip("/") + if password: + separator = "&" if "?" in base else "?" + base = f"{base}{separator}password={urllib.parse.quote(password, safe='')}" + return base + + +def extract_access_token(data: Any) -> str | None: + if isinstance(data, dict): + for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in data.values(): + nested = extract_access_token(value) + if nested: + return nested + elif isinstance(data, list): + for value in data: + nested = extract_access_token(value) + if nested: + return nested + return None + + +def parse_json_loose(text: str) -> Any: + try: + return json.loads(text) + except Exception: + pass + match = re.search(r"[\[{]", text or "") + if not match: + return {} + start = match.start() + for end in range(len(text), start, -1): + try: + return json.loads(text[start:end]) + except Exception: + continue + return {} + + +def fetch_access_token( + *, + api_base: str, + username: str = "admin", + password: str = "ch@user", + login_path: str = "/api/v1/auth/login/username", + timeout: float = 90, + verify_ssl: bool = False, +) -> AuthResult: + """Login once and return the extracted access token.""" + url = build_login_url(api_base, login_path, password) + payload = {"username": username} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout, context=ssl_context(verify_ssl=verify_ssl)) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = parse_json_loose(raw) + token = extract_access_token(parsed) + if not token: + raise AuthError( + "登录接口已响应但未找到 access_token", + url=redact_url(url), + status=getattr(resp, "status", None), + response=raw, + ) + return AuthResult(token=token, url=redact_url(url), status=getattr(resp, "status", None), raw=raw) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" + raise AuthError( + "登录接口返回错误", + url=redact_url(url), + status=exc.code, + response=body, + exception=exc, + ) from exc + except AuthError: + raise + except Exception as exc: # noqa: BLE001 - auth errors must preserve detail for user feedback. + raise AuthError( + "登录接口调用异常", + url=redact_url(url), + response=str(exc), + exception=exc, + ) from exc diff --git a/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_flow.py b/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_flow.py index 52880e82..5f405cc8 100644 --- a/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_flow.py +++ b/skills-data/xdfx-step6-pathway-strategy/scripts/xdfx_flow.py @@ -26,7 +26,10 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +import xdfx_auth + +# Keep every backend path in one table so endpoint drift is easy to audit. ENDPOINTS = { "aiuser": "/gpt/chat/chat_aiuser", "group": "/gpt/chat/chat_qianqian_group_chat", @@ -44,10 +47,12 @@ ENDPOINTS = { "save_narrate": "/api/base/base-narrate", "save_message": "/api/task/task-action-message", "save_strategy_relation": "/api/task/task-action-design-strategy-relation", + "dict_data": "/api/sys/sys-dict-data", } +# Human-readable names are used only in the Chinese feedback logs. ENDPOINT_LABELS = { - "login": "登录获取访问 token", + "login": "登录获取访问令牌", "aiuser": "调用候选账号/受众分析模型接口", "group": "调用受众目标结构化抽取接口", "spread": "调用关键传播节点结构化抽取接口", @@ -64,8 +69,86 @@ ENDPOINT_LABELS = { "save_narrate": "保存策略信息主题", "save_message": "保存策略消息配置", "save_strategy_relation": "保存策略与受众/传播节点关系", + "dict_data": "读取前端字典数据", } +LOG_PREFIXES = { + "operation": "操作反馈", + "error": "错误反馈", + "json_review": "JSON解析复核", + "save": "入库反馈", + "step": "步骤反馈", + "done": "执行完成", +} + +STATUS_LABELS = { + "ok": "成功", + "partial": "部分完成", + "skipped": "已跳过", +} + +STEP_NUMBER_LABELS = { + "step1": "第一步", + "step2": "第二步", + "step3": "第三步", + "step4": "第四步", + "step5": "第五步", + "step6": "第六步", +} + +HTTP_METHOD_LABELS = { + "GET": "读取（GET）", + "POST": "提交（POST）", + "PUT": "更新（PUT）", + "PATCH": "局部更新（PATCH）", + "DELETE": "删除（DELETE）", +} + +STAGE_SEGMENT_LABELS = { + "analyze": "模型分析", + "extract": "结构化抽取", + "save": "保存数据", + "read": "读取数据", + "status": "同步状态", +} + +REQUEST_FIELD_LABELS = { + "history": "历史消息", + "max_tokens": "最大生成令牌数", + "model_name": "模型名称", + "otherinfo": "其他信息", + "prompt_name": "提示词名称", + "stream": "是否流式", + "temperature": "温度", + "usermessage": "用户消息", + "user_message": "用户消息", + "mubiao": "行动目标", + "zhanghao": "账号列表", + "taskId": "任务ID", + "actionId": "行动ID", + "taskActionId": "行动ID", + "designId": "受众ID", + "pathwayId": "传播节点ID", + "strategyId": "策略ID", + "accountId": "账号ID", + "accountName": "账号名称", + "accountType": "账号类型", + "targetAudience": "受众目标", + "vulnerabilities": "脆弱点", + "topic": "信息标题", + "narrate": "叙事主题", + "platform": "平台", + "persona": "发布人设", + "modelType": "信息形式", + "quantity": "数量", + "publishTime": "发布时间段", + "reason": "理由", + "ids": "ID列表", + "id": "ID", + "step": "步骤状态", +} + +# Platform values mirror the frontend dictionaries used by xdfx strategy pages. PLATFORM_MAP = { "facebook": 1, "fb": 1, @@ -78,6 +161,8 @@ PLATFORM_MAP = { "yt": 4, } +# The model service sometimes returns a successful HTTP response with these +# strings instead of useful content. Treat them like transient failures. MODEL_RETRY_MARKERS = ( "y模型没有响应", "模型没有响应", @@ -118,6 +203,12 @@ class FlowError(RuntimeError): @dataclass class FlowContext: + """Runtime options shared by every step. + + Keep this object boring and serializable-ish: it is passed through every + helper, while durable progress belongs in the state JSON file. + """ + api_base: str headers: dict[str, str] timeout: float @@ -129,6 +220,8 @@ class FlowContext: max_pathways: int = 3 max_vulns: int = 3 max_strategies: int = 3 + downstream_audiences: int = 2 + downstream_pathways: int = 2 sync_step_status: bool = True model_retries: int = 1 verify_ssl: bool = False @@ -153,7 +246,7 @@ def compact(value: Any, limit: int = 1400) -> str: text = repr(value) if len(text) <= limit: return text - return text[:limit] + f"... " + return text[:limit] + f"... <已截断 {len(text) - limit} 个字符>" def stable_json(value: Any) -> str: @@ -223,6 +316,11 @@ def save_json_file(path: str | Path, value: Any) -> None: def default_state() -> dict[str, Any]: + """Create a resumable state container. + + The script can be run as six independent commands. IDs returned by one step + are persisted here so later steps can resume without redoing earlier saves. + """ return { "api_base": "", "taskId": "", @@ -247,6 +345,7 @@ def default_state() -> dict[str, Any]: def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> None: + # Accept both frontend-style camelCase and CLI-friendly snake_case inputs. aliases = { "task_id": "taskId", "action_id": "actionId", @@ -260,16 +359,12 @@ def merge_config_into_state(state: dict[str, Any], config: dict[str, Any]) -> No continue if dest in {"taskId", "actionId"}: state[dest] = str(value) - elif dest in state or dest in { - "api_base", - "headers", - "auth_token", - "authorization", - }: + elif dest in state or dest in {"api_base", "headers"}: state[dest] = value def record_event(state: dict[str, Any], stage: str, message: str, data: Any | None = None) -> None: + """Append a compact diagnostic event to the state file.""" state.setdefault("events", []).append( { "time": utc_now(), @@ -286,9 +381,13 @@ def feedback_step(stage: str) -> str: return match.group(1) if match else "" +def feedback_step_label(stage: str) -> str: + return STEP_NUMBER_LABELS.get(feedback_step(stage), "") + + def request_keys(payload: Any) -> list[str]: if isinstance(payload, dict): - return sorted(str(key) for key in payload.keys()) + return sorted(REQUEST_FIELD_LABELS.get(str(key), str(key)) for key in payload.keys()) return [] @@ -306,6 +405,77 @@ def endpoint_label(endpoint_key: str | None, method: str, url: str) -> str: return f"调用接口 {method}" +def method_label(method: str) -> str: + return HTTP_METHOD_LABELS.get(str(method).upper(), str(method)) + + +def yes_no(value: bool) -> str: + return "是" if value else "否" + + +def stage_label(stage: Any) -> str: + text = str(stage or "") + if not text: + return "" + if text in STEP_LABELS: + return STEP_LABELS[text] + if text == "runner": + return "流程执行器" + parts = text.split("/") + base = parts[0] + match = re.search(r"(step[1-6])", base) + labels: list[str] = [] + if match: + labels.append(STEP_LABELS.get(match.group(1), match.group(1))) + elif base == "runner": + labels.append("流程执行器") + else: + labels.append(base) + for segment in parts[1:]: + normalized = segment.replace("-", "_") + attempt = re.fullmatch(r"model-call-attempt-(\d+)", segment) + json_attempt = re.fullmatch(r"model-json-attempt-(\d+)", segment) + if attempt: + labels.append(f"模型接口第 {attempt.group(1)} 次尝试") + elif json_attempt: + labels.append(f"JSON解析第 {json_attempt.group(1)} 次尝试") + elif segment == "auto-login": + labels.append("自动登录") + elif segment in STAGE_SEGMENT_LABELS: + labels.append(STAGE_SEGMENT_LABELS[segment]) + elif normalized in ENDPOINT_LABELS: + labels.append(ENDPOINT_LABELS[normalized]) + else: + labels.append(segment) + return " / ".join(labels) + + +def status_label(status: Any) -> str: + if status is None: + return "" + return STATUS_LABELS.get(str(status), str(status)) + + +def mode_label(mode: str) -> str: + if mode == "full": + return "全流程" + return STEP_LABELS.get(mode, mode) + + +def exception_type_label(exception_type: Any) -> str: + name = str(exception_type or "") + labels = { + "HTTPError": "HTTP 调用异常", + "URLError": "网络地址异常", + "TimeoutError": "请求超时异常", + "FlowError": "流程异常", + "ValueError": "数据格式异常", + "KeyError": "字段缺失异常", + "RuntimeError": "运行异常", + } + return labels.get(name, "程序异常" if name else "") + + def emit_operation_feedback( state: dict[str, Any], stage: str, @@ -317,39 +487,41 @@ def emit_operation_feedback( attempt: int = 1, dry_run: bool = False, ) -> None: + # Feedback lines are a mini protocol consumed by the calling agent. They + # must stay single-line JSON so weak models/tools can relay them reliably. payload_obj = { - "type": "OP_FEEDBACK", - "step": feedback_step(stage), - "stage": stage, - "operation": endpoint_label(endpoint_key, method, url), - "method": method, - "url": url, - "endpoint": endpoint_key or "", - "attempt": attempt, - "dryRun": dry_run, - "requestKeys": request_keys(payload), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "操作反馈", + "步骤": feedback_step_label(stage), + "阶段": stage_label(stage), + "操作": endpoint_label(endpoint_key, method, url), + "请求方式": method_label(method), + "接口地址": url, + "接口名称": endpoint_label(endpoint_key, method, url), + "第几次尝试": attempt, + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("OP_FEEDBACK " + json.dumps(payload_obj, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["operation"] + " " + json.dumps(payload_obj, ensure_ascii=False), flush=True) def emit_error_feedback(error: dict[str, Any]) -> None: payload = { - "type": "ERROR_FEEDBACK", - "stage": error.get("stage"), - "step": feedback_step(str(error.get("stage") or "")), - "message": error.get("message"), - "url": error.get("url"), - "status": error.get("status"), - "request": error.get("request"), - "response": error.get("response"), - "exceptionType": error.get("exceptionType"), - "exception": error.get("exception"), - "time": error.get("time"), + "日志类型": "错误反馈", + "阶段": stage_label(error.get("stage")), + "步骤": feedback_step_label(str(error.get("stage") or "")), + "错误信息": error.get("message"), + "接口地址": error.get("url"), + "状态码": error.get("status"), + "请求摘要": error.get("request"), + "响应摘要": error.get("response"), + "异常类型": exception_type_label(error.get("exceptionType")), + "异常信息": error.get("exception"), + "时间": error.get("time"), } - print("ERROR_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["error"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def emit_json_parse_review( @@ -360,18 +532,89 @@ def emit_json_parse_review( candidate_count: int, ) -> None: payload = { - "type": "JSON_PARSE_REVIEW", - "stage": stage, - "step": feedback_step(stage), - "endpoint": endpoint_key, - "message": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", - "candidateCount": candidate_count, - "responseExcerpt": compact(text, 3000), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "time": utc_now(), + "日志类型": "JSON解析复核", + "阶段": stage_label(stage), + "步骤": feedback_step_label(stage), + "接口名称": ENDPOINT_LABELS.get(endpoint_key, endpoint_key), + "提示": "模型输出经过多候选提取和修复后仍无法解析为可用 JSON，请执行智能体复核是否需要重新调用本步骤接口。", + "候选数量": candidate_count, + "响应片段": compact(text, 3000), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), } - print("JSON_PARSE_REVIEW " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["json_review"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) + + +SAVE_ENTITY_LABELS = { + "save_design": "受众目标", + "save_design_account": "受众目标账号", + "save_pathway": "关键传播节点", + "save_relation": "受众-传播节点关系", + "save_design_vuln": "受众脆弱点", + "save_pathway_vuln": "传播节点脆弱点", + "save_strategy": "策略", + "save_channel": "策略平台与人设", + "save_narrate": "策略信息主题", + "save_message": "策略消息配置", + "save_strategy_relation": "策略关联关系", +} + + +def saved_entity_name(endpoint_key: str, payload: dict[str, Any]) -> str: + """Return the most user-readable name for an inserted row.""" + candidates = [ + payload.get("targetAudience"), + payload.get("accountName"), + payload.get("vulnerabilities"), + payload.get("topic"), + payload.get("narrate"), + payload.get("platform"), + payload.get("strategyId"), + payload.get("designId"), + payload.get("pathwayId"), + ] + for value in candidates: + if value is not None and str(value).strip(): + return str(value).strip() + return endpoint_key + + +def emit_save_feedback( + state: dict[str, Any], + stage: str, + endpoint_key: str, + payload: dict[str, Any], + row_id: int, + *, + dry_run: bool, +) -> None: + # Emit only after an id is known. Downstream steps use these ids and should + # fail loudly rather than inventing relations against missing rows. + label = SAVE_ENTITY_LABELS.get(endpoint_key, endpoint_key) + name = saved_entity_name(endpoint_key, payload) + feedback = { + "日志类型": "入库反馈", + "步骤": feedback_step(stage), + "阶段": stage, + "入库对象类型": label, + "入库对象名称": name, + "返回ID": row_id, + "接口名称": ENDPOINT_LABELS.get(endpoint_key, label), + "接口路径": ENDPOINTS.get(endpoint_key, ""), + "是否演练": yes_no(dry_run), + "请求字段": request_keys(payload), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "时间": utc_now(), + } + print(LOG_PREFIXES["save"] + " " + json.dumps(feedback, ensure_ascii=False), flush=True) + record_event( + state, + stage, + f"入库{label}成功：{name}", + {"id": row_id, "endpoint": endpoint_key, "dryRun": dry_run}, + ) def record_error( @@ -385,6 +628,8 @@ def record_error( response: Any | None = None, exception: BaseException | None = None, ) -> None: + # Persist full-enough error context for post-run diagnosis, then immediately + # emit a short line the agent can show to the user. item = { "time": utc_now(), "stage": stage, @@ -430,65 +675,25 @@ def url_for_path(ctx: FlowContext, path: str) -> str: return ctx.api_base.rstrip("/") + "/" + path.lstrip("/") -def parse_headers(header_values: list[str], auth_token: str | None, authorization: str | None) -> dict[str, str]: +def parse_headers(header_values: list[str]) -> dict[str, str]: headers = {"Content-Type": "application/json", "Accept": "application/json"} - if auth_token: - headers["Authorization"] = auth_token if auth_token.lower().startswith("bearer ") else f"Bearer {auth_token}" - if authorization: - headers["Authorization"] = authorization for raw in header_values: if ":" not in raw: raise FlowError(f"header 格式错误，应为 Key: Value，实际：{raw}") key, value = raw.split(":", 1) - headers[key.strip()] = value.strip() + normalized = key.strip() + if normalized.lower() == "authorization": + continue + headers[normalized] = value.strip() return headers -def has_authorization(headers: dict[str, str]) -> bool: - return any(key.lower() == "authorization" and str(value).strip() for key, value in headers.items()) - - def ssl_context(ctx: FlowContext) -> ssl.SSLContext | None: if ctx.verify_ssl: return None return ssl._create_unverified_context() -def auth_header_value(token: str) -> str: - token = token.strip() - return token if token.lower().startswith("bearer ") else f"Bearer {token}" - - -def extract_access_token(data: Any) -> str | None: - if isinstance(data, dict): - for key in ("access_token", "accessToken", "token", "jwt", "id_token", "authorization", "Authorization"): - value = data.get(key) - if isinstance(value, str) and value.strip(): - return value.strip() - for value in data.values(): - nested = extract_access_token(value) - if nested: - return nested - elif isinstance(data, list): - for value in data: - nested = extract_access_token(value) - if nested: - return nested - return None - - -def redact_url(url: str) -> str: - return re.sub(r"([?&](?:password|token|authToken|access_token)=)[^&]+", r"\1***", url, flags=re.IGNORECASE) - - -def build_login_url(ctx: FlowContext, path: str) -> str: - base = path if re.match(r"^https?://", path, flags=re.IGNORECASE) else url_for_path(ctx, path) - if ctx.login_password: - separator = "&" if "?" in base else "?" - base = f"{base}{separator}password={urllib.parse.quote(ctx.login_password, safe='')}" - return base - - def login_candidate_paths(ctx: FlowContext) -> list[str]: paths = [ ctx.login_path or "/api/v1/auth/login/username", @@ -506,6 +711,7 @@ def login_candidate_paths(ctx: FlowContext) -> list[str]: def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force: bool = False) -> bool: + """Fetch a fresh token through the local auth package before business calls.""" if not ctx.auto_login: return False if ctx.dry_run: @@ -515,7 +721,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force record_error(state, stage, "自动登录失败：缺少 login_username") return False if not force and ctx.login_attempt_count > 0: - return has_authorization(ctx.headers) + return any(key.lower() == "authorization" and str(value).strip() for key, value in ctx.headers.items()) if force and ctx.login_attempt_count >= 2: return False @@ -525,8 +731,7 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force last_response = "" last_exc: BaseException | None = None for attempt, path in enumerate(login_candidate_paths(ctx), start=1): - url = build_login_url(ctx, path) - display_url = redact_url(url) + display_url = xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, path, ctx.login_password)) emit_operation_feedback( state, stage, @@ -537,63 +742,39 @@ def try_auto_login(ctx: FlowContext, state: dict[str, Any], stage: str, *, force attempt=attempt, ) try: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=ctx.timeout, context=ssl_context(ctx)) as resp: - raw = resp.read().decode("utf-8", errors="replace") - parsed = parse_json_lenient(raw) - token = extract_access_token(parsed) - if not token: - last_status = getattr(resp, "status", None) - last_response = raw - record_event( - state, - stage, - "登录接口已响应但未找到 access_token", - {"url": display_url, "status": last_status, "response": compact(raw, 1000)}, - ) - continue - ctx.headers["Authorization"] = auth_header_value(token) - record_event( - state, - stage, - "自动登录成功，已写入 Authorization Bearer token", - { - "url": display_url, - "status": getattr(resp, "status", None), - "username": ctx.login_username, - }, - ) - return True - except urllib.error.HTTPError as exc: - last_status = exc.code - last_response = exc.read().decode("utf-8", errors="replace") if exc.fp else "" - last_exc = exc - record_event( - state, - stage, - "登录接口返回错误，准备尝试下一个候选登录路径", - {"url": display_url, "status": exc.code, "response": compact(last_response, 1000)}, + result = xdfx_auth.fetch_access_token( + api_base=ctx.api_base, + username=ctx.login_username, + password=ctx.login_password, + login_path=path, + timeout=ctx.timeout, + verify_ssl=ctx.verify_ssl, ) - except Exception as exc: # noqa: BLE001 - auto-login must report every failure. - last_exc = exc - last_response = traceback.format_exc() + ctx.headers["Authorization"] = xdfx_auth.auth_header_value(result.token) + state["_tokenSource"] = "xdfx_auth" record_event( state, stage, - "登录接口调用异常，准备尝试下一个候选登录路径", - {"url": display_url, "exceptionType": type(exc).__name__, "exception": str(exc)}, + "启动阶段已通过 xdfx_auth 包获取 token，并写入 Authorization", + {"url": result.url, "status": result.status, "username": ctx.login_username}, + ) + return True + except xdfx_auth.AuthError as exc: + last_status = exc.status + last_response = exc.response + last_exc = exc + record_event( + state, + stage, + "xdfx_auth 获取 token 失败，准备尝试下一个候选登录路径", + {"url": exc.url, "status": exc.status, "response": compact(exc.response, 1000), "exception": str(exc)}, ) record_error( state, stage, - "自动登录失败，未能获取 access_token", - url=redact_url(build_login_url(ctx, ctx.login_path or "/api/v1/auth/login/username")), + "自动登录失败，未能通过 xdfx_auth 获取 access_token", + url=xdfx_auth.redact_url(xdfx_auth.build_login_url(ctx.api_base, ctx.login_path or "/api/v1/auth/login/username", ctx.login_password)), status=last_status, request=payload, response=last_response, @@ -612,6 +793,7 @@ def http_request_json_url( endpoint_key: str | None = None, allow_auto_login: bool = True, ) -> tuple[Any | None, str]: + """Call an HTTP JSON endpoint with logging, retries, and 401 recovery.""" method = method.upper() if ctx.dry_run: emit_operation_feedback(state, stage, method, url, endpoint_key=endpoint_key, payload=payload, dry_run=True) @@ -640,6 +822,7 @@ def http_request_json_url( except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") if exc.fp else "" if exc.code == 401 and allow_auto_login and ctx.auto_login: + # Refresh once on 401, then replay the original request exactly. refreshed = try_auto_login(ctx, state, f"{stage}/auto-login", force=True) if refreshed: record_event( @@ -709,6 +892,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p emit_operation_feedback(state, stage, "POST", url, endpoint_key=endpoint_key, payload=payload, dry_run=True) record_event(state, stage, f"DRY RUN POST {url}", {"payload": payload}) if endpoint_key.startswith("save_"): + # Synthetic ids let downstream dry-run code exercise relation wiring. synthetic_id = abs(hash(stage + stable_json(payload))) % 100000000 return {"id": synthetic_id, "dryRun": True}, json.dumps({"id": synthetic_id, "dryRun": True}) return {"dryRun": True}, json.dumps({"dryRun": True}) @@ -716,6 +900,7 @@ def http_post_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, p def parse_json_lenient(text: str) -> Any | None: + """Parse normal JSON plus adjacent/SSE-like JSON objects.""" stripped = text.strip() if not stripped: return None @@ -730,6 +915,7 @@ def parse_json_lenient(text: str) -> Any | None: def parse_concatenated_json_objects(text: str) -> list[Any]: + """Read streams such as {"answer": "..."}{"answer": "..."} or data: lines.""" decoder = json.JSONDecoder() objects: list[Any] = [] i = 0 @@ -766,6 +952,7 @@ def parse_concatenated_json_objects(text: str) -> list[Any]: def model_text_from_response(parsed: Any, raw: str) -> tuple[str, list[Any]]: + """Flatten streamed model chunks into one text buffer.""" objects: list[Any] if isinstance(parsed, list): objects = parsed @@ -824,6 +1011,7 @@ def extract_json_from_text(text: str) -> Any | None: def extract_json_candidates_from_text(text: str) -> list[Any]: + """Collect every plausible JSON object/array from messy model text.""" cleaned = strip_think(text) candidates: list[Any] = [] seen: set[str] = set() @@ -860,6 +1048,7 @@ def extract_json_candidates_from_text(text: str) -> list[Any]: def select_json_candidate(candidates: list[Any], endpoint_key: str, stage: str) -> Any | None: + """Pick the candidate whose keys best match the current step.""" if not candidates: return None scored = [(json_candidate_score(candidate, endpoint_key, stage), idx, candidate) for idx, candidate in enumerate(candidates)] @@ -1010,6 +1199,7 @@ def extract_first_balanced(text: str, open_ch: str, close_ch: str) -> str | None def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[str, list[Any], Any | None]: + """Call a model endpoint and retry only known empty/timeout/no-response cases.""" max_attempts = max(ctx.model_retries, 0) + 1 last_text = "" last_objects: list[Any] = [] @@ -1048,6 +1238,11 @@ def call_model_text(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, def call_model_json(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[dict[str, Any], str, list[Any]]: + """Return the best structured JSON for a model step. + + If the first parse fails, the second attempt includes an explicit repair + instruction so weak models can re-emit only the required JSON. + """ last_text = "" last_objects: list[Any] = [] last_candidate_count = 0 @@ -1202,23 +1397,85 @@ def profile_info(account: dict[str, Any]) -> str: def parse_index_list(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v).strip() for v in value if str(v).strip()] - text = str(value) - return [part.strip().strip("<>").strip() for part in re.split(r"[,，;；、\s]+", text) if part.strip()] + refs: list[str] = [] + + def add(raw: Any) -> None: + if raw is None: + return + if isinstance(raw, dict): + keys = ( + "编号id", + "编号ID", + "用户编号id", + "用户编号ID", + "用户编号", + "账号编号", + "引用序号", + "索引", + "index", + "id", + "ids", + "accountIndex", + "accountIndexes", + "userIndex", + "userIndexes", + ) + for key in keys: + if key in raw: + add(raw.get(key)) + return + if isinstance(raw, (list, tuple, set)): + for item in raw: + add(item) + return + text = str(raw).strip() + if not text: + return + angle_refs = re.findall(r"<<\s*([^<>]+?)\s*>>", text) + if angle_refs: + for token in angle_refs: + add(token) + return + for part in re.split(r"[,，;；、\s]+", text): + token = part.strip().strip("<>").strip() + if token: + refs.append(token) + + add(value) + deduped: list[str] = [] + seen: set[str] = set() + for ref in refs: + if ref in seen: + continue + seen.add(ref) + deduped.append(ref) + return deduped + + +def account_ref_numbers(account: dict[str, Any]) -> set[int]: + numbers: set[int] = set() + meta = account.get("metadata") if isinstance(account.get("metadata"), dict) else {} + for key in ("编号id", "编号ID", "用户编号id", "用户编号ID", "用户编号", "账号编号", "引用序号", "索引", "index", "accountIndex"): + for container in (account, meta): + number = as_int(container.get(key)) if isinstance(container, dict) else None + if number is not None: + numbers.add(number) + return numbers def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, Any] | None: + """Resolve model references like <<3>>, index numbers, names, or account ids.""" if not accounts: return None text = str(ref).strip().strip("<>").strip() number = as_int(text) if number is not None: for account in accounts: - if any(as_int(account.get(key)) == number for key in ("引用序号", "索引", "index")): + if number in account_ref_numbers(account): return account + # Step1 group extraction returns 编号/id values that refer to the + # previous zhanghao array index. Prefer exact index fields above, then + # fall back to zero-based and one-based array positions. if 0 <= number < len(accounts): return accounts[number] if 1 <= number <= len(accounts): @@ -1232,11 +1489,25 @@ def resolve_account_ref(ref: Any, accounts: list[dict[str, Any]]) -> dict[str, A def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize audience JSON whether the model used numbered fields or items[].""" items: list[dict[str, Any]] = [] for i in range(1, 21): group = first_present(data.get(f"群体{i}"), data.get(f"受众{i}"), data.get(f"targetAudience{i}")) reason = first_present(data.get(f"理由{i}"), data.get(f"推荐理由{i}"), data.get(f"recReason{i}")) - users = first_present(data.get(f"用户编号列表{i}"), data.get(f"用户列表{i}"), data.get(f"账号编号列表{i}")) + users = first_present( + data.get(f"用户编号列表{i}"), + data.get(f"编号id列表{i}"), + data.get(f"编号ID列表{i}"), + data.get(f"用户编号id列表{i}"), + data.get(f"用户编号ID列表{i}"), + data.get(f"用户列表{i}"), + data.get(f"账号编号列表{i}"), + data.get(f"索引列表{i}"), + data.get(f"编号{i}"), + data.get(f"id{i}"), + data.get(f"ids{i}"), + data.get(f"索引{i}"), + ) if group: items.append( { @@ -1255,7 +1526,27 @@ def extract_audiences(data: dict[str, Any]) -> list[dict[str, Any]]: { "targetAudience": str(group).strip(), "recReason": str(first_present(item.get("理由"), item.get("recReason"), item.get("reason")) or "").strip(), - "userIndexes": parse_index_list(first_present(item.get("用户编号列表"), item.get("users"), item.get("accounts"))), + "userIndexes": parse_index_list( + first_present( + item.get("用户编号列表"), + item.get("编号id列表"), + item.get("编号ID列表"), + item.get("用户编号id列表"), + item.get("用户编号ID列表"), + item.get("编号id"), + item.get("编号ID"), + item.get("用户编号id"), + item.get("用户编号ID"), + item.get("编号"), + item.get("索引"), + item.get("id"), + item.get("ids"), + item.get("users"), + item.get("accounts"), + item.get("userIndexes"), + item.get("accountIndexes"), + ) + ), "rank": i, } ) @@ -1272,6 +1563,7 @@ def audience_score(item: dict[str, Any]) -> tuple[int, int]: def select_top_audiences(items: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """Deduplicate audience names and keep the strongest candidates.""" unique: dict[str, dict[str, Any]] = {} for item in items: key = item.get("targetAudience", "") @@ -1301,6 +1593,7 @@ def pathway_score(item: dict[str, Any]) -> tuple[int, int, int]: def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge the same account found from different audiences before saving top 3.""" merged: dict[str, dict[str, Any]] = {} for item in candidates: account = item.get("account") or {} @@ -1325,6 +1618,7 @@ def merge_pathway_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: + """Normalize weak-point model output into a capped list.""" vulns: list[dict[str, str]] = [] for i in range(1, 21): vuln = first_present(data.get(f"脆弱点{i}"), data.get(f"vulnerability{i}"), data.get(f"vuln{i}")) @@ -1339,6 +1633,7 @@ def extract_vulns(data: dict[str, Any], limit: int) -> list[dict[str, str]]: def extract_strategies(data: dict[str, Any], limit: int) -> list[dict[str, Any]]: + """Normalize strategy output and preserve the modal text for later mapping.""" strategies: list[dict[str, Any]] = [] for i in range(1, 21): title = first_present(data.get(f"信息标题{i}"), data.get(f"主题{i}"), data.get(f"topic{i}")) @@ -1395,11 +1690,29 @@ def response_id(response: Any) -> int | None: return None +def response_rows(response: Any) -> list[Any]: + """Extract list rows from common CRUD response shapes.""" + if isinstance(response, list): + return response + if not isinstance(response, dict): + return [] + for key in ("data", "rows", "items", "records", "list"): + value = response.get(key) + if isinstance(value, list): + return value + rows = response_rows(value) + if rows: + return rows + return [] + + def save_with_id(ctx: FlowContext, state: dict[str, Any], endpoint_key: str, payload: dict[str, Any], stage: str) -> tuple[int | None, Any | None]: response, _ = http_post_json(ctx, state, endpoint_key, payload, stage) rid = response_id(response) if rid is None: - record_error(state, stage, "保存接口未返回 id，视为失败", request=payload, response=response) + record_error(state, stage, "保存接口未返回 ID，视为失败", request=payload, response=response) + else: + emit_save_feedback(state, stage, endpoint_key, payload, rid, dry_run=ctx.dry_run) return rid, response @@ -1429,6 +1742,7 @@ def parse_step_flags(raw: Any) -> dict[str, Any]: def mark_step_status(ctx: FlowContext, state: dict[str, Any], links: list[str], stage: str) -> None: + """Update the frontend step bar without overwriting unrelated step flags.""" if not ctx.sync_step_status: record_event(state, stage, "跳过 task-action.step 状态同步") return @@ -1477,12 +1791,126 @@ def strategy_platform(value: Any) -> Any: return platform_to_number(value) or 1 -def strategy_model_type(value: Any) -> str: - # Frontend defaults to text model type "1" when no modal type is selected. - raw = first_present(value, "1") - if isinstance(raw, list): - return ",".join(str(v) for v in raw if str(v).strip()) or "1" - return str(raw or "1") +def choice_tokens(value: Any) -> list[Any]: + """Split frontend choice values from arrays, JSON strings, and mixed delimiters.""" + if value is None: + return [] + if isinstance(value, dict): + return choice_tokens( + first_present(value.get("value"), value.get("dictValue"), value.get("label"), value.get("dictLabel")) + ) + if isinstance(value, (list, tuple, set)): + tokens: list[Any] = [] + for item in value: + tokens.extend(choice_tokens(item)) + return tokens + text = str(value).strip() + if not text: + return [] + if text[0] in "[{": + try: + parsed = json.loads(text) + if parsed != value: + return choice_tokens(parsed) + except Exception: + pass + return [part.strip() for part in re.split(r"[,，、;/|]+", text) if part.strip()] + + +def dict_choices( + ctx: FlowContext, + state: dict[str, Any], + dict_type: str, + stage: str, +) -> list[dict[str, Any]]: + """Fetch and cache frontend dictionary choices such as modelType.""" + cache = state.setdefault("_dictChoiceCache", {}) + if dict_type in cache: + cached = cache[dict_type] + return cached if isinstance(cached, list) else [] + if ctx.dry_run: + cache[dict_type] = [] + return [] + + query = urllib.parse.urlencode({"filter": f"dictType||$eq||{dict_type}", "limit": 999, "page": 1}) + url = f"{url_for(ctx, 'dict_data')}?{query}" + response, _ = http_request_json_url(ctx, state, "GET", url, None, stage, endpoint_key="dict_data") + choices: list[dict[str, Any]] = [] + for row in response_rows(response): + if not isinstance(row, dict): + continue + label = str(first_present(row.get("dictLabel"), row.get("label"), "") or "").strip() + raw_value = first_present(row.get("dictValue"), row.get("value")) + if not label or raw_value is None: + continue + number = as_int(raw_value) + choices.append({"label": label, "value": number if number is not None else str(raw_value).strip()}) + for alias in choice_tokens(row.get("alias")): + alias_label = str(alias).strip() + if alias_label: + choices.append({"label": alias_label, "value": number if number is not None else str(raw_value).strip()}) + cache[dict_type] = choices + return choices + + +def choice_value_by_label(choices: list[dict[str, Any]] | None, label: str) -> Any | None: + needle = re.sub(r"\s+", "", str(label or "").lower()) + if not needle: + return None + for item in choices or []: + item_label = re.sub(r"\s+", "", str(item.get("label") or "").lower()) + if item_label and (needle == item_label or needle in item_label or item_label in needle): + return item.get("value") + return None + + +def model_type_token_value(token: Any, choices: list[dict[str, Any]] | None = None) -> Any: + """Convert modal labels like 图片/视频 into frontend dictionary values.""" + number = as_int(token) + if number is not None: + return number + + text = str(token or "").strip() + lower = text.lower() + exact = choice_value_by_label(choices, text) + if exact is not None: + return exact + + label = "" + fallback_value = 2 + if "视" in text or "video" in lower: + label, fallback_value = "视频", 3 + elif "图" in text or "image" in lower or "picture" in lower or "photo" in lower: + label, fallback_value = "图片", 2 + elif "音" in text or "audio" in lower or "sound" in lower: + label, fallback_value = "音频", 4 + elif "发文" in text or "文本" in text or "文字" in text or "text" in lower: + label, fallback_value = "文本", 1 + + if label: + matched = choice_value_by_label(choices, label) + if matched is not None: + return matched + return fallback_value + + # Mirrors the frontend helper's unknown-token default. + return 2 + + +def strategy_model_type(value: Any, choices: list[dict[str, Any]] | None = None) -> str: + """Return the comma-separated value string expected by task-action-message.""" + tokens = choice_tokens(value) + if not tokens: + # Frontend save paths fall back to "1" when no modal type is selected. + return "1" + values: list[str] = [] + seen: set[str] = set() + for token in tokens: + normalized = str(model_type_token_value(token, choices)).strip() + if normalized and normalized not in seen: + values.append(normalized) + seen.add(normalized) + return ",".join(values) or "1" def strategy_language(value: Any) -> int: @@ -1501,11 +1929,28 @@ def save_strategy_details( *, audience: bool, ) -> None: - platform = strategy_platform(first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {}))) + """Save all frontend-readable strategy child rows. + + The strategy table and arrangement page do not read only the main strategy + row; they also expect channel, narrate, message, and relation rows. + """ + platform = strategy_platform( + first_present(strategy.get("platform"), state.get("platform"), platform_label(item.get("account") or {})) + ) + account_type = str( + first_present( + strategy.get("accountType"), + item.get("accountType"), + item.get("type"), + "目标受众账号" if audience else "关键传播节点账号", + ) + or "" + ).strip() persona = strategy.get("persona") or "专业议题分析者" persona_size = chat_number(strategy.get("personaSize"), 3) responding_count = chat_number(first_present(strategy.get("respondingCount"), strategy.get("数量")), 10) title = str(strategy.get("title") or "").strip() + model_type_choices = dict_choices(ctx, state, "modelType", stage + "/dict-modelType") channel_id, _ = save_with_id( ctx, @@ -1516,6 +1961,7 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "platform": platform, + "accountType": account_type, "personaJson": json.dumps([{"persona": persona, "size": persona_size}], ensure_ascii=False), }, stage + "/save-channel", @@ -1531,7 +1977,10 @@ def save_strategy_details( "actionId": as_int(state["actionId"]) or state["actionId"], "strategyId": strategy_id, "channelId": channel_id, - "modelType": strategy_model_type(first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType"))), + "modelType": strategy_model_type( + first_present(strategy.get("format"), strategy.get("信息形式"), strategy.get("modelType")), + model_type_choices, + ), "language": strategy_language(first_present(strategy.get("language"), strategy.get("语种"))), "respondingCount": responding_count, "messageCount": responding_count, @@ -1564,7 +2013,39 @@ def object_for_audience(audience: dict[str, Any]) -> dict[str, Any]: } +def downstream_limit(ctx: FlowContext, *, audience: bool) -> int: + """Limit expensive downstream analysis while keeping 3 upstream rows stored.""" + raw = ctx.downstream_audiences if audience else ctx.downstream_pathways + return max(1, raw) + + +def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, audience: bool) -> list[dict[str, Any]]: + """Select the 1-2 best stored rows for vulnerability/strategy generation.""" + source_key = "topAudiences" if audience else "topPathways" + id_key = "designId" if audience else "pathwayAccountId" + selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways" + items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)] + limit = downstream_limit(ctx, audience=audience) + selected = items[:limit] + state[selected_key] = [ + { + "id": item.get(id_key), + "name": item.get("targetAudience") if audience else item.get("accountName"), + } + for item in selected + ] + if items: + record_event( + state, + stage, + f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}", + {"limit": limit, "selected": state[selected_key]}, + ) + return selected + + def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze audiences, save top 3, and attach any referenced accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step1-audience" if state.get("topAudiences") and not ctx.dry_run: @@ -1582,12 +2063,15 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") state["audienceRawText"] = text + # The first model call may return raw text plus embedded account objects. accounts = accounts_from_response(parsed, objects, text) state["audienceAccounts"] = accounts + # The second model call converts the long audience analysis into numbered + # groups that can be saved as task-action-design rows. group_payload = { "mubiao": state["mubiao"], - "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", "zhanghao": accounts, "history": [], "stream": True, @@ -1602,6 +2086,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: top = select_top_audiences(candidates, ctx.max_audiences) saved: list[dict[str, Any]] = [] for item in top: + # Save the audience first; account rows and downstream analyses depend + # on the returned designId. design_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], @@ -1614,6 +2100,8 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: continue item = {**item, "designId": design_id, "saveResponse": resp, "accounts": [], "savedAccountIds": []} for ref in item.get("userIndexes") or []: + # Model references are intentionally fuzzy, so resolve them against + # both explicit indexes and account names/ids. account = resolve_account_ref(ref, accounts) if account is None: record_error(state, stage + "/save-design-account", f"用户编号 {ref} 未匹配到候选账号") @@ -1645,6 +2133,7 @@ def run_step1(ctx: FlowContext, state: dict[str, Any]) -> None: def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: + """Analyze key pathway accounts for saved audiences and store top 3.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step2-pathway" audiences = [a for a in state.get("topAudiences") or [] if a.get("designId")] @@ -1656,6 +2145,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: return all_candidates: list[dict[str, Any]] = [] for audience in audiences: + # Analyze pathway candidates in the context of each saved audience. payload = { "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请分析哪些社交账号是你本次任务的核心传播节点，通过撬动该账号可实现扩大传播影响力的目的。可以从账号粉丝量、发文内容互动量、所在领域的影响力、账号画像与你传播议题的贴合度等角度分析。", "otherinfo": { @@ -1673,6 +2163,8 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: } text, objects, parsed = call_model_text(ctx, state, "aiuser", payload, stage + "/analyze") accounts = accounts_from_response(parsed, objects, text) + # Ask the model to point back to candidate accounts, then resolve those + # references to concrete account objects. spread_payload = { "mubiao": state["mubiao"], "user_message": "分析其中关键信息。请从候选账号中抽取核心传播节点，字段必须是：账号1、理由1、账号2、理由2 ...，最多输出5个。账号字段优先返回候选数组的引用编号或索引，只返回JSON。", @@ -1707,6 +2199,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None: top = sorted(merged, key=pathway_score, reverse=True)[: ctx.max_pathways] saved: list[dict[str, Any]] = [] for item in top: + # Save the account row before writing the design-pathway relation. account = item["account"] aid = account_id(account) name = account_name(account) @@ -1757,12 +2250,14 @@ def run_step4(ctx: FlowContext, state: dict[str, Any]) -> None: def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate and save weak points for selected audiences or pathway accounts.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step3-audience-vulns" if audience else "step4-pathway-vulns" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) output_key = "audienceVulns" if audience else "pathwayVulns" state.setdefault(output_key, {}) for item in items: + # owner_id is designId for audiences and pathwayAccountId for accounts. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") if not owner_id: continue @@ -1789,6 +2284,8 @@ def run_vuln_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> vulns = extract_vulns(data, ctx.max_vulns) saved: list[dict[str, Any]] = [] for vuln in vulns: + # The two weak-point tables have nearly identical payloads but + # different foreign-key names. if audience: save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], @@ -1835,14 +2332,16 @@ def run_step6(ctx: FlowContext, state: dict[str, Any]) -> None: def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool) -> None: + """Generate strategies from saved weak points and persist frontend child rows.""" require_context(state, "taskId", "actionId", "mubiao") stage = "step5-audience-strategy" if audience else "step6-pathway-strategy" - items = state.get("topAudiences" if audience else "topPathways") or [] + items = downstream_items(ctx, state, stage, audience=audience) vulns_by_owner = state.get("audienceVulns" if audience else "pathwayVulns") or {} output_key = "audienceStrategies" if audience else "pathwayStrategies" state.setdefault(output_key, []) existing_topics = {s.get("topic") for s in state[output_key] if isinstance(s, dict)} for item in items: + # Strategies are only meaningful after the matching weak points exist. owner_id = item.get("designId") if audience else item.get("pathwayAccountId") vulns = vulns_by_owner.get(str(owner_id)) or [] vuln_texts = [v.get("vulnerabilities") for v in vulns if v.get("vulnerabilities")] @@ -1876,16 +2375,20 @@ def run_strategy_step(ctx: FlowContext, state: dict[str, Any], *, audience: bool continue if topic in existing_topics: continue + # Save the main row first; channel/narrate/message/relation rows all + # require the returned strategy id. save_payload = { "taskId": as_int(state["taskId"]) or state["taskId"], "actionId": as_int(state["actionId"]) or state["actionId"], "targetAudience": target, "accountType": account_type, "topic": topic, - "strategyTitle": "", + "strategyTitle": str(strategy.get("title") or "").strip(), } strategy_id, resp = save_with_id(ctx, state, "save_strategy", save_payload, stage + "/save") if strategy_id is not None: + # Frontend strategy pages read these child tables, so a strategy + # is not complete until this helper succeeds as far as possible. save_strategy_details(ctx, state, strategy_id, item, strategy, topic, stage, audience=audience) existing_topics.add(topic) state[output_key].append({**strategy, "id": strategy_id, "targetAudience": target, "accountType": account_type, "topic": topic, "saveResponse": resp}) @@ -1914,13 +2417,14 @@ STEP_LABELS = { def state_counts(state: dict[str, Any]) -> dict[str, int]: + """Summarize stored objects for step feedback.""" return { - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceVulns": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), - "pathwayVulns": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "受众脆弱点数量": sum(len(v) for v in (state.get("audienceVulns") or {}).values()), + "传播节点脆弱点数量": sum(len(v) for v in (state.get("pathwayVulns") or {}).values()), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), } @@ -1932,23 +2436,26 @@ def emit_step_feedback( status: str | None = None, note: str = "", ) -> None: + # One feedback line per step keeps the single-skill and split-skill versions + # equally transparent to the user. total_errors = len(state.get("errors") or []) new_errors = max(0, total_errors - before_error_count) final_status = status or ("partial" if new_errors else "ok") payload = { - "type": "STEP_FEEDBACK", - "step": step, - "label": STEP_LABELS.get(step, step), - "status": final_status, - "note": note, - "newErrors": new_errors, - "totalErrors": total_errors, - "counts": state_counts(state), - "state": state.get("_statePath", ""), - "summary": state.get("_summaryPath", ""), - "updatedAt": state.get("updatedAt"), + "日志类型": "步骤反馈", + "步骤": STEP_NUMBER_LABELS.get(step, step), + "步骤名称": STEP_LABELS.get(step, step), + "状态": STATUS_LABELS.get(final_status, final_status), + "执行状态": status_label(final_status), + "备注": note, + "本步骤新增错误数": new_errors, + "累计错误数": total_errors, + "累计入库数量": state_counts(state), + "状态文件": state.get("_statePath", ""), + "摘要文件": state.get("_summaryPath", ""), + "更新时间": state.get("updatedAt"), } - print("STEP_FEEDBACK " + json.dumps(payload, ensure_ascii=False), flush=True) + print(LOG_PREFIXES["step"] + " " + json.dumps(payload, ensure_ascii=False), flush=True) def finish_step( @@ -1960,6 +2467,8 @@ def finish_step( status: str | None = None, note: str = "", ) -> None: + # Always persist before emitting step feedback so the user can inspect files + # immediately after a reported step completes. state["updatedAt"] = utc_now() save_json_file(ctx.state_path, state) if ctx.summary_path: @@ -1980,6 +2489,11 @@ def skip_step_with_feedback(ctx: FlowContext, state: dict[str, Any], step: str, def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: + """Legacy full runner. + + The wrapper script is preferred for real-time logs, but this remains useful + for dry-runs and unattended local troubleshooting. + """ run_step_with_feedback(ctx, state, "step1") run_step_with_feedback(ctx, state, "step2") run_step_with_feedback(ctx, state, "step3") @@ -1995,6 +2509,7 @@ def run_full(ctx: FlowContext, state: dict[str, Any]) -> None: def write_summary(path: Path, state: dict[str, Any]) -> None: + """Write a human-readable checkpoint beside the machine state JSON.""" audience_vuln_count = sum(len(v) for v in (state.get("audienceVulns") or {}).values()) pathway_vuln_count = sum(len(v) for v in (state.get("pathwayVulns") or {}).values()) lines = [ @@ -2005,6 +2520,8 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: f"- actionId：{state.get('actionId')}", f"- 受众入库：{len(state.get('topAudiences') or [])}", f"- 传播节点入库：{len(state.get('topPathways') or [])}", + f"- 下游受众选择：{len(state.get('selectedDownstreamAudiences') or [])}", + f"- 下游传播节点选择：{len(state.get('selectedDownstreamPathways') or [])}", f"- 受众脆弱点入库：{audience_vuln_count}", f"- 传播节点脆弱点入库：{pathway_vuln_count}", f"- 受众策略入库：{len(state.get('audienceStrategies') or [])}", @@ -2021,9 +2538,9 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: if state.get("errors"): lines += ["", "## 失败明细"] for err in state["errors"]: - lines.append(f"- [{err.get('stage')}] {err.get('message')} status={err.get('status')} url={err.get('url')}") + lines.append(f"- [{stage_label(err.get('stage'))}] {err.get('message')} 状态码={err.get('status')} 接口={err.get('url')}") if err.get("response"): - lines.append(f" - response: {err.get('response')}") + lines.append(f" - 响应摘要：{err.get('response')}") text = "\n".join(lines) + "\n" try: path.parent.mkdir(parents=True, exist_ok=True) @@ -2034,34 +2551,34 @@ def write_summary(path: Path, state: dict[str, Any]) -> None: def build_arg_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Run xdfx analysis/storage flow.") + parser = argparse.ArgumentParser(description="运行 xdfx 分析与入库流程。") parser.add_argument("--mode", choices=["full", *STEP_FUNCS.keys()], default="full") - parser.add_argument("--input", help="JSON config file. Keys: api_base, taskId, actionId, mubiao, direction, platform, headers.") - parser.add_argument("--state", default="xdfx-flow-state.json", help="State JSON path for resume and step handoff.") - parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown summary output path.") + parser.add_argument("--input", help="JSON 配置文件。常用字段：api_base、taskId、actionId、mubiao、direction、platform、headers。") + parser.add_argument("--state", default="xdfx-flow-state.json", help="用于断点续跑和步骤交接的状态 JSON 文件路径。") + parser.add_argument("--summary", default="xdfx-flow-summary.md", help="Markdown 执行摘要输出路径。") parser.add_argument("--api-base") parser.add_argument("--task-id") parser.add_argument("--action-id") parser.add_argument("--mubiao") parser.add_argument("--direction", default=None) parser.add_argument("--platform", default=None) - parser.add_argument("--auth-token", default=os.getenv("XDFX_AUTH_TOKEN")) - parser.add_argument("--authorization", default=os.getenv("AUTHORIZATION")) - parser.add_argument("--header", action="append", default=[], help="Extra HTTP header, format 'Key: Value'.") - parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="Username for automatic login when Authorization is missing or stale.") - parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="Password query parameter for automatic username login.") - parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="Login endpoint path or absolute URL.") - parser.add_argument("--no-auto-login", action="store_true", help="Disable automatic username login and 401 token refresh.") - parser.add_argument("--verify-ssl", action="store_true", help="Verify HTTPS certificates. Default is disabled for internal/UAT certificates.") + parser.add_argument("--header", action="append", default=[], help="附加非鉴权 HTTP 请求头，格式为 'Key: Value'；Authorization 会被忽略。") + parser.add_argument("--login-username", default=os.getenv("XDFX_LOGIN_USERNAME", "admin"), help="xdfx_auth 启动阶段自动登录使用的用户名。") + parser.add_argument("--login-password", default=os.getenv("XDFX_LOGIN_PASSWORD", "ch@user"), help="用户名登录接口的 password 查询参数。") + parser.add_argument("--login-path", default=os.getenv("XDFX_LOGIN_PATH", "/api/v1/auth/login/username"), help="登录接口路径或完整 URL。") + parser.add_argument("--no-auto-login", action="store_true", help="关闭 xdfx_auth 自动登录和 401 刷新。") + parser.add_argument("--verify-ssl", action="store_true", help="校验 HTTPS 证书。默认不校验，以兼容内网/UAT 自签证书。") parser.add_argument("--timeout", type=float, default=90) parser.add_argument("--retries", type=int, default=2) parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--no-step-status", action="store_true", help="Do not update task/task-action.step after completed steps.") - parser.add_argument("--model-retries", type=int, default=1, help="Retry model/stream endpoints when the first response is empty, timeout-like, or says the model did not respond.") + parser.add_argument("--no-step-status", action="store_true", help="步骤完成后不更新 task/task-action.step。") + parser.add_argument("--model-retries", type=int, default=1, help="模型/流式接口首次空响应、超时或提示模型无响应时的重试次数。") parser.add_argument("--max-audiences", type=int, default=3) parser.add_argument("--max-pathways", type=int, default=3) parser.add_argument("--max-vulns", type=int, default=3) parser.add_argument("--max-strategies", type=int, default=3) + parser.add_argument("--downstream-audiences", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库受众数量，默认 2。") + parser.add_argument("--downstream-pathways", type=int, default=2, help="继续进入脆弱点/策略步骤的已入库传播节点数量，默认 2。") return parser @@ -2070,6 +2587,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) state_path = Path(args.state) state = default_state() + # Load previous state first, then overlay input JSON and finally CLI flags. + # That ordering makes resume safe while still allowing explicit overrides. if state_path.exists(): loaded = load_json_file(state_path) if isinstance(loaded, dict): @@ -2081,9 +2600,7 @@ def main(argv: list[str] | None = None) -> int: merge_config_into_state(state, config) input_headers = config.get("headers") if isinstance(input_headers, dict): - args.header.extend(f"{k}: {v}" for k, v in input_headers.items()) - args.auth_token = args.auth_token or config.get("auth_token") - args.authorization = args.authorization or config.get("authorization") + args.header.extend(f"{k}: {v}" for k, v in input_headers.items() if str(k).lower() != "authorization") if config.get("login_username") is not None: args.login_username = str(config.get("login_username") or "") if config.get("login_password") is not None: @@ -2094,6 +2611,10 @@ def main(argv: list[str] | None = None) -> int: args.no_auto_login = True if config.get("verify_ssl") is not None: args.verify_ssl = bool(config.get("verify_ssl")) + if config.get("downstream_audiences") is not None: + args.downstream_audiences = as_int(config.get("downstream_audiences")) or args.downstream_audiences + if config.get("downstream_pathways") is not None: + args.downstream_pathways = as_int(config.get("downstream_pathways")) or args.downstream_pathways cli_config = { "api_base": args.api_base, @@ -2105,7 +2626,8 @@ def main(argv: list[str] | None = None) -> int: } merge_config_into_state(state, {k: v for k, v in cli_config.items() if v is not None}) api_base = state.get("api_base") or "https://ch1.b.uat.4.cn" - headers = parse_headers(args.header, args.auth_token, args.authorization) + headers = parse_headers(args.header) + # ctx owns runtime knobs; state owns business data and resumable ids. ctx = FlowContext( api_base=api_base, headers=headers, @@ -2118,6 +2640,8 @@ def main(argv: list[str] | None = None) -> int: max_pathways=max(args.max_pathways, 1), max_vulns=max(args.max_vulns, 1), max_strategies=max(args.max_strategies, 1), + downstream_audiences=max(args.downstream_audiences, 1), + downstream_pathways=max(args.downstream_pathways, 1), sync_step_status=not args.no_step_status, model_retries=max(args.model_retries, 0), verify_ssl=bool(args.verify_ssl), @@ -2131,11 +2655,17 @@ def main(argv: list[str] | None = None) -> int: state["_summaryPath"] = str(ctx.summary_path) if ctx.summary_path else "" state["_autoLogin"] = ctx.auto_login state["_verifySsl"] = ctx.verify_ssl + state["_downstreamLimits"] = { + "audiences": ctx.downstream_audiences, + "pathways": ctx.downstream_pathways, + } record_event(state, "runner", f"start mode={args.mode} dry_run={ctx.dry_run}") try: require_context(state, "taskId", "actionId", "mubiao") - if ctx.auto_login and not has_authorization(ctx.headers): - try_auto_login(ctx, state, "runner/auto-login") + if ctx.auto_login and not ctx.dry_run: + # Always acquire a token through the imported auth package before the first business call. + if not try_auto_login(ctx, state, "runner/initial-login", force=True): + raise FlowError("启动阶段获取 token 失败，已停止执行，避免无 token 调用业务接口") if args.mode == "full": run_full(ctx, state) else: @@ -2145,13 +2675,14 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(f"ERROR: {exc}", file=sys.stderr) + print(f"错误：{exc}", file=sys.stderr) return 2 except Exception as exc: # noqa: BLE001 record_error(state, "runner", "未捕获异常", response=traceback.format_exc(), exception=exc) save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) + print("未捕获异常堆栈：", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) return 1 @@ -2159,20 +2690,23 @@ def main(argv: list[str] | None = None) -> int: save_json_file(state_path, state) if ctx.summary_path: write_summary(ctx.summary_path, state) - print(json.dumps( - { - "ok": True, - "mode": args.mode, - "state": str(state_path), - "summary": str(ctx.summary_path) if ctx.summary_path else "", - "audiences": len(state.get("topAudiences") or []), - "pathways": len(state.get("topPathways") or []), - "audienceStrategies": len(state.get("audienceStrategies") or []), - "pathwayStrategies": len(state.get("pathwayStrategies") or []), - "errors": len(state.get("errors") or []), - }, - ensure_ascii=False, - )) + # Final JSON is intentionally compact; detailed per-step status has already + # been emitted as Chinese step feedback. + final_payload = { + "日志类型": "执行完成", + "是否成功": "是", + "执行模式": mode_label(args.mode), + "状态文件": str(state_path), + "摘要文件": str(ctx.summary_path) if ctx.summary_path else "", + "受众数量": len(state.get("topAudiences") or []), + "传播节点数量": len(state.get("topPathways") or []), + "已选择受众数": len(state.get("selectedDownstreamAudiences") or []), + "已选择传播节点数": len(state.get("selectedDownstreamPathways") or []), + "受众策略数量": len(state.get("audienceStrategies") or []), + "传播节点策略数量": len(state.get("pathwayStrategies") or []), + "错误数": len(state.get("errors") or []), + } + print(LOG_PREFIXES["done"] + " " + json.dumps(final_payload, ensure_ascii=False)) return 0 diff --git a/xdfx-auth-index-check-state.json b/xdfx-auth-index-check-state.json new file mode 100644 index 00000000..98090243 --- /dev/null +++ b/xdfx-auth-index-check-state.json @@ -0,0 +1,80 @@ +{ + "api_base": "https://ch1.b.uat.4.cn", + "taskId": "1", + "actionId": "2", + "mubiao": "索引映射验证", + "direction": "U方向", + "platform": "", + "audienceRawText": "{\"dryRun\": true}", + "audienceCandidates": [], + "topAudiences": [], + "pathwayCandidates": [], + "topPathways": [], + "audienceVulns": {}, + "pathwayVulns": {}, + "audienceStrategies": [], + "pathwayStrategies": [], + "errors": [], + "events": [ + { + "time": "2026-06-29T08:49:10.660975+00:00", + "stage": "runner", + "message": "start mode=step1 dry_run=True", + "data": null + }, + { + "time": "2026-06-29T08:49:10.662968+00:00", + "stage": "step1-audience/analyze/model-call-attempt-1", + "message": "DRY RUN POST https://ch1.b.uat.4.cn/gpt/chat/chat_aiuser", + "data": { + "payload": { + "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请详细分析你在各社交平台的受众目标群体，可以从账号画像和账号近期发文等方面考虑。可以从受众目标的兴趣爱好、关注议题、职业定位、政治立场等方面划分。", + "otherinfo": { + "方向": "U方向" + }, + "history": [], + "stream": true, + "model_name": "OpenAI3", + "temperature": 0.7, + "max_tokens": 20000, + "prompt_name": "任务背景和目的：索引映射验证" + } + } + }, + { + "time": "2026-06-29T08:49:10.668998+00:00", + "stage": "step1-audience/extract/model-json-attempt-1/model-call-attempt-1", + "message": "DRY RUN POST https://ch1.b.uat.4.cn/gpt/chat/chat_qianqian_group_chat", + "data": { + "payload": { + "mubiao": "索引映射验证", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体。字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。用户编号列表中的编号/id必须对应本次入参 zhanghao 数组里的账号索引值；例如返回 0 或 <<0>> 表示 zhanghao[0]，返回 1 或 <<1>> 表示 zhanghao[1]。不要生成不存在于 zhanghao 的编号。", + "zhanghao": [], + "history": [], + "stream": true, + "model_name": "OpenAI3", + "temperature": 0.7, + "max_tokens": 20000, + "prompt_name": "索引映射验证" + } + } + }, + { + "time": "2026-06-29T08:49:10.668998+00:00", + "stage": "step1-audience", + "message": "完成，保存受众 0 个", + "data": null + } + ], + "startedAt": "2026-06-29T08:49:10.659972+00:00", + "updatedAt": "2026-06-29T08:49:10.674982+00:00", + "_statePath": "C:\\tmp\\xdfx-auth-index-check-state.json", + "_summaryPath": "C:\\tmp\\xdfx-auth-index-check-summary.md", + "_autoLogin": true, + "_verifySsl": false, + "_downstreamLimits": { + "audiences": 2, + "pathways": 2 + }, + "audienceAccounts": [] +} \ No newline at end of file diff --git a/xdfx-auth-index-check-summary.md b/xdfx-auth-index-check-summary.md new file mode 100644 index 00000000..85b459cb --- /dev/null +++ b/xdfx-auth-index-check-summary.md @@ -0,0 +1,18 @@ +# xdfx 分析入库执行摘要 + +- 更新时间：2026-06-29T08:49:10.674982+00:00 +- taskId：1 +- actionId：2 +- 受众入库：0 +- 传播节点入库：0 +- 下游受众选择：0 +- 下游传播节点选择：0 +- 受众脆弱点入库：0 +- 传播节点脆弱点入库：0 +- 受众策略入库：0 +- 传播节点策略入库：0 +- 错误数：0 + +## 受众 + +## 传播节点 diff --git a/xdfx-log-check-state.json b/xdfx-log-check-state.json new file mode 100644 index 00000000..1ad0e726 --- /dev/null +++ b/xdfx-log-check-state.json @@ -0,0 +1,88 @@ +{ + "api_base": "https://ch1.b.uat.4.cn", + "taskId": "1", + "actionId": "2", + "mubiao": "日志中文化验证", + "direction": "U方向", + "platform": "", + "audienceRawText": "{\"dryRun\": true}", + "audienceCandidates": [], + "topAudiences": [], + "pathwayCandidates": [], + "topPathways": [], + "audienceVulns": {}, + "pathwayVulns": {}, + "audienceStrategies": [], + "pathwayStrategies": [], + "errors": [], + "events": [ + { + "time": "2026-06-29T07:16:23.252171+00:00", + "stage": "runner", + "message": "start mode=step1 dry_run=True", + "data": null + }, + { + "time": "2026-06-29T07:16:23.252171+00:00", + "stage": "runner/auto-login", + "message": "dry-run 跳过自动登录", + "data": { + "username": "admin" + } + }, + { + "time": "2026-06-29T07:16:23.253180+00:00", + "stage": "step1-audience/analyze/model-call-attempt-1", + "message": "DRY RUN POST https://ch1.b.uat.4.cn/gpt/chat/chat_aiuser", + "data": { + "payload": { + "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请详细分析你在各社交平台的受众目标群体，可以从账号画像和账号近期发文等方面考虑。可以从受众目标的兴趣爱好、关注议题、职业定位、政治立场等方面划分。", + "otherinfo": { + "方向": "U方向" + }, + "history": [], + "stream": true, + "model_name": "OpenAI3", + "temperature": 0.7, + "max_tokens": 20000, + "prompt_name": "任务背景和目的：日志中文化验证" + } + } + }, + { + "time": "2026-06-29T07:16:23.261131+00:00", + "stage": "step1-audience/extract/model-json-attempt-1/model-call-attempt-1", + "message": "DRY RUN POST https://ch1.b.uat.4.cn/gpt/chat/chat_qianqian_group_chat", + "data": { + "payload": { + "mubiao": "日志中文化验证", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "zhanghao": [], + "history": [], + "stream": true, + "model_name": "OpenAI3", + "temperature": 0.7, + "max_tokens": 20000, + "prompt_name": "日志中文化验证" + } + } + }, + { + "time": "2026-06-29T07:16:23.261131+00:00", + "stage": "step1-audience", + "message": "完成，保存受众 0 个", + "data": null + } + ], + "startedAt": "2026-06-29T07:16:23.252171+00:00", + "updatedAt": "2026-06-29T07:16:23.266115+00:00", + "_statePath": "C:\\tmp\\xdfx-log-check-state.json", + "_summaryPath": "C:\\tmp\\xdfx-log-check-summary.md", + "_autoLogin": true, + "_verifySsl": false, + "_downstreamLimits": { + "audiences": 2, + "pathways": 2 + }, + "audienceAccounts": [] +} \ No newline at end of file diff --git a/xdfx-log-check-summary.md b/xdfx-log-check-summary.md new file mode 100644 index 00000000..add40f51 --- /dev/null +++ b/xdfx-log-check-summary.md @@ -0,0 +1,18 @@ +# xdfx 分析入库执行摘要 + +- 更新时间：2026-06-29T07:16:23.266115+00:00 +- taskId：1 +- actionId：2 +- 受众入库：0 +- 传播节点入库：0 +- 下游受众选择：0 +- 下游传播节点选择：0 +- 受众脆弱点入库：0 +- 传播节点脆弱点入库：0 +- 受众策略入库：0 +- 传播节点策略入库：0 +- 错误数：0 + +## 受众 + +## 传播节点 diff --git a/xdfx-runner-dry-state.json b/xdfx-runner-dry-state.json new file mode 100644 index 00000000..127027a9 --- /dev/null +++ b/xdfx-runner-dry-state.json @@ -0,0 +1,88 @@ +{ + "api_base": "https://ch1.b.uat.4.cn", + "taskId": "1", + "actionId": "2", + "mubiao": "dry-run-test", + "direction": "U方向", + "platform": "fb", + "audienceRawText": "{\"dryRun\": true}", + "audienceCandidates": [], + "topAudiences": [], + "pathwayCandidates": [], + "topPathways": [], + "audienceVulns": {}, + "pathwayVulns": {}, + "audienceStrategies": [], + "pathwayStrategies": [], + "errors": [], + "events": [ + { + "time": "2026-06-29T03:43:58.007253+00:00", + "stage": "runner", + "message": "start mode=step1 dry_run=True", + "data": null + }, + { + "time": "2026-06-29T03:43:58.007253+00:00", + "stage": "runner/auto-login", + "message": "dry-run 跳过自动登录", + "data": { + "username": "admin" + } + }, + { + "time": "2026-06-29T03:43:58.007253+00:00", + "stage": "step1-audience/analyze/model-call-attempt-1", + "message": "DRY RUN POST https://ch1.b.uat.4.cn/gpt/chat/chat_aiuser", + "data": { + "payload": { + "usermessage": "你是中国对台湾方向的认知域专家，你要围绕驱动目标传播行动中的叙事主题，请详细分析你在各社交平台的受众目标群体，可以从账号画像和账号近期发文等方面考虑。可以从受众目标的兴趣爱好、关注议题、职业定位、政治立场等方面划分。", + "otherinfo": { + "方向": "U方向" + }, + "history": [], + "stream": true, + "model_name": "OpenAI3", + "temperature": 0.7, + "max_tokens": 20000, + "prompt_name": "任务背景和目的：dry-run-test" + } + } + }, + { + "time": "2026-06-29T03:43:58.017225+00:00", + "stage": "step1-audience/extract/model-json-attempt-1/model-call-attempt-1", + "message": "DRY RUN POST https://ch1.b.uat.4.cn/gpt/chat/chat_qianqian_group_chat", + "data": { + "payload": { + "mubiao": "dry-run-test", + "user_message": "分析其中关键信息。请基于上一轮受众分析结果，抽取受众目标群体，字段必须是：群体1、用户编号列表1、理由1、群体2、用户编号列表2、理由2 ...，最多输出5个，只返回JSON。", + "zhanghao": [], + "history": [], + "stream": true, + "model_name": "OpenAI3", + "temperature": 0.7, + "max_tokens": 20000, + "prompt_name": "dry-run-test" + } + } + }, + { + "time": "2026-06-29T03:43:58.018223+00:00", + "stage": "step1-audience", + "message": "完成，保存受众 0 个", + "data": null + } + ], + "startedAt": "2026-06-29T03:43:58.006257+00:00", + "updatedAt": "2026-06-29T03:43:58.021216+00:00", + "_statePath": "C:\\tmp\\xdfx-runner-dry-state.json", + "_summaryPath": "C:\\tmp\\xdfx-runner-dry-summary.md", + "_autoLogin": true, + "_verifySsl": false, + "_downstreamLimits": { + "audiences": 2, + "pathways": 2 + }, + "audienceAccounts": [] +} \ No newline at end of file diff --git a/xdfx-runner-dry-summary.md b/xdfx-runner-dry-summary.md new file mode 100644 index 00000000..2ac9168f --- /dev/null +++ b/xdfx-runner-dry-summary.md @@ -0,0 +1,18 @@ +# xdfx 分析入库执行摘要 + +- 更新时间：2026-06-29T03:43:58.021216+00:00 +- taskId：1 +- actionId：2 +- 受众入库：0 +- 传播节点入库：0 +- 下游受众选择：0 +- 下游传播节点选择：0 +- 受众脆弱点入库：0 +- 传播节点脆弱点入库：0 +- 受众策略入库：0 +- 传播节点策略入库：0 +- 错误数：0 + +## 受众 + +## 传播节点