Merge branch 'dhk-1' into 'master'
代码修改 See merge request knowledge-base-bj/deerflow-server!167
This commit is contained in:
commit
ba0c32dcee
23 files changed
+1415
-115
No files matched your search
@@ -414,6 +414,8 @@ function AIWritingBody({
|
|||||||
const routeBase = useAIWritingRouteBase();
|
const routeBase = useAIWritingRouteBase();
|
||||||
const agentRoute = useAIWritingAgentRouteState(routeBase);
|
const agentRoute = useAIWritingAgentRouteState(routeBase);
|
||||||
const isAdmin = getStoredAuth()?.system_role === 'admin';
|
const isAdmin = getStoredAuth()?.system_role === 'admin';
|
||||||
|
const externalNotebookMode =
|
||||||
|
studioToken != null || initialNotebookId != null || initialDocumentIds != null;
|
||||||
|
|
||||||
// 需求 0616 #5:左栏点「笔记本 / 所有空间」→ 在右侧内嵌打开对应页,替换右侧内容
|
// 需求 0616 #5:左栏点「笔记本 / 所有空间」→ 在右侧内嵌打开对应页,替换右侧内容
|
||||||
// (写作布局不卸载,仅 display:none 隐藏,避免 WritingSetupChat 重挂丢线程)。
|
// (写作布局不卸载,仅 display:none 隐藏,避免 WritingSetupChat 重挂丢线程)。
|
||||||
@@ -432,12 +434,12 @@ function AIWritingBody({
|
|||||||
}
|
}
|
||||||
}, [agentRoute.active]);
|
}, [agentRoute.active]);
|
||||||
|
|
||||||
const agentsActive = sidebarTab === 'agents';
|
const agentsActive = !externalNotebookMode && sidebarTab === 'agents';
|
||||||
|
|
||||||
// 研究模板侧栏在「写作配置(idle)」与「历史回看」两态都展示(点历史记录后保留侧栏、
|
// 研究模板侧栏在「写作配置(idle)」与「历史回看」两态都展示(点历史记录后保留侧栏、
|
||||||
// 历史内容显示到右侧区域);写作进行中收起;嵌入态(studioToken)始终不展示。
|
// 历史内容显示到右侧区域);写作进行中收起;嵌入态(studioToken)始终不展示。
|
||||||
const showSidebar =
|
const showSidebar =
|
||||||
!studioToken && (state.status === 'idle' || isHistoryMode || agentsActive);
|
!externalNotebookMode && (state.status === 'idle' || isHistoryMode || agentsActive);
|
||||||
const compactAppearanceActions = compactChrome ? (
|
const compactAppearanceActions = compactChrome ? (
|
||||||
<AIWritingCompactAppearanceActions />
|
<AIWritingCompactAppearanceActions />
|
||||||
) : null;
|
) : null;
|
||||||
|
|||||||
@@ -273,7 +273,7 @@ The AI-writing graph (`packages/harness/deerflow/agents/ai_writing/`) has four s
|
|||||||
|
|
||||||
Seeding mirrors the roundtable pattern: verified `config.yaml` + `SOUL.md` ship in `app/gateway/routers/_ai_writing_seed_assets/<id>/`; `_ai_writing_seed.py::ensure_ai_writing_functional_agents()` copies them to `.deer-flow/agents/` when missing (never overwrites local edits), called from the startup lifespan (before `_sync_legacy_agents`, so the agents appear in `/api/agents` on first boot) and self-heals from `GET /api/ai-writing/sessions` + `GET /api/ai-writing/article-types`. Tests: `tests/test_ai_writing_seed.py`, `tests/test_ai_writing_agent_runner.py`.
|
Seeding mirrors the roundtable pattern: verified `config.yaml` + `SOUL.md` ship in `app/gateway/routers/_ai_writing_seed_assets/<id>/`; `_ai_writing_seed.py::ensure_ai_writing_functional_agents()` copies them to `.deer-flow/agents/` when missing (never overwrites local edits), called from the startup lifespan (before `_sync_legacy_agents`, so the agents appear in `/api/agents` on first boot) and self-heals from `GET /api/ai-writing/sessions` + `GET /api/ai-writing/article-types`. Tests: `tests/test_ai_writing_seed.py`, `tests/test_ai_writing_agent_runner.py`.
|
||||||
|
|
||||||
**Skill-driven retrieval (检索类型).** `material_source` has three values: `general` (通用检索, default), `knowledge_base` (知识库检索), and `notebook` (我的空间); legacy `skill` is normalized to `general` by the researcher node. **`knowledge_base` 直连内网 ES 接口(= 旧版「内网直连检索」)** — it bypasses skill orchestration entirely and calls `IntranetSearchProvider` over `ai_writing.intranet_search_url` (concurrent over the generated keywords; `intranet_knowledge_base`/`intranet_search_timeout` apply). **An empty/unconfigured `intranet_search_url` does NOT disable it** — the empty string is passed straight to `IntranetSearchProvider`, which falls back to its built-in default endpoint (`intranet_stub._DEFAULT_ENDPOINT`), exactly like the legacy `get_search_provider(search_provider=intranet)` path. So 内网直连检索 works out of the box without configuring an address (configure `intranet_search_url` only to point at a different endpoint). This is the "弱模型识别不了技能" escape hatch — the old no-skill retrieval path restored as an explicit option (weak models like deepseek-chat often fail to invoke configured skills). Tests: `tests/test_ai_writing_knowledge_base.py`. General retrieval is orchestrated from the **skills configured on the `ai-writing-researcher` agent** (`config.yaml → skills`, editable from the agent management page): the built-in `knowledge-base-search` skill (知识库检索) runs the native keyword fan-out web/intranet search (the old default path), while any other configured skill searches with a `description`-targeted query; skills run in configured order with early-stop at `max_materials`, and an empty/unresolvable skills config falls back to the built-in knowledge search so retrieval always works. The skill itself is seeded from `_ai_writing_seed_assets/skills/knowledge-base-search/SKILL.md` into `skills/public/` (never overwrites), and a pre-existing researcher `config.yaml` lacking the `skills` key is patched in place with the default (explicit `skills: []` is respected). **PAUSE-1 input append flow**: when the user types natural language into the material-confirm input (`re_search`/legacy `enable_skill_search` + `user_query`), the node matches the most relevant configured skill via `SkillPicker(candidates=…)` (skipped when only one skill is configured), searches with the user's words, and **appends** the new materials to the existing package (URL-dedup, original order preserved, no LLM re-rank/truncation, 100-item hard cap). **Skill-failure fault tolerance**: a configured skill that times out / raises / has all its keyword searches fail (vs. legitimately returning 0 results) no longer just silently moves on — the researcher node emits a `skill_search_step` of type `skill_error` (carrying the error `reason`), and if the configured skills collected **no** new materials at all it runs a `fallback` pass over other available skills (`_fallback_candidate_skills`: the built-in `knowledge-base-search` native path first, then other enabled catalog skills not yet tried) instead of erroring out. The frontend `SkillStepsCard` renders `skill_error` (rose) and `fallback` (amber) so the timeline shows "技能报错 → 改用其他技能搜集". Tests: `tests/test_ai_writing_material_source.py`.
|
**Skill-driven retrieval (检索类型).** `material_source` has three values: `general` (通用检索, default), `knowledge_base` (知识库检索), and `notebook` (我的空间); legacy `skill` is normalized to `general` by the researcher node. **`knowledge_base` 直连内网 ES 接口(= 旧版「内网直连检索」)** — it bypasses skill orchestration entirely and calls `IntranetSearchProvider` over `ai_writing.intranet_search_url` (concurrent over the generated keywords; `intranet_knowledge_base`/`intranet_search_timeout` apply). The client uses `trust_env=False` and defaults `intranet_verify_ssl=false`, so internal HTTPS endpoints with self-signed/internal-CA certificates do not fail on certificate verification; set `ai_writing.intranet_verify_ssl: true` only for environments that require strict certificate checks. **An empty/unconfigured `intranet_search_url` does NOT disable it** — the empty string is passed straight to `IntranetSearchProvider`, which falls back to its built-in default endpoint (`intranet_stub._DEFAULT_ENDPOINT`), exactly like the legacy `get_search_provider(search_provider=intranet)` path. So 内网直连检索 works out of the box without configuring an address (configure `intranet_search_url` only to point at a different endpoint). This is the "弱模型识别不了技能" escape hatch — the old no-skill retrieval path restored as an explicit option (weak models like deepseek-chat often fail to invoke configured skills). Tests: `tests/test_ai_writing_knowledge_base.py`. General retrieval is orchestrated from the **skills configured on the `ai-writing-researcher` agent** (`config.yaml → skills`, editable from the agent management page): the built-in `knowledge-base-search` skill (知识库检索) runs the native keyword fan-out web/intranet search (the old default path), while any other configured skill searches with a `description`-targeted query; skills run in configured order with early-stop at `max_materials`, and an empty/unresolvable skills config falls back to the built-in knowledge search so retrieval always works. The skill itself is seeded from `_ai_writing_seed_assets/skills/knowledge-base-search/SKILL.md` into `skills/public/` (never overwrites), and a pre-existing researcher `config.yaml` lacking the `skills` key is patched in place with the default (explicit `skills: []` is respected). **PAUSE-1 input append flow**: when the user types natural language into the material-confirm input (`re_search`/legacy `enable_skill_search` + `user_query`), the node matches the most relevant configured skill via `SkillPicker(candidates=…)` (skipped when only one skill is configured), searches with the user's words, and **appends** the new materials to the existing package (URL-dedup, original order preserved, no LLM re-rank/truncation, 100-item hard cap). **Skill-failure fault tolerance**: a configured skill that times out / raises / has all its keyword searches fail (vs. legitimately returning 0 results) no longer just silently moves on — the researcher node emits a `skill_search_step` of type `skill_error` (carrying the error `reason`), and if the configured skills collected **no** new materials at all it runs a `fallback` pass over other available skills (`_fallback_candidate_skills`: the built-in `knowledge-base-search` native path first, then other enabled catalog skills not yet tried) instead of erroring out. The frontend `SkillStepsCard` renders `skill_error` (rose) and `fallback` (amber) so the timeline shows "技能报错 → 改用其他技能搜集". Tests: `tests/test_ai_writing_material_source.py`.
|
||||||
|
|
||||||
**Skill-call timeout** is hardcoded **300s (5 min)** — `researcher.py` `_SKILL_SEARCH_TIMEOUT_SECONDS` (per-skill wrapper) and `_skill_agent.py` `_REAL_AGENT_TIMEOUT_SECONDS` (whole real-agent skill run). The intranet ES fallback (`IntranetSearchProvider`) gets a longer **600s (10 min)** default (`intranet_stub.py`; overridable via `ai_writing.intranet_search_timeout`) — that interface is the slow one, so our own fallback timeout is generous to avoid being blamed for "检不出".
|
**Skill-call timeout** is hardcoded **300s (5 min)** — `researcher.py` `_SKILL_SEARCH_TIMEOUT_SECONDS` (per-skill wrapper) and `_skill_agent.py` `_REAL_AGENT_TIMEOUT_SECONDS` (whole real-agent skill run). The intranet ES fallback (`IntranetSearchProvider`) gets a longer **600s (10 min)** default (`intranet_stub.py`; overridable via `ai_writing.intranet_search_timeout`) — that interface is the slow one, so our own fallback timeout is generous to avoid being blamed for "检不出".
|
||||||
|
|
||||||
|
|||||||
@@ -135,6 +135,10 @@ FastAPI application providing REST endpoints for frontend integration:
|
|||||||
|
|
||||||
Custom agents are indexed in the `agents` database table. The `id` column is the stable runtime identifier and filesystem directory name (`.deer-flow/agents/{id}/SOUL.md`); `name` is display-only, may be Chinese, and may be duplicated. Rows with `user_id = NULL` are built-in agents, user rows are private unless `published = true`, and users can only update or delete agents they created.
|
Custom agents are indexed in the `agents` database table. The `id` column is the stable runtime identifier and filesystem directory name (`.deer-flow/agents/{id}/SOUL.md`); `name` is display-only, may be Chinese, and may be duplicated. Rows with `user_id = NULL` are built-in agents, user rows are private unless `published = true`, and users can only update or delete agents they created.
|
||||||
|
|
||||||
|
### AI Writing Intranet Retrieval
|
||||||
|
|
||||||
|
AI 写作的「知识库检索」会直连 `ai_writing.intranet_search_url` 内网 ES/知识库接口,并默认跳过代理环境变量和 HTTPS 证书校验,适配自签名或内部 CA 证书的内网服务。需要强校验时可在 `config.yaml` 里设置 `ai_writing.intranet_verify_ssl: true`。
|
||||||
|
|
||||||
### IM Channels
|
### IM Channels
|
||||||
|
|
||||||
The IM bridge supports Feishu, Slack, and Telegram. Slack and Telegram still use the final `runs.wait()` response path, while Feishu now streams through `runs.stream(["messages-tuple", "values"])` and updates a single in-thread card in place.
|
The IM bridge supports Feishu, Slack, and Telegram. Slack and Telegram still use the final `runs.wait()` response path, while Feishu now streams through `runs.stream(["messages-tuple", "values"])` and updates a single in-thread card in place.
|
||||||
|
|||||||
+13
-7
@@ -27,13 +27,15 @@ from types import SimpleNamespace
|
|||||||
|
|
||||||
from langchain_core.runnables import RunnableConfig
|
from langchain_core.runnables import RunnableConfig
|
||||||
|
|
||||||
from ..search import get_search_provider, SearchResult
|
from ..prompts.researcher_prompts import (
|
||||||
|
EXTRACT_KEYWORDS_SYSTEM,
|
||||||
|
EXTRACT_KEYWORDS_USER,
|
||||||
|
RANK_MATERIALS_SYSTEM,
|
||||||
|
RANK_MATERIALS_USER,
|
||||||
|
)
|
||||||
|
from ..search import SearchResult, get_search_provider
|
||||||
from ..search.skill_search import SkillPicker
|
from ..search.skill_search import SkillPicker
|
||||||
from ..state import AIWritingState, Material, MaterialPackage
|
from ..state import AIWritingState, Material, MaterialPackage
|
||||||
from ..prompts.researcher_prompts import (
|
|
||||||
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
|
|
||||||
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
|
|
||||||
)
|
|
||||||
from ._agent_runner import (
|
from ._agent_runner import (
|
||||||
KEYWORDS_READY_MARKER,
|
KEYWORDS_READY_MARKER,
|
||||||
MATERIALS_RANKED_MARKER,
|
MATERIALS_RANKED_MARKER,
|
||||||
@@ -620,6 +622,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
|||||||
# 内网慢检索超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
|
# 内网慢检索超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
|
||||||
# 默认 600s,且最低 300s。
|
# 默认 600s,且最低 300s。
|
||||||
timeout=_intranet_timeout_seconds(ai_writing_config),
|
timeout=_intranet_timeout_seconds(ai_writing_config),
|
||||||
|
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
|
||||||
)
|
)
|
||||||
endpoint_label = intranet_url or getattr(provider, "endpoint", "内置默认内网端点")
|
endpoint_label = intranet_url or getattr(provider, "endpoint", "内置默认内网端点")
|
||||||
intranet_debug = {
|
intranet_debug = {
|
||||||
@@ -628,6 +631,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
|||||||
"endpoint": endpoint_label,
|
"endpoint": endpoint_label,
|
||||||
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
|
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
|
||||||
"timeout": _intranet_timeout_seconds(ai_writing_config),
|
"timeout": _intranet_timeout_seconds(ai_writing_config),
|
||||||
|
"verify_ssl": getattr(provider, "verify_ssl", False),
|
||||||
"status": "running",
|
"status": "running",
|
||||||
"queries": [],
|
"queries": [],
|
||||||
"calls": [],
|
"calls": [],
|
||||||
@@ -780,7 +784,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
|||||||
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
|
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
|
||||||
)
|
)
|
||||||
return (results or []), ""
|
return (results or []), ""
|
||||||
except asyncio.TimeoutError:
|
except TimeoutError:
|
||||||
msg = f"检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)"
|
msg = f"检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)"
|
||||||
logger.warning("skill[%s] search timed out after %ss", skill_name, _SKILL_SEARCH_TIMEOUT_SECONDS)
|
logger.warning("skill[%s] search timed out after %ss", skill_name, _SKILL_SEARCH_TIMEOUT_SECONDS)
|
||||||
search_errors.append(f"skill[{skill_name}]: {msg}")
|
search_errors.append(f"skill[{skill_name}]: {msg}")
|
||||||
@@ -923,13 +927,14 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
|||||||
intranet_url = (ai_writing_config.get("intranet_search_url") or "").strip()
|
intranet_url = (ai_writing_config.get("intranet_search_url") or "").strip()
|
||||||
# 诊断开关:开启后即使技能成功也照样跑完内网并记录完整结果(用于验证内网接口是否真能通)。
|
# 诊断开关:开启后即使技能成功也照样跑完内网并记录完整结果(用于验证内网接口是否真能通)。
|
||||||
intranet_debug_force = bool(ai_writing_config.get("intranet_search_debug", False))
|
intranet_debug_force = bool(ai_writing_config.get("intranet_search_debug", False))
|
||||||
intranet_task: "asyncio.Task | None" = None
|
intranet_task: asyncio.Task | None = None
|
||||||
intranet_holder: dict = {}
|
intranet_holder: dict = {}
|
||||||
intranet_debug = {
|
intranet_debug = {
|
||||||
"configured": bool(intranet_url),
|
"configured": bool(intranet_url),
|
||||||
"endpoint": intranet_url or "(未配置 ai_writing.intranet_search_url)",
|
"endpoint": intranet_url or "(未配置 ai_writing.intranet_search_url)",
|
||||||
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
|
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
|
||||||
"timeout": _intranet_timeout_seconds(ai_writing_config),
|
"timeout": _intranet_timeout_seconds(ai_writing_config),
|
||||||
|
"verify_ssl": bool(ai_writing_config.get("intranet_verify_ssl", False)),
|
||||||
"debug_force": intranet_debug_force,
|
"debug_force": intranet_debug_force,
|
||||||
"status": "pending" if intranet_url else "not_configured",
|
"status": "pending" if intranet_url else "not_configured",
|
||||||
"queries": [], # 每个检索词:{query, count, error?}
|
"queries": [], # 每个检索词:{query, count, error?}
|
||||||
@@ -952,6 +957,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
|||||||
# 兜底接口超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
|
# 兜底接口超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
|
||||||
# 默认 600s(10 分钟),且最低 300s。
|
# 默认 600s(10 分钟),且最低 300s。
|
||||||
timeout=_intranet_timeout_seconds(ai_writing_config),
|
timeout=_intranet_timeout_seconds(ai_writing_config),
|
||||||
|
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
|
||||||
)
|
)
|
||||||
intranet_holder["provider"] = intranet
|
intranet_holder["provider"] = intranet
|
||||||
fb_keywords = keywords[:keyword_count] or [intent]
|
fb_keywords = keywords[:keyword_count] or [intent]
|
||||||
|
|||||||
+4
-2
@@ -1,8 +1,8 @@
|
|||||||
from .base import BaseSearchProvider, SearchResult
|
from .base import BaseSearchProvider, SearchResult
|
||||||
from .internet_search import InternetSearchProvider
|
|
||||||
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout
|
|
||||||
from .demo_search import DemoSearchProvider
|
from .demo_search import DemoSearchProvider
|
||||||
from .duckduckgo_search import DuckDuckGoSearchProvider
|
from .duckduckgo_search import DuckDuckGoSearchProvider
|
||||||
|
from .internet_search import InternetSearchProvider
|
||||||
|
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout, normalize_intranet_verify_ssl
|
||||||
from .skill_search import SkillSearchProvider
|
from .skill_search import SkillSearchProvider
|
||||||
|
|
||||||
|
|
||||||
@@ -14,6 +14,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
|||||||
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
|
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
|
||||||
intranet_knowledge_base: samples
|
intranet_knowledge_base: samples
|
||||||
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
|
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
|
||||||
|
intranet_verify_ssl: false # 内网 HTTPS 自签名证书场景默认关闭证书校验
|
||||||
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
|
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
|
||||||
|
|
||||||
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
|
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
|
||||||
@@ -26,6 +27,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
|||||||
base_url=cfg.get("intranet_search_url", ""),
|
base_url=cfg.get("intranet_search_url", ""),
|
||||||
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
|
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
|
||||||
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
|
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
|
||||||
|
verify_ssl=normalize_intranet_verify_ssl(cfg.get("intranet_verify_ssl")),
|
||||||
)
|
)
|
||||||
if provider == "tavily":
|
if provider == "tavily":
|
||||||
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
|
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
|
||||||
|
|||||||
+29
-7
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from typing import List
|
|
||||||
|
|
||||||
from .base import BaseSearchProvider, SearchResult, simplify_query
|
from .base import BaseSearchProvider, SearchResult, simplify_query
|
||||||
|
|
||||||
@@ -15,6 +14,8 @@ _DEFAULT_KB = "samples"
|
|||||||
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
|
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
|
||||||
_MIN_TIMEOUT_SECONDS = 300
|
_MIN_TIMEOUT_SECONDS = 300
|
||||||
_DEFAULT_TIMEOUT_SECONDS = 600
|
_DEFAULT_TIMEOUT_SECONDS = 600
|
||||||
|
# 内网 HTTPS 检索服务常使用自签名/内部 CA 证书,默认跳过证书校验避免 SSL 证书链错误。
|
||||||
|
_DEFAULT_VERIFY_SSL = False
|
||||||
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
|
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
|
||||||
_RAW_SNIPPET_LIMIT = 2000
|
_RAW_SNIPPET_LIMIT = 2000
|
||||||
|
|
||||||
@@ -28,6 +29,23 @@ def normalize_intranet_timeout(timeout: object, default: int = _DEFAULT_TIMEOUT_
|
|||||||
return max(value, _MIN_TIMEOUT_SECONDS)
|
return max(value, _MIN_TIMEOUT_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_intranet_verify_ssl(value: object, default: bool = _DEFAULT_VERIFY_SSL) -> bool:
|
||||||
|
"""返回内网检索是否校验 SSL 证书,兼容 config.yaml 里的 bool / 字符串写法。"""
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return value
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, (int, float)):
|
||||||
|
return bool(value)
|
||||||
|
if isinstance(value, str):
|
||||||
|
normalized = value.strip().lower()
|
||||||
|
if normalized in {"1", "true", "yes", "on"}:
|
||||||
|
return True
|
||||||
|
if normalized in {"0", "false", "no", "off"}:
|
||||||
|
return False
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
def _pick(d: dict, keys: tuple) -> str:
|
def _pick(d: dict, keys: tuple) -> str:
|
||||||
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
|
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
|
||||||
for k in keys:
|
for k in keys:
|
||||||
@@ -62,12 +80,14 @@ class IntranetSearchProvider(BaseSearchProvider):
|
|||||||
base_url: str = "",
|
base_url: str = "",
|
||||||
knowledge_base_name: str = _DEFAULT_KB,
|
knowledge_base_name: str = _DEFAULT_KB,
|
||||||
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
|
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
|
||||||
|
verify_ssl: bool = _DEFAULT_VERIFY_SSL,
|
||||||
**_ignored,
|
**_ignored,
|
||||||
):
|
):
|
||||||
self.endpoint = base_url or _DEFAULT_ENDPOINT
|
self.endpoint = base_url or _DEFAULT_ENDPOINT
|
||||||
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
|
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
|
||||||
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
|
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
|
||||||
self.timeout = normalize_intranet_timeout(timeout)
|
self.timeout = normalize_intranet_timeout(timeout)
|
||||||
|
self.verify_ssl = normalize_intranet_verify_ssl(verify_ssl)
|
||||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
|
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
|
||||||
self.errors: list[str] = []
|
self.errors: list[str] = []
|
||||||
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
|
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
|
||||||
@@ -82,7 +102,7 @@ class IntranetSearchProvider(BaseSearchProvider):
|
|||||||
if snippet:
|
if snippet:
|
||||||
self.raw_responses.append(snippet)
|
self.raw_responses.append(snippet)
|
||||||
|
|
||||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
async def search(self, query: str, max_results: int = 5, **kwargs) -> list[SearchResult]:
|
||||||
results = await self._search_once(query, max_results)
|
results = await self._search_once(query, max_results)
|
||||||
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
|
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
|
||||||
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
|
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
|
||||||
@@ -93,7 +113,7 @@ class IntranetSearchProvider(BaseSearchProvider):
|
|||||||
results = await self._search_once(simpler, max_results)
|
results = await self._search_once(simpler, max_results)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
async def _search_once(self, query: str, max_results: int = 5) -> List[SearchResult]:
|
async def _search_once(self, query: str, max_results: int = 5) -> list[SearchResult]:
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
@@ -112,12 +132,13 @@ class IntranetSearchProvider(BaseSearchProvider):
|
|||||||
"status": "pending",
|
"status": "pending",
|
||||||
"ok": False,
|
"ok": False,
|
||||||
"count": 0,
|
"count": 0,
|
||||||
|
"verify_ssl": self.verify_ssl,
|
||||||
}
|
}
|
||||||
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
|
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
|
||||||
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
|
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
|
||||||
logger.info(
|
logger.info(
|
||||||
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss query=%r",
|
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss verify_ssl=%s query=%r",
|
||||||
self.endpoint, self.knowledge_base_name, self.timeout, query,
|
self.endpoint, self.knowledge_base_name, self.timeout, self.verify_ssl, query,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
|
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
|
||||||
@@ -125,7 +146,8 @@ class IntranetSearchProvider(BaseSearchProvider):
|
|||||||
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
|
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
|
||||||
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
|
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
|
||||||
# 调用同款修法)。
|
# 调用同款修法)。
|
||||||
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False) as client:
|
# verify=False:内网 HTTPS 常见自签名/内部 CA 证书;这里仅跳过本内网检索端点的证书校验。
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False, verify=self.verify_ssl) as client:
|
||||||
resp = await client.post(self.endpoint, json=payload)
|
resp = await client.post(self.endpoint, json=payload)
|
||||||
call["http_status"] = getattr(resp, "status_code", None)
|
call["http_status"] = getattr(resp, "status_code", None)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
@@ -210,7 +232,7 @@ class IntranetSearchProvider(BaseSearchProvider):
|
|||||||
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
|
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
|
||||||
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
|
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
|
||||||
)
|
)
|
||||||
results: List[SearchResult] = []
|
results: list[SearchResult] = []
|
||||||
for i, item in enumerate(items[:max_results]):
|
for i, item in enumerate(items[:max_results]):
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
continue
|
continue
|
||||||
|
|||||||
+60
-9
@@ -6,7 +6,7 @@ from datetime import UTC, datetime
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import and_, case, delete, func, or_, select
|
from sqlalchemy import and_, case, delete, func, literal, not_, or_, select
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
@@ -19,6 +19,7 @@ from deerflow.persistence.agents.model import (
|
|||||||
AgentRow,
|
AgentRow,
|
||||||
AgentSkillRow,
|
AgentSkillRow,
|
||||||
)
|
)
|
||||||
|
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||||
from deerflow.persistence.tags.model import TagAssignmentRow
|
from deerflow.persistence.tags.model import TagAssignmentRow
|
||||||
|
|
||||||
|
|
||||||
@@ -36,6 +37,30 @@ class AgentRepository(AgentStore):
|
|||||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||||
self._sf = session_factory
|
self._sf = session_factory
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _business_chain_agent_exists(agent_id_col):
|
||||||
|
return (
|
||||||
|
select(RoundtableChainRow.id)
|
||||||
|
.where(
|
||||||
|
or_(
|
||||||
|
RoundtableChainRow.seats.like(literal('%"agent_id": "') + agent_id_col + literal('"%')),
|
||||||
|
RoundtableChainRow.seats.like(literal('%"agent_id":"') + agent_id_col + literal('"%')),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _valid_favorite_filter(cls):
|
||||||
|
return or_(
|
||||||
|
AgentFavoriteRow.origin == "user",
|
||||||
|
and_(
|
||||||
|
AgentFavoriteRow.origin == "position",
|
||||||
|
AgentRow.user_id.is_not(None),
|
||||||
|
not_(cls._business_chain_agent_exists(AgentFavoriteRow.agent_id)),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _visible_filter(user_id: str):
|
def _visible_filter(user_id: str):
|
||||||
"""Visibility for single-record fetches (``get_visible``).
|
"""Visibility for single-record fetches (``get_visible``).
|
||||||
@@ -53,13 +78,21 @@ class AgentRepository(AgentStore):
|
|||||||
- favorited — the "我的" listing keeps favorited agents even after
|
- favorited — the "我的" listing keeps favorited agents even after
|
||||||
the owner unpublishes them, so reads keep honoring the favorite.
|
the owner unpublishes them, so reads keep honoring the favorite.
|
||||||
"""
|
"""
|
||||||
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
|
valid_favorite_exists = (
|
||||||
|
select(AgentFavoriteRow.id)
|
||||||
|
.where(
|
||||||
|
AgentFavoriteRow.user_id == user_id,
|
||||||
|
AgentFavoriteRow.agent_id == AgentRow.id,
|
||||||
|
AgentRepository._valid_favorite_filter(),
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
return or_(
|
return or_(
|
||||||
AgentRow.user_id.is_(None),
|
AgentRow.user_id.is_(None),
|
||||||
AgentRow.user_id == user_id,
|
AgentRow.user_id == user_id,
|
||||||
AgentRow.published.is_(True),
|
AgentRow.published.is_(True),
|
||||||
AgentRow.is_private.is_(True),
|
AgentRow.is_private.is_(True),
|
||||||
AgentRow.id.in_(fav_subq),
|
valid_favorite_exists,
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -269,10 +302,19 @@ class AgentRepository(AgentStore):
|
|||||||
AgentRow.is_private.is_(False),
|
AgentRow.is_private.is_(False),
|
||||||
)
|
)
|
||||||
if scope == "mine":
|
if scope == "mine":
|
||||||
# owned OR favorited. A correlated IN-subquery (not a JOIN) keeps a
|
# owned OR valid favorites. Built-in agents and business-chain
|
||||||
# single row per agent even when it is both owned and favorited.
|
# seats only enter "mine" when explicitly favorited by the user;
|
||||||
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
|
# the original position-granted behavior stays for ordinary agents.
|
||||||
return or_(AgentRow.user_id == user_id, AgentRow.id.in_(fav_subq))
|
valid_favorite_exists = (
|
||||||
|
select(AgentFavoriteRow.id)
|
||||||
|
.where(
|
||||||
|
AgentFavoriteRow.user_id == user_id,
|
||||||
|
AgentFavoriteRow.agent_id == AgentRow.id,
|
||||||
|
self._valid_favorite_filter(),
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
return or_(AgentRow.user_id == user_id, valid_favorite_exists)
|
||||||
if scope == "private":
|
if scope == "private":
|
||||||
return AgentRow.is_private.is_(True)
|
return AgentRow.is_private.is_(True)
|
||||||
if scope == "builtin":
|
if scope == "builtin":
|
||||||
@@ -441,12 +483,16 @@ class AgentRepository(AgentStore):
|
|||||||
async with self._sf() as session:
|
async with self._sf() as session:
|
||||||
existing = (
|
existing = (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(AgentFavoriteRow.id)
|
select(AgentFavoriteRow)
|
||||||
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
|
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
|
||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
|
if existing.origin != "user" or existing.position_id is not None:
|
||||||
|
existing.origin = "user"
|
||||||
|
existing.position_id = None
|
||||||
|
await session.commit()
|
||||||
return
|
return
|
||||||
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
|
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
|
||||||
try:
|
try:
|
||||||
@@ -470,7 +516,12 @@ class AgentRepository(AgentStore):
|
|||||||
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
async def list_favorite_ids(self, user_id: str) -> set[str]:
|
||||||
async with self._sf() as session:
|
async with self._sf() as session:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
|
select(AgentFavoriteRow.agent_id)
|
||||||
|
.join(AgentRow, AgentRow.id == AgentFavoriteRow.agent_id)
|
||||||
|
.where(
|
||||||
|
AgentFavoriteRow.user_id == user_id,
|
||||||
|
self._valid_favorite_filter(),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
return {row[0] for row in result.all()}
|
return {row[0] for row in result.all()}
|
||||||
|
|
||||||
|
|||||||
+90
@@ -0,0 +1,90 @@
|
|||||||
|
"""Stop position-granted favorites from collecting built-ins/chain seats.
|
||||||
|
|
||||||
|
Built-in agents and agents used as business-chain seats should only enter
|
||||||
|
"mine" when a user explicitly clicks the favorite action. This migration
|
||||||
|
removes historical automatic ``origin='position'`` rows for those agents while
|
||||||
|
preserving user-created favorites (``origin='user'``) and ordinary
|
||||||
|
position-granted agent favorites.
|
||||||
|
|
||||||
|
Revision ID: 20260701_01
|
||||||
|
Revises: 20260625_04
|
||||||
|
Create Date: 2026-07-01
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import inspect
|
||||||
|
|
||||||
|
revision = "20260701_01"
|
||||||
|
down_revision = "20260625_04"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def _has_table(table_name: str) -> bool:
|
||||||
|
inspector = inspect(op.get_bind())
|
||||||
|
return table_name in inspector.get_table_names()
|
||||||
|
|
||||||
|
|
||||||
|
def _has_column(table_name: str, column_name: str) -> bool:
|
||||||
|
if not _has_table(table_name):
|
||||||
|
return False
|
||||||
|
inspector = inspect(op.get_bind())
|
||||||
|
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
if not (
|
||||||
|
_has_table("agent_favorites")
|
||||||
|
and _has_column("agent_favorites", "origin")
|
||||||
|
and _has_column("agent_favorites", "agent_id")
|
||||||
|
):
|
||||||
|
return
|
||||||
|
|
||||||
|
favorites = sa.table(
|
||||||
|
"agent_favorites",
|
||||||
|
sa.column("agent_id", sa.String(64)),
|
||||||
|
sa.column("origin", sa.String(16)),
|
||||||
|
)
|
||||||
|
cleanup_conditions = []
|
||||||
|
if _has_table("agents") and _has_column("agents", "user_id"):
|
||||||
|
agents = sa.table(
|
||||||
|
"agents",
|
||||||
|
sa.column("id", sa.String(64)),
|
||||||
|
sa.column("user_id", sa.String(64)),
|
||||||
|
)
|
||||||
|
cleanup_conditions.append(
|
||||||
|
favorites.c.agent_id.in_(sa.select(agents.c.id).where(agents.c.user_id.is_(None)))
|
||||||
|
)
|
||||||
|
if _has_table("roundtable_chains") and _has_column("roundtable_chains", "seats"):
|
||||||
|
chains = sa.table(
|
||||||
|
"roundtable_chains",
|
||||||
|
sa.column("seats", sa.Text()),
|
||||||
|
)
|
||||||
|
cleanup_conditions.append(
|
||||||
|
sa.select(sa.literal(1))
|
||||||
|
.select_from(chains)
|
||||||
|
.where(
|
||||||
|
sa.or_(
|
||||||
|
chains.c.seats.like(sa.literal('%"agent_id": "') + favorites.c.agent_id + sa.literal('"%')),
|
||||||
|
chains.c.seats.like(sa.literal('%"agent_id":"') + favorites.c.agent_id + sa.literal('"%')),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
if not cleanup_conditions:
|
||||||
|
return
|
||||||
|
|
||||||
|
op.get_bind().execute(
|
||||||
|
favorites.delete().where(
|
||||||
|
favorites.c.origin == "position",
|
||||||
|
sa.or_(*cleanup_conditions),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Data cleanup only; removed auto-grants are not reconstructable.
|
||||||
|
pass
|
||||||
+49
-7
@@ -12,7 +12,8 @@ Resolution rule (per resource type):
|
|||||||
* For a resource type **with** bound tags → the union of resources carrying any
|
* For a resource type **with** bound tags → the union of resources carrying any
|
||||||
of those tags.
|
of those tags.
|
||||||
* For a resource type **without** bound tags → the *default set*:
|
* For a resource type **without** bound tags → the *default set*:
|
||||||
- ``agent`` → all built-in agents (``user_id IS NULL``)
|
- ``agent`` -> built-ins first, then filtered out of
|
||||||
|
position-granted favorites
|
||||||
- ``scheduled_task`` → empty (no built-in tasks)
|
- ``scheduled_task`` → empty (no built-in tasks)
|
||||||
- ``skill`` → empty (the 广场 still shows every visible skill;
|
- ``skill`` → empty (the 广场 still shows every visible skill;
|
||||||
a position only *adds* its tagged skills to the member's "我的技能")
|
a position only *adds* its tagged skills to the member's "我的技能")
|
||||||
@@ -24,9 +25,12 @@ Resolution rule (per resource type):
|
|||||||
|
|
||||||
Materialization (``sync_user``) only ever touches rows it created itself
|
Materialization (``sync_user``) only ever touches rows it created itself
|
||||||
(``origin='position'``), so a user's own favorites / subscriptions are never
|
(``origin='position'``), so a user's own favorites / subscriptions are never
|
||||||
clobbered. ``skill`` is materialized into ``skill_favorites`` exactly like
|
clobbered. ``agent`` keeps the original position-granted behavior for ordinary
|
||||||
``agent`` → ``agent_favorites``: the tagged skills are *added* to the user's
|
tagged agents, but filters out built-ins and business-chain seats; those only
|
||||||
"我的技能" while the browse-all 广场 stays complete (it is **not** restricted).
|
enter "mine" after an explicit user favorite. ``skill`` is still materialized
|
||||||
|
into ``skill_favorites`` so tagged skills are added to the user's "mine"
|
||||||
|
collection while the browse-all square stays complete (it is **not**
|
||||||
|
restricted).
|
||||||
Only recommended-question visibility is computed live by :meth:`resolve_visible`
|
Only recommended-question visibility is computed live by :meth:`resolve_visible`
|
||||||
and is *not* materialized. (The admin-configured prompts shown to users ARE the
|
and is *not* materialized. (The admin-configured prompts shown to users ARE the
|
||||||
recommended questions — there is no separate prompt-template store.)
|
recommended questions — there is no separate prompt-template store.)
|
||||||
@@ -34,6 +38,7 @@ recommended questions — there is no separate prompt-template store.)
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
@@ -43,13 +48,15 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|||||||
|
|
||||||
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
|
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
|
||||||
from deerflow.persistence.positions.base import PositionStore
|
from deerflow.persistence.positions.base import PositionStore
|
||||||
|
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||||
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskSubscriptionRow
|
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskSubscriptionRow
|
||||||
from deerflow.persistence.skills.model import SkillFavoriteRow
|
from deerflow.persistence.skills.model import SkillFavoriteRow
|
||||||
from deerflow.persistence.tags.base import TagStore
|
from deerflow.persistence.tags.base import TagStore
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Materialized into "我的" tables by sync_user.
|
# Reconciled by sync_user. ``agent`` preserves ordinary position-granted rows,
|
||||||
|
# but filters built-ins and business-chain seats.
|
||||||
_MATERIALIZED_TYPES = ("agent", "skill", "scheduled_task")
|
_MATERIALIZED_TYPES = ("agent", "skill", "scheduled_task")
|
||||||
# Resolved live at query time (visibility filters), never materialized.
|
# Resolved live at query time (visibility filters), never materialized.
|
||||||
_VISIBILITY_TYPES = ("recommended_question",)
|
_VISIBILITY_TYPES = ("recommended_question",)
|
||||||
@@ -93,8 +100,12 @@ class PositionSyncService:
|
|||||||
bindings = await self._positions.get_tag_bindings(pid) if pid else {}
|
bindings = await self._positions.get_tag_bindings(pid) if pid else {}
|
||||||
tag_ids = bindings.get(resource_type) or []
|
tag_ids = bindings.get(resource_type) or []
|
||||||
if tag_ids:
|
if tag_ids:
|
||||||
return set(await self._tags.list_targets_for_tags(resource_type, tag_ids))
|
target = set(await self._tags.list_targets_for_tags(resource_type, tag_ids))
|
||||||
return await self._default_set(resource_type, session)
|
else:
|
||||||
|
target = await self._default_set(resource_type, session)
|
||||||
|
if resource_type == "agent":
|
||||||
|
return await self._filter_materializable_agents(target, session)
|
||||||
|
return target
|
||||||
|
|
||||||
async def _default_set(self, resource_type: str, session: AsyncSession) -> set[str]:
|
async def _default_set(self, resource_type: str, session: AsyncSession) -> set[str]:
|
||||||
if resource_type == "agent":
|
if resource_type == "agent":
|
||||||
@@ -103,6 +114,37 @@ class PositionSyncService:
|
|||||||
# scheduled_task (and any other materialized type) defaults to empty.
|
# scheduled_task (and any other materialized type) defaults to empty.
|
||||||
return set()
|
return set()
|
||||||
|
|
||||||
|
async def _filter_materializable_agents(self, agent_ids: set[str], session: AsyncSession) -> set[str]:
|
||||||
|
if not agent_ids:
|
||||||
|
return set()
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(AgentRow.id).where(
|
||||||
|
AgentRow.id.in_(agent_ids),
|
||||||
|
AgentRow.user_id.is_not(None),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
candidate_ids = {r[0] for r in rows}
|
||||||
|
return candidate_ids - await self._business_chain_agent_ids(session)
|
||||||
|
|
||||||
|
async def _business_chain_agent_ids(self, session: AsyncSession) -> set[str]:
|
||||||
|
rows = (await session.execute(select(RoundtableChainRow.seats))).scalars()
|
||||||
|
agent_ids: set[str] = set()
|
||||||
|
for raw in rows:
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
seats = json.loads(raw) if isinstance(raw, str) else raw
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if not isinstance(seats, list):
|
||||||
|
continue
|
||||||
|
for seat in seats:
|
||||||
|
if isinstance(seat, dict) and isinstance(seat.get("agent_id"), str):
|
||||||
|
agent_ids.add(seat["agent_id"])
|
||||||
|
return agent_ids
|
||||||
|
|
||||||
# --------------------------------------------------------- materialize
|
# --------------------------------------------------------- materialize
|
||||||
|
|
||||||
async def sync_user(self, user_id: str) -> None:
|
async def sync_user(self, user_id: str) -> None:
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
import yaml
|
import yaml
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
from app.gateway.routers.agents import (
|
from app.gateway.routers.agents import (
|
||||||
_backfill_listing_cache,
|
_backfill_listing_cache,
|
||||||
@@ -11,7 +17,13 @@ from app.gateway.routers.agents import (
|
|||||||
from app.gateway.services import build_run_config
|
from app.gateway.services import build_run_config
|
||||||
from deerflow.config import paths as paths_module
|
from deerflow.config import paths as paths_module
|
||||||
from deerflow.config.agents_config import AgentConfig
|
from deerflow.config.agents_config import AgentConfig
|
||||||
from deerflow.persistence.agents import MemoryAgentStore
|
from deerflow.persistence.agents import AgentRepository, MemoryAgentStore
|
||||||
|
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
|
||||||
|
from deerflow.persistence.base import Base
|
||||||
|
from deerflow.persistence.positions.memory import MemoryPositionStore
|
||||||
|
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||||
|
from deerflow.persistence.tags.memory import MemoryTagStore
|
||||||
|
from deerflow.services.position_sync import PositionSyncService
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -23,6 +35,92 @@ def isolated_paths(tmp_path, monkeypatch):
|
|||||||
monkeypatch.setattr(paths_module, "_paths", None)
|
monkeypatch.setattr(paths_module, "_paths", None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def sql_session_factory(tmp_path):
|
||||||
|
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'agent-tests.db'}")
|
||||||
|
async with engine.begin() as conn:
|
||||||
|
# Importing PositionSyncService above registers the agent / skill /
|
||||||
|
# scheduled-task rows it materializes into.
|
||||||
|
await conn.run_sync(Base.metadata.create_all)
|
||||||
|
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
try:
|
||||||
|
yield sf
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
async def _add_sql_agent(sf, *, agent_id: str, user_id: str | None) -> None:
|
||||||
|
async with sf() as session:
|
||||||
|
session.add(
|
||||||
|
AgentRow(
|
||||||
|
id=agent_id,
|
||||||
|
name=agent_id,
|
||||||
|
description="",
|
||||||
|
user_id=user_id,
|
||||||
|
published=True,
|
||||||
|
is_private=False,
|
||||||
|
square_id="",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def _add_sql_favorite(sf, *, user_id: str, agent_id: str, origin: str) -> None:
|
||||||
|
async with sf() as session:
|
||||||
|
session.add(
|
||||||
|
AgentFavoriteRow(
|
||||||
|
id=uuid4().hex,
|
||||||
|
user_id=user_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
origin=origin,
|
||||||
|
position_id="default" if origin == "position" else None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def _add_sql_chain(sf, *, chain_id: str, agent_ids: list[str]) -> None:
|
||||||
|
async with sf() as session:
|
||||||
|
session.add(
|
||||||
|
RoundtableChainRow(
|
||||||
|
id=chain_id,
|
||||||
|
user_id="chain-owner",
|
||||||
|
title=chain_id,
|
||||||
|
description="",
|
||||||
|
seats=json.dumps(
|
||||||
|
[{"agent_id": agent_id, "name": agent_id, "description": ""} for agent_id in agent_ids],
|
||||||
|
ensure_ascii=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def _sql_favorite_rows(sf, user_id: str) -> set[tuple[str, str]]:
|
||||||
|
async with sf() as session:
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(AgentFavoriteRow.agent_id, AgentFavoriteRow.origin).where(
|
||||||
|
AgentFavoriteRow.user_id == user_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return {(agent_id, origin) for agent_id, origin in rows}
|
||||||
|
|
||||||
|
|
||||||
|
async def _default_positions() -> MemoryPositionStore:
|
||||||
|
positions = MemoryPositionStore()
|
||||||
|
await positions.create_position(
|
||||||
|
{
|
||||||
|
"id": "default",
|
||||||
|
"name": "Default",
|
||||||
|
"description": "",
|
||||||
|
"is_default": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return positions
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_agent_store_allows_duplicate_chinese_names_and_filters_visibility() -> None:
|
async def test_agent_store_allows_duplicate_chinese_names_and_filters_visibility() -> None:
|
||||||
store = MemoryAgentStore()
|
store = MemoryAgentStore()
|
||||||
@@ -39,6 +137,136 @@ async def test_agent_store_allows_duplicate_chinese_names_and_filters_visibility
|
|||||||
assert user_c_ids == {"builtin-agent", "published-b"}
|
assert user_c_ids == {"builtin-agent", "published-b"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_position_sync_default_does_not_collect_builtin_agents(sql_session_factory) -> None:
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="builtin-auto", user_id=None)
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="builtin-manual", user_id=None)
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="builtin-auto",
|
||||||
|
origin="position",
|
||||||
|
)
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="builtin-manual",
|
||||||
|
origin="user",
|
||||||
|
)
|
||||||
|
|
||||||
|
sync = PositionSyncService(
|
||||||
|
sql_session_factory,
|
||||||
|
await _default_positions(),
|
||||||
|
MemoryTagStore(),
|
||||||
|
)
|
||||||
|
await sync.sync_user("user-a")
|
||||||
|
|
||||||
|
assert await _sql_favorite_rows(sql_session_factory, "user-a") == {
|
||||||
|
("builtin-manual", "user")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_position_sync_tagged_agents_are_not_auto_favorited(sql_session_factory) -> None:
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="builtin-tagged", user_id=None)
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="custom-tagged", user_id="owner")
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="chain-tagged", user_id="owner")
|
||||||
|
await _add_sql_chain(sql_session_factory, chain_id="chain-1", agent_ids=["chain-tagged"])
|
||||||
|
|
||||||
|
positions = await _default_positions()
|
||||||
|
tags = MemoryTagStore()
|
||||||
|
await tags.create_tag({"id": "agent-tag", "name": "Agent tag", "tag_type": "agent"})
|
||||||
|
await tags.set_target_tags("agent", "builtin-tagged", ["agent-tag"])
|
||||||
|
await tags.set_target_tags("agent", "custom-tagged", ["agent-tag"])
|
||||||
|
await tags.set_target_tags("agent", "chain-tagged", ["agent-tag"])
|
||||||
|
await positions.set_tag_bindings("default", {"agent": ["agent-tag"]})
|
||||||
|
|
||||||
|
sync = PositionSyncService(sql_session_factory, positions, tags)
|
||||||
|
await sync.sync_user("user-a")
|
||||||
|
|
||||||
|
assert await _sql_favorite_rows(sql_session_factory, "user-a") == {
|
||||||
|
("custom-tagged", "position")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_mine_scope_ignores_auto_agent_favorites_but_keeps_manual_builtin(sql_session_factory) -> None:
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="builtin-auto", user_id=None)
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="builtin-manual", user_id=None)
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="custom-auto", user_id="owner")
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="chain-auto", user_id="owner")
|
||||||
|
await _add_sql_chain(sql_session_factory, chain_id="chain-1", agent_ids=["chain-auto"])
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="builtin-auto",
|
||||||
|
origin="position",
|
||||||
|
)
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="builtin-manual",
|
||||||
|
origin="user",
|
||||||
|
)
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="custom-auto",
|
||||||
|
origin="position",
|
||||||
|
)
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="chain-auto",
|
||||||
|
origin="position",
|
||||||
|
)
|
||||||
|
|
||||||
|
store = AgentRepository(sql_session_factory)
|
||||||
|
rows, total = await store.list_paginated(
|
||||||
|
user_id="user-a",
|
||||||
|
scope="mine",
|
||||||
|
search=None,
|
||||||
|
tag_ids=None,
|
||||||
|
page=1,
|
||||||
|
page_size=20,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert total == 2
|
||||||
|
assert {row["id"] for row in rows} == {"builtin-manual", "custom-auto"}
|
||||||
|
assert await store.list_favorite_ids("user-a") == {"builtin-manual", "custom-auto"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_manual_favorite_upgrades_existing_auto_builtin_favorite(sql_session_factory) -> None:
|
||||||
|
await _add_sql_agent(sql_session_factory, agent_id="builtin-agent", user_id=None)
|
||||||
|
await _add_sql_favorite(
|
||||||
|
sql_session_factory,
|
||||||
|
user_id="user-a",
|
||||||
|
agent_id="builtin-agent",
|
||||||
|
origin="position",
|
||||||
|
)
|
||||||
|
|
||||||
|
store = AgentRepository(sql_session_factory)
|
||||||
|
assert await store.list_favorite_ids("user-a") == set()
|
||||||
|
|
||||||
|
await store.add_favorite("user-a", "builtin-agent")
|
||||||
|
|
||||||
|
assert await _sql_favorite_rows(sql_session_factory, "user-a") == {
|
||||||
|
("builtin-agent", "user")
|
||||||
|
}
|
||||||
|
assert await store.list_favorite_ids("user-a") == {"builtin-agent"}
|
||||||
|
rows, total = await store.list_paginated(
|
||||||
|
user_id="user-a",
|
||||||
|
scope="mine",
|
||||||
|
search=None,
|
||||||
|
tag_ids=None,
|
||||||
|
page=1,
|
||||||
|
page_size=20,
|
||||||
|
)
|
||||||
|
assert total == 1
|
||||||
|
assert [row["id"] for row in rows] == ["builtin-agent"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_agent_store_only_owner_can_update_or_delete() -> None:
|
async def test_agent_store_only_owner_can_update_or_delete() -> None:
|
||||||
store = MemoryAgentStore()
|
store = MemoryAgentStore()
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ from types import SimpleNamespace
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from deerflow.agents.ai_writing.nodes import researcher as R
|
from deerflow.agents.ai_writing.nodes import researcher as R
|
||||||
|
from deerflow.agents.ai_writing.search import get_search_provider
|
||||||
from deerflow.agents.ai_writing.search.base import SearchResult
|
from deerflow.agents.ai_writing.search.base import SearchResult
|
||||||
|
from deerflow.agents.ai_writing.search.intranet_stub import IntranetSearchProvider
|
||||||
|
|
||||||
|
|
||||||
class _FakeIntranet:
|
class _FakeIntranet:
|
||||||
@@ -21,6 +23,7 @@ class _FakeIntranet:
|
|||||||
self.raw_responses = []
|
self.raw_responses = []
|
||||||
self.calls = []
|
self.calls = []
|
||||||
self.kw = kw
|
self.kw = kw
|
||||||
|
self.verify_ssl = kw.get("verify_ssl", False)
|
||||||
_FakeIntranet.instances.append(self)
|
_FakeIntranet.instances.append(self)
|
||||||
|
|
||||||
async def search(self, query, max_results=5, **_):
|
async def search(self, query, max_results=5, **_):
|
||||||
@@ -55,12 +58,24 @@ class _SkillProviderShouldNotRun:
|
|||||||
raise AssertionError("knowledge_base 模式不应调用通用 provider / 技能检索")
|
raise AssertionError("knowledge_base 模式不应调用通用 provider / 技能检索")
|
||||||
|
|
||||||
|
|
||||||
def _patch(monkeypatch, intranet_url="http://es/api", intranet_cls=_FakeIntranet):
|
class _FakeResponse:
|
||||||
|
status_code = 200
|
||||||
|
text = '[{"title":"内网标题","content":"内网正文","url":"https://intranet/doc"}]'
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return [{"title": "内网标题", "content": "内网正文", "url": "https://intranet/doc"}]
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(monkeypatch, intranet_url="http://es/api", intranet_cls=_FakeIntranet, intranet_verify_ssl=False):
|
||||||
_FakeIntranet.instances = []
|
_FakeIntranet.instances = []
|
||||||
cfg = {
|
cfg = {
|
||||||
"max_materials": 20,
|
"max_materials": 20,
|
||||||
"intranet_search_url": intranet_url,
|
"intranet_search_url": intranet_url,
|
||||||
"intranet_knowledge_base": "samples",
|
"intranet_knowledge_base": "samples",
|
||||||
|
"intranet_verify_ssl": intranet_verify_ssl,
|
||||||
}
|
}
|
||||||
monkeypatch.setattr(R, "get_ai_writing_config", lambda: cfg)
|
monkeypatch.setattr(R, "get_ai_writing_config", lambda: cfg)
|
||||||
monkeypatch.setattr(R, "use_builtin_agent", lambda *_: False)
|
monkeypatch.setattr(R, "use_builtin_agent", lambda *_: False)
|
||||||
@@ -125,6 +140,15 @@ async def test_knowledge_base_no_url_falls_back_to_default_endpoint(monkeypatch)
|
|||||||
assert _FakeIntranet.instances[0].kw.get("base_url") == "", "空地址应原样下传,由 provider 回退默认端点"
|
assert _FakeIntranet.instances[0].kw.get("base_url") == "", "空地址应原样下传,由 provider 回退默认端点"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_knowledge_base_passes_verify_ssl_config(monkeypatch):
|
||||||
|
_patch(monkeypatch, intranet_verify_ssl=True)
|
||||||
|
out = await R.researcher_node(_state(), {"configurable": {}})
|
||||||
|
|
||||||
|
assert _FakeIntranet.instances[0].kw.get("verify_ssl") is True
|
||||||
|
assert out["material_package"]["intranet_debug"]["verify_ssl"] is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_knowledge_base_does_not_orchestrate_skills(monkeypatch):
|
async def test_knowledge_base_does_not_orchestrate_skills(monkeypatch):
|
||||||
"""knowledge_base 不编排技能:通用 provider 一旦被调用就会 raise。"""
|
"""knowledge_base 不编排技能:通用 provider 一旦被调用就会 raise。"""
|
||||||
@@ -163,3 +187,44 @@ async def test_knowledge_base_failure_is_visible(monkeypatch):
|
|||||||
steps = pkg["skill_search_steps"]
|
steps = pkg["skill_search_steps"]
|
||||||
assert any(step["type"] == "skill_error" for step in steps)
|
assert any(step["type"] == "skill_error" for step in steps)
|
||||||
assert any(event["type"] == "notice" and event.get("raw") for event in out["progress_events"])
|
assert any(event["type"] == "notice" and event.get("raw") for event in out["progress_events"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_intranet_search_disables_ssl_verification_for_internal_https(monkeypatch):
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
seen: dict = {}
|
||||||
|
|
||||||
|
class _FakeAsyncClient:
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
seen.update(kwargs)
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, tb):
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def post(self, endpoint, json):
|
||||||
|
seen["endpoint"] = endpoint
|
||||||
|
seen["payload"] = json
|
||||||
|
return _FakeResponse()
|
||||||
|
|
||||||
|
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
|
||||||
|
|
||||||
|
provider = IntranetSearchProvider(base_url="https://intranet.example/search", knowledge_base_name="kb", timeout=300)
|
||||||
|
results = await provider.search("测试", max_results=1)
|
||||||
|
|
||||||
|
assert results
|
||||||
|
assert seen["verify"] is False
|
||||||
|
assert seen["trust_env"] is False
|
||||||
|
assert seen["endpoint"] == "https://intranet.example/search"
|
||||||
|
assert seen["payload"]["knowledge_base_name"] == "kb"
|
||||||
|
assert provider.calls[0]["verify_ssl"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_intranet_verify_ssl_can_be_reenabled_from_config():
|
||||||
|
provider = get_search_provider({"search_provider": "intranet", "intranet_verify_ssl": "true"})
|
||||||
|
|
||||||
|
assert isinstance(provider, IntranetSearchProvider)
|
||||||
|
assert provider.verify_ssl is True
|
||||||
@@ -194,7 +194,8 @@ Step1 受众目标分析完成:当前已入库受众 X 个,本步骤新增
|
|||||||
- 单个受众、账号、传播节点、脆弱点或策略失败,不中断整个流程。
|
- 单个受众、账号、传播节点、脆弱点或策略失败,不中断整个流程。
|
||||||
- 保存接口必须返回 `id`,否则不会进入下游步骤。
|
- 保存接口必须返回 `id`,否则不会进入下游步骤。
|
||||||
- 受众目标默认入库 3 个,关键传播节点默认入库 3 个。
|
- 受众目标默认入库 3 个,关键传播节点默认入库 3 个。
|
||||||
- 下游脆弱点/策略默认只选择前 2 个受众和前 2 个传播节点继续,可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
- 下游受众脆弱点/策略只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
|
||||||
|
- 下游传播节点脆弱点/策略默认只选择前 2 个已入库传播节点继续,可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
||||||
- 每个被选中的对象默认最多生成 3 个脆弱点、3 个策略。
|
- 每个被选中的对象默认最多生成 3 个脆弱点、3 个策略。
|
||||||
- 模型流式 `answer` 会自动拼接。
|
- 模型流式 `answer` 会自动拼接。
|
||||||
- 流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时,会自动再调用一次;仍失败才写入 `errors`。
|
- 流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时,会自动再调用一次;仍失败才写入 `errors`。
|
||||||
|
|||||||
@@ -86,7 +86,8 @@ python scripts/xdfx_flow.py --mode full --input xdfx-input.json --dry-run
|
|||||||
|
|
||||||
- 受众目标默认入库最优 3 个,可用 `--max-audiences` 调整。
|
- 受众目标默认入库最优 3 个,可用 `--max-audiences` 调整。
|
||||||
- 关键传播节点全局去重后默认入库最优 3 个,可用 `--max-pathways` 调整。
|
- 关键传播节点全局去重后默认入库最优 3 个,可用 `--max-pathways` 调整。
|
||||||
- 下游脆弱点/策略默认只从已入库结果中选择前 2 个受众、前 2 个传播节点继续,可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
- 下游受众脆弱点/策略只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
|
||||||
|
- 下游传播节点脆弱点/策略默认从已入库传播节点中选择前 2 个继续;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
||||||
- 每个被选中的受众/传播节点默认生成最多 3 个脆弱点,可用 `--max-vulns` 调整。
|
- 每个被选中的受众/传播节点默认生成最多 3 个脆弱点,可用 `--max-vulns` 调整。
|
||||||
- 每个被选中的受众/传播节点默认生成最多 3 个策略,可用 `--max-strategies` 调整。
|
- 每个被选中的受众/传播节点默认生成最多 3 个策略,可用 `--max-strategies` 调整。
|
||||||
- 每个保存接口必须拿到返回 `id` 才进入下游。
|
- 每个保存接口必须拿到返回 `id` 才进入下游。
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -77,8 +77,9 @@ python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-
|
|||||||
- Step1 的受众目标抽取结果里,`用户编号列表` / `编号id` / `id` 必须对应上一轮 `zhanghao` 账号数组的索引值;脚本会按这个索引依次找到账号并保存受众账号关系。
|
- Step1 的受众目标抽取结果里,`用户编号列表` / `编号id` / `id` 必须对应上一轮 `zhanghao` 账号数组的索引值;脚本会按这个索引依次找到账号并保存受众账号关系。
|
||||||
- Step2 成功后会写入 `topPathways`。
|
- Step2 成功后会写入 `topPathways`。
|
||||||
- Step1 默认入库最优 3 个受众;Step2 默认入库最优 3 个传播节点。
|
- Step1 默认入库最优 3 个受众;Step2 默认入库最优 3 个传播节点。
|
||||||
- Step3/Step4 默认只选择前 2 个已入库受众/传播节点继续,每个生成最多 3 个脆弱点;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
- Step3/Step5 的受众下游分析只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
|
||||||
- Step5/Step6 默认对上述选中的每个受众/传播节点生成最多 3 个策略。
|
- Step4/Step6 的传播节点下游分析默认从已入库传播节点中选择前 2 个继续;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
||||||
|
- Step3-Step6 对上述选中的每个受众/传播节点默认生成最多 3 个脆弱点或策略。
|
||||||
- 各步骤会同步前端 `task/task-action.step`;需要禁用时每条命令追加 `--no-step-status`。
|
- 各步骤会同步前端 `task/task-action.step`;需要禁用时每条命令追加 `--no-step-status`。
|
||||||
- 策略步骤会同步写入 channel、narrate、message、策略与受众/传播节点关系表,保证原前端策略表格/编排页可继续读取。
|
- 策略步骤会同步写入 channel、narrate、message、策略与受众/传播节点关系表,保证原前端策略表格/编排页可继续读取。
|
||||||
- 策略步骤保存 `信息形式/模态` 时,必须使用脚本读取 `sys/sys-dict-data` 的 `modelType` 字典并把文字转成 value 逗号串,不能把“图片,视频”等文字直接写入 `task-action-message.modelType`。
|
- 策略步骤保存 `信息形式/模态` 时,必须使用脚本读取 `sys/sys-dict-data` 的 `modelType` 字典并把文字转成 value 逗号串,不能把“图片,视频”等文字直接写入 `task-action-message.modelType`。
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -134,8 +134,9 @@ python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-
|
|||||||
|
|
||||||
- Step1 入库最优 3 个受众。
|
- Step1 入库最优 3 个受众。
|
||||||
- Step2 入库最优 3 个传播节点。
|
- Step2 入库最优 3 个传播节点。
|
||||||
- Step3/Step4 只从已入库结果中默认选择前 2 个受众/传播节点继续,每个生成最多 3 个脆弱点;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
- Step3/Step5 的受众下游分析只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
|
||||||
- Step5/Step6 对上述选中的每个受众/传播节点默认生成最多 3 个策略。
|
- Step4/Step6 的传播节点下游分析默认从已入库传播节点中选择前 2 个继续;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
|
||||||
|
- Step3-Step6 对上述选中的每个受众/传播节点默认生成最多 3 个脆弱点或策略。
|
||||||
- Step5/Step6 的策略模态必须按前端 `modelType` 字典保存为 value 逗号串;脚本会自动把“图片,视频/文本,图片”等文字转成 `task-action-message.modelType` 需要的值。
|
- Step5/Step6 的策略模态必须按前端 `modelType` 字典保存为 value 逗号串;脚本会自动把“图片,视频/文本,图片”等文字转成 `task-action-message.modelType` 需要的值。
|
||||||
|
|
||||||
总控反馈用户时建议使用这种格式:
|
总控反馈用户时建议使用这种格式:
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
|
|||||||
"topAudiences": [],
|
"topAudiences": [],
|
||||||
"pathwayCandidates": [],
|
"pathwayCandidates": [],
|
||||||
"topPathways": [],
|
"topPathways": [],
|
||||||
|
"audiencesWithPathways": [],
|
||||||
"audienceVulns": {},
|
"audienceVulns": {},
|
||||||
"pathwayVulns": {},
|
"pathwayVulns": {},
|
||||||
"audienceStrategies": [],
|
"audienceStrategies": [],
|
||||||
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
|
|||||||
return [item.get("account") or item]
|
return [item.get("account") or item]
|
||||||
|
|
||||||
|
|
||||||
|
def id_ref_key(value: Any) -> str:
|
||||||
|
number = as_int(value)
|
||||||
|
if number is not None:
|
||||||
|
return str(number)
|
||||||
|
return str(value or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def audience_name_key(value: Any) -> str:
|
||||||
|
return re.sub(r"\s+", "", str(value or "")).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
|
||||||
|
"accountId": pathway.get("accountId"),
|
||||||
|
"accountName": pathway.get("accountName"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
|
||||||
|
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
|
||||||
|
if not key:
|
||||||
|
return
|
||||||
|
seen = {
|
||||||
|
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
|
||||||
|
for item in target
|
||||||
|
}
|
||||||
|
if key not in seen:
|
||||||
|
target.append(pathway)
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
|
||||||
|
"""Return only audiences that produced at least one saved key pathway."""
|
||||||
|
id_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
name_links: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for pathway in state.get("topPathways") or []:
|
||||||
|
if not isinstance(pathway, dict):
|
||||||
|
continue
|
||||||
|
for design_id in pathway.get("designIds") or []:
|
||||||
|
key = id_ref_key(design_id)
|
||||||
|
if key:
|
||||||
|
id_links.setdefault(key, []).append(pathway)
|
||||||
|
for name in pathway.get("relatedAudiences") or []:
|
||||||
|
key = audience_name_key(name)
|
||||||
|
if key:
|
||||||
|
name_links.setdefault(key, []).append(pathway)
|
||||||
|
|
||||||
|
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
|
||||||
|
for audience_item in state.get("topAudiences") or []:
|
||||||
|
if not isinstance(audience_item, dict):
|
||||||
|
continue
|
||||||
|
linked: list[dict[str, Any]] = []
|
||||||
|
design_key = id_ref_key(audience_item.get("designId"))
|
||||||
|
for pathway in id_links.get(design_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
name_key = audience_name_key(audience_item.get("targetAudience"))
|
||||||
|
for pathway in name_links.get(name_key, []):
|
||||||
|
add_unique_pathway(linked, pathway)
|
||||||
|
if linked:
|
||||||
|
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
|
||||||
|
pairs.append((item, linked))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
summary: list[dict[str, Any]] = []
|
||||||
|
for audience_item, pathways in audience_pathway_pairs(state):
|
||||||
|
summary.append(
|
||||||
|
{
|
||||||
|
"designId": audience_item.get("designId"),
|
||||||
|
"targetAudience": audience_item.get("targetAudience"),
|
||||||
|
"pathwayCount": len(pathways),
|
||||||
|
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
|
||||||
|
return bool(audience_pathway_pairs(state))
|
||||||
|
|
||||||
|
|
||||||
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
|
||||||
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
|
||||||
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
|
||||||
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
|
|||||||
source_key = "topAudiences" if audience else "topPathways"
|
source_key = "topAudiences" if audience else "topPathways"
|
||||||
id_key = "designId" if audience else "pathwayAccountId"
|
id_key = "designId" if audience else "pathwayAccountId"
|
||||||
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
|
||||||
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
|
||||||
|
if audience:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
|
items = [item for item, _ in audience_pathway_pairs(state)]
|
||||||
|
if not items:
|
||||||
|
state[selected_key] = []
|
||||||
|
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
items = all_items
|
||||||
limit = downstream_limit(ctx, audience=audience)
|
limit = downstream_limit(ctx, audience=audience)
|
||||||
selected = items[:limit]
|
selected = items[:limit]
|
||||||
state[selected_key] = [
|
selected_rows: list[dict[str, Any]] = []
|
||||||
{
|
for item in selected:
|
||||||
|
row = {
|
||||||
"id": item.get(id_key),
|
"id": item.get(id_key),
|
||||||
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
"name": item.get("targetAudience") if audience else item.get("accountName"),
|
||||||
}
|
}
|
||||||
for item in selected
|
if audience:
|
||||||
]
|
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
|
||||||
if items:
|
row["pathways"] = item.get("downstreamPathways") or []
|
||||||
|
selected_rows.append(row)
|
||||||
|
state[selected_key] = selected_rows
|
||||||
|
if selected:
|
||||||
record_event(
|
record_event(
|
||||||
state,
|
state,
|
||||||
stage,
|
stage,
|
||||||
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
|
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
|
||||||
{"limit": limit, "selected": state[selected_key]},
|
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
|
||||||
)
|
)
|
||||||
return selected
|
return selected
|
||||||
|
|
||||||
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
record_error(state, stage, "没有已入库受众,无法分析传播节点")
|
||||||
return
|
return
|
||||||
if state.get("topPathways") and not ctx.dry_run:
|
if state.get("topPathways") and not ctx.dry_run:
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
|
||||||
return
|
return
|
||||||
all_candidates: list[dict[str, Any]] = []
|
all_candidates: list[dict[str, Any]] = []
|
||||||
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
|
|||||||
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
|
||||||
saved.append(item)
|
saved.append(item)
|
||||||
state["topPathways"] = saved
|
state["topPathways"] = saved
|
||||||
|
state["audiencesWithPathways"] = audience_pathway_summary(state)
|
||||||
if saved:
|
if saved:
|
||||||
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
|
||||||
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
|
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
|
||||||
|
|
||||||
|
|
||||||
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
|
||||||
|
|||||||
Reference in new issue
Block a user