Merge branch 'dhk-1' into 'master'

代码修改

See merge request knowledge-base-bj/deerflow-server!167
This commit is contained in:
董宏凯 committed 2026-07-01 09:51:15 +00:00
commit ba0c32dcee
23 files changed
+1483 -183

No files matched your search

@@ -414,6 +414,8 @@ function AIWritingBody({
const routeBase = useAIWritingRouteBase();
const agentRoute = useAIWritingAgentRouteState(routeBase);
const isAdmin = getStoredAuth()?.system_role === 'admin';
const externalNotebookMode =
studioToken != null || initialNotebookId != null || initialDocumentIds != null;
// 需求 0616 #5:左栏点「笔记本 / 所有空间」→ 在右侧内嵌打开对应页,替换右侧内容
// (写作布局不卸载,仅 display:none 隐藏,避免 WritingSetupChat 重挂丢线程)。
@@ -432,12 +434,12 @@ function AIWritingBody({
}
}, [agentRoute.active]);
const agentsActive = sidebarTab === 'agents';
const agentsActive = !externalNotebookMode && sidebarTab === 'agents';
// 研究模板侧栏在「写作配置(idle)」与「历史回看」两态都展示(点历史记录后保留侧栏、
// 历史内容显示到右侧区域);写作进行中收起;嵌入态(studioToken)始终不展示。
const showSidebar =
!studioToken && (state.status === 'idle' || isHistoryMode || agentsActive);
!externalNotebookMode && (state.status === 'idle' || isHistoryMode || agentsActive);
const compactAppearanceActions = compactChrome ? (
<AIWritingCompactAppearanceActions />
) : null;
+1 -1
View File
@@ -273,7 +273,7 @@ The AI-writing graph (`packages/harness/deerflow/agents/ai_writing/`) has four s
Seeding mirrors the roundtable pattern: verified `config.yaml` + `SOUL.md` ship in `app/gateway/routers/_ai_writing_seed_assets/<id>/`; `_ai_writing_seed.py::ensure_ai_writing_functional_agents()` copies them to `.deer-flow/agents/` when missing (never overwrites local edits), called from the startup lifespan (before `_sync_legacy_agents`, so the agents appear in `/api/agents` on first boot) and self-heals from `GET /api/ai-writing/sessions` + `GET /api/ai-writing/article-types`. Tests: `tests/test_ai_writing_seed.py`, `tests/test_ai_writing_agent_runner.py`.
**Skill-driven retrieval (检索类型).** `material_source` has three values: `general` (通用检索, default), `knowledge_base` (知识库检索), and `notebook` (我的空间); legacy `skill` is normalized to `general` by the researcher node. **`knowledge_base` 直连内网 ES 接口(= 旧版「内网直连检索」)** — it bypasses skill orchestration entirely and calls `IntranetSearchProvider` over `ai_writing.intranet_search_url` (concurrent over the generated keywords; `intranet_knowledge_base`/`intranet_search_timeout` apply). **An empty/unconfigured `intranet_search_url` does NOT disable it** — the empty string is passed straight to `IntranetSearchProvider`, which falls back to its built-in default endpoint (`intranet_stub._DEFAULT_ENDPOINT`), exactly like the legacy `get_search_provider(search_provider=intranet)` path. So 内网直连检索 works out of the box without configuring an address (configure `intranet_search_url` only to point at a different endpoint). This is the "弱模型识别不了技能" escape hatch — the old no-skill retrieval path restored as an explicit option (weak models like deepseek-chat often fail to invoke configured skills). Tests: `tests/test_ai_writing_knowledge_base.py`. General retrieval is orchestrated from the **skills configured on the `ai-writing-researcher` agent** (`config.yaml → skills`, editable from the agent management page): the built-in `knowledge-base-search` skill (知识库检索) runs the native keyword fan-out web/intranet search (the old default path), while any other configured skill searches with a `description`-targeted query; skills run in configured order with early-stop at `max_materials`, and an empty/unresolvable skills config falls back to the built-in knowledge search so retrieval always works. The skill itself is seeded from `_ai_writing_seed_assets/skills/knowledge-base-search/SKILL.md` into `skills/public/` (never overwrites), and a pre-existing researcher `config.yaml` lacking the `skills` key is patched in place with the default (explicit `skills: []` is respected). **PAUSE-1 input append flow**: when the user types natural language into the material-confirm input (`re_search`/legacy `enable_skill_search` + `user_query`), the node matches the most relevant configured skill via `SkillPicker(candidates=…)` (skipped when only one skill is configured), searches with the user's words, and **appends** the new materials to the existing package (URL-dedup, original order preserved, no LLM re-rank/truncation, 100-item hard cap). **Skill-failure fault tolerance**: a configured skill that times out / raises / has all its keyword searches fail (vs. legitimately returning 0 results) no longer just silently moves on — the researcher node emits a `skill_search_step` of type `skill_error` (carrying the error `reason`), and if the configured skills collected **no** new materials at all it runs a `fallback` pass over other available skills (`_fallback_candidate_skills`: the built-in `knowledge-base-search` native path first, then other enabled catalog skills not yet tried) instead of erroring out. The frontend `SkillStepsCard` renders `skill_error` (rose) and `fallback` (amber) so the timeline shows "技能报错 → 改用其他技能搜集". Tests: `tests/test_ai_writing_material_source.py`.
**Skill-driven retrieval (检索类型).** `material_source` has three values: `general` (通用检索, default), `knowledge_base` (知识库检索), and `notebook` (我的空间); legacy `skill` is normalized to `general` by the researcher node. **`knowledge_base` 直连内网 ES 接口(= 旧版「内网直连检索」)** — it bypasses skill orchestration entirely and calls `IntranetSearchProvider` over `ai_writing.intranet_search_url` (concurrent over the generated keywords; `intranet_knowledge_base`/`intranet_search_timeout` apply). The client uses `trust_env=False` and defaults `intranet_verify_ssl=false`, so internal HTTPS endpoints with self-signed/internal-CA certificates do not fail on certificate verification; set `ai_writing.intranet_verify_ssl: true` only for environments that require strict certificate checks. **An empty/unconfigured `intranet_search_url` does NOT disable it** — the empty string is passed straight to `IntranetSearchProvider`, which falls back to its built-in default endpoint (`intranet_stub._DEFAULT_ENDPOINT`), exactly like the legacy `get_search_provider(search_provider=intranet)` path. So 内网直连检索 works out of the box without configuring an address (configure `intranet_search_url` only to point at a different endpoint). This is the "弱模型识别不了技能" escape hatch — the old no-skill retrieval path restored as an explicit option (weak models like deepseek-chat often fail to invoke configured skills). Tests: `tests/test_ai_writing_knowledge_base.py`. General retrieval is orchestrated from the **skills configured on the `ai-writing-researcher` agent** (`config.yaml → skills`, editable from the agent management page): the built-in `knowledge-base-search` skill (知识库检索) runs the native keyword fan-out web/intranet search (the old default path), while any other configured skill searches with a `description`-targeted query; skills run in configured order with early-stop at `max_materials`, and an empty/unresolvable skills config falls back to the built-in knowledge search so retrieval always works. The skill itself is seeded from `_ai_writing_seed_assets/skills/knowledge-base-search/SKILL.md` into `skills/public/` (never overwrites), and a pre-existing researcher `config.yaml` lacking the `skills` key is patched in place with the default (explicit `skills: []` is respected). **PAUSE-1 input append flow**: when the user types natural language into the material-confirm input (`re_search`/legacy `enable_skill_search` + `user_query`), the node matches the most relevant configured skill via `SkillPicker(candidates=…)` (skipped when only one skill is configured), searches with the user's words, and **appends** the new materials to the existing package (URL-dedup, original order preserved, no LLM re-rank/truncation, 100-item hard cap). **Skill-failure fault tolerance**: a configured skill that times out / raises / has all its keyword searches fail (vs. legitimately returning 0 results) no longer just silently moves on — the researcher node emits a `skill_search_step` of type `skill_error` (carrying the error `reason`), and if the configured skills collected **no** new materials at all it runs a `fallback` pass over other available skills (`_fallback_candidate_skills`: the built-in `knowledge-base-search` native path first, then other enabled catalog skills not yet tried) instead of erroring out. The frontend `SkillStepsCard` renders `skill_error` (rose) and `fallback` (amber) so the timeline shows "技能报错 → 改用其他技能搜集". Tests: `tests/test_ai_writing_material_source.py`.
**Skill-call timeout** is hardcoded **300s (5 min)** — `researcher.py` `_SKILL_SEARCH_TIMEOUT_SECONDS` (per-skill wrapper) and `_skill_agent.py` `_REAL_AGENT_TIMEOUT_SECONDS` (whole real-agent skill run). The intranet ES fallback (`IntranetSearchProvider`) gets a longer **600s (10 min)** default (`intranet_stub.py`; overridable via `ai_writing.intranet_search_timeout`) — that interface is the slow one, so our own fallback timeout is generous to avoid being blamed for "检不出".
@@ -135,6 +135,10 @@ FastAPI application providing REST endpoints for frontend integration:
Custom agents are indexed in the `agents` database table. The `id` column is the stable runtime identifier and filesystem directory name (`.deer-flow/agents/{id}/SOUL.md`); `name` is display-only, may be Chinese, and may be duplicated. Rows with `user_id = NULL` are built-in agents, user rows are private unless `published = true`, and users can only update or delete agents they created.
### AI Writing Intranet Retrieval
AI 写作的「知识库检索」会直连 `ai_writing.intranet_search_url` 内网 ES/知识库接口,并默认跳过代理环境变量和 HTTPS 证书校验,适配自签名或内部 CA 证书的内网服务。需要强校验时可在 `config.yaml` 里设置 `ai_writing.intranet_verify_ssl: true`。
### IM Channels
The IM bridge supports Feishu, Slack, and Telegram. Slack and Telegram still use the final `runs.wait()` response path, while Feishu now streams through `runs.stream(["messages-tuple", "values"])` and updates a single in-thread card in place.
@@ -27,13 +27,15 @@ from types import SimpleNamespace
from langchain_core.runnables import RunnableConfig
from ..search import get_search_provider, SearchResult
from ..prompts.researcher_prompts import (
EXTRACT_KEYWORDS_SYSTEM,
EXTRACT_KEYWORDS_USER,
RANK_MATERIALS_SYSTEM,
RANK_MATERIALS_USER,
)
from ..search import SearchResult, get_search_provider
from ..search.skill_search import SkillPicker
from ..state import AIWritingState, Material, MaterialPackage
from ..prompts.researcher_prompts import (
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
RANK_MATERIALS_SYSTEM, RANK_MATERIALS_USER,
)
from ._agent_runner import (
KEYWORDS_READY_MARKER,
MATERIALS_RANKED_MARKER,
@@ -620,6 +622,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 内网慢检索超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
# 默认 600s,且最低 300s。
timeout=_intranet_timeout_seconds(ai_writing_config),
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
)
endpoint_label = intranet_url or getattr(provider, "endpoint", "内置默认内网端点")
intranet_debug = {
@@ -628,6 +631,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
"endpoint": endpoint_label,
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
"timeout": _intranet_timeout_seconds(ai_writing_config),
"verify_ssl": getattr(provider, "verify_ssl", False),
"status": "running",
"queries": [],
"calls": [],
@@ -780,7 +784,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
)
return (results or []), ""
except asyncio.TimeoutError:
except TimeoutError:
msg = f"检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)"
logger.warning("skill[%s] search timed out after %ss", skill_name, _SKILL_SEARCH_TIMEOUT_SECONDS)
search_errors.append(f"skill[{skill_name}]: {msg}")
@@ -923,13 +927,14 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
intranet_url = (ai_writing_config.get("intranet_search_url") or "").strip()
# 诊断开关:开启后即使技能成功也照样跑完内网并记录完整结果(用于验证内网接口是否真能通)。
intranet_debug_force = bool(ai_writing_config.get("intranet_search_debug", False))
intranet_task: "asyncio.Task | None" = None
intranet_task: asyncio.Task | None = None
intranet_holder: dict = {}
intranet_debug = {
"configured": bool(intranet_url),
"endpoint": intranet_url or "(未配置 ai_writing.intranet_search_url)",
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
"timeout": _intranet_timeout_seconds(ai_writing_config),
"verify_ssl": bool(ai_writing_config.get("intranet_verify_ssl", False)),
"debug_force": intranet_debug_force,
"status": "pending" if intranet_url else "not_configured",
"queries": [], # 每个检索词:{query, count, error?}
@@ -952,6 +957,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 兜底接口超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
# 默认 600s(10 分钟),且最低 300s。
timeout=_intranet_timeout_seconds(ai_writing_config),
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
)
intranet_holder["provider"] = intranet
fb_keywords = keywords[:keyword_count] or [intent]
@@ -1,8 +1,8 @@
from .base import BaseSearchProvider, SearchResult
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout
from .demo_search import DemoSearchProvider
from .duckduckgo_search import DuckDuckGoSearchProvider
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout, normalize_intranet_verify_ssl
from .skill_search import SkillSearchProvider
@@ -14,6 +14,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
intranet_knowledge_base: samples
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
intranet_verify_ssl: false # 内网 HTTPS 自签名证书场景默认关闭证书校验
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
@@ -26,6 +27,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
base_url=cfg.get("intranet_search_url", ""),
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
verify_ssl=normalize_intranet_verify_ssl(cfg.get("intranet_verify_ssl")),
)
if provider == "tavily":
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
@@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import time
from typing import List
from .base import BaseSearchProvider, SearchResult, simplify_query
@@ -15,6 +14,8 @@ _DEFAULT_KB = "samples"
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
_MIN_TIMEOUT_SECONDS = 300
_DEFAULT_TIMEOUT_SECONDS = 600
# 内网 HTTPS 检索服务常使用自签名/内部 CA 证书,默认跳过证书校验避免 SSL 证书链错误。
_DEFAULT_VERIFY_SSL = False
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
_RAW_SNIPPET_LIMIT = 2000
@@ -28,6 +29,23 @@ def normalize_intranet_timeout(timeout: object, default: int = _DEFAULT_TIMEOUT_
return max(value, _MIN_TIMEOUT_SECONDS)
def normalize_intranet_verify_ssl(value: object, default: bool = _DEFAULT_VERIFY_SSL) -> bool:
"""返回内网检索是否校验 SSL 证书,兼容 config.yaml 里的 bool / 字符串写法。"""
if isinstance(value, bool):
return value
if value is None:
return default
if isinstance(value, (int, float)):
return bool(value)
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return default
def _pick(d: dict, keys: tuple) -> str:
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
for k in keys:
@@ -62,12 +80,14 @@ class IntranetSearchProvider(BaseSearchProvider):
base_url: str = "",
knowledge_base_name: str = _DEFAULT_KB,
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
verify_ssl: bool = _DEFAULT_VERIFY_SSL,
**_ignored,
):
self.endpoint = base_url or _DEFAULT_ENDPOINT
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
self.timeout = normalize_intranet_timeout(timeout)
self.verify_ssl = normalize_intranet_verify_ssl(verify_ssl)
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
self.errors: list[str] = []
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
@@ -82,7 +102,7 @@ class IntranetSearchProvider(BaseSearchProvider):
if snippet:
self.raw_responses.append(snippet)
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
async def search(self, query: str, max_results: int = 5, **kwargs) -> list[SearchResult]:
results = await self._search_once(query, max_results)
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
@@ -93,7 +113,7 @@ class IntranetSearchProvider(BaseSearchProvider):
results = await self._search_once(simpler, max_results)
return results
async def _search_once(self, query: str, max_results: int = 5) -> List[SearchResult]:
async def _search_once(self, query: str, max_results: int = 5) -> list[SearchResult]:
import httpx
payload = {
@@ -112,12 +132,13 @@ class IntranetSearchProvider(BaseSearchProvider):
"status": "pending",
"ok": False,
"count": 0,
"verify_ssl": self.verify_ssl,
}
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
logger.info(
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss query=%r",
self.endpoint, self.knowledge_base_name, self.timeout, query,
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss verify_ssl=%s query=%r",
self.endpoint, self.knowledge_base_name, self.timeout, self.verify_ssl, query,
)
try:
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
@@ -125,7 +146,8 @@ class IntranetSearchProvider(BaseSearchProvider):
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
# 调用同款修法)。
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False) as client:
# verify=False:内网 HTTPS 常见自签名/内部 CA 证书;这里仅跳过本内网检索端点的证书校验。
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False, verify=self.verify_ssl) as client:
resp = await client.post(self.endpoint, json=payload)
call["http_status"] = getattr(resp, "status_code", None)
resp.raise_for_status()
@@ -210,7 +232,7 @@ class IntranetSearchProvider(BaseSearchProvider):
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
)
results: List[SearchResult] = []
results: list[SearchResult] = []
for i, item in enumerate(items[:max_results]):
if not isinstance(item, dict):
continue
@@ -6,20 +6,21 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, case, delete, func, or_, select
from sqlalchemy import and_, case, delete, func, literal, not_, or_, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.config.agent_name_desensitize import source_terms_for_needle
from deerflow.persistence.agents.base import AgentStore
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.tags.model import TagAssignmentRow
from deerflow.persistence.agents.model import (
AgentExtrasRow,
AgentFavoriteRow,
AgentPinRow,
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.tags.model import TagAssignmentRow
def _row_to_dict(row: AgentRow) -> dict[str, Any]:
@@ -32,9 +33,33 @@ def _row_to_dict(row: AgentRow) -> dict[str, Any]:
return data
class AgentRepository(AgentStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
class AgentRepository(AgentStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _business_chain_agent_exists(agent_id_col):
return (
select(RoundtableChainRow.id)
.where(
or_(
RoundtableChainRow.seats.like(literal('%"agent_id": "') + agent_id_col + literal('"%')),
RoundtableChainRow.seats.like(literal('%"agent_id":"') + agent_id_col + literal('"%')),
)
)
.exists()
)
@classmethod
def _valid_favorite_filter(cls):
return or_(
AgentFavoriteRow.origin == "user",
and_(
AgentFavoriteRow.origin == "position",
AgentRow.user_id.is_not(None),
not_(cls._business_chain_agent_exists(AgentFavoriteRow.agent_id)),
),
)
@staticmethod
def _visible_filter(user_id: str):
@@ -52,15 +77,23 @@ class AgentRepository(AgentStore):
``published=false``, so this clause cannot lean on ``published``;
- favorited — the "我的" listing keeps favorited agents even after
the owner unpublishes them, so reads keep honoring the favorite.
"""
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
return or_(
AgentRow.user_id.is_(None),
AgentRow.user_id == user_id,
AgentRow.published.is_(True),
AgentRow.is_private.is_(True),
AgentRow.id.in_(fav_subq),
)
"""
valid_favorite_exists = (
select(AgentFavoriteRow.id)
.where(
AgentFavoriteRow.user_id == user_id,
AgentFavoriteRow.agent_id == AgentRow.id,
AgentRepository._valid_favorite_filter(),
)
.exists()
)
return or_(
AgentRow.user_id.is_(None),
AgentRow.user_id == user_id,
AgentRow.published.is_(True),
AgentRow.is_private.is_(True),
valid_favorite_exists,
)
@staticmethod
def _listing_filter(user_id: str):
@@ -268,11 +301,20 @@ class AgentRepository(AgentStore):
or_(AgentRow.user_id.is_(None), AgentRow.published.is_(True)),
AgentRow.is_private.is_(False),
)
if scope == "mine":
# owned OR favorited. A correlated IN-subquery (not a JOIN) keeps a
# single row per agent even when it is both owned and favorited.
fav_subq = select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
return or_(AgentRow.user_id == user_id, AgentRow.id.in_(fav_subq))
if scope == "mine":
# owned OR valid favorites. Built-in agents and business-chain
# seats only enter "mine" when explicitly favorited by the user;
# the original position-granted behavior stays for ordinary agents.
valid_favorite_exists = (
select(AgentFavoriteRow.id)
.where(
AgentFavoriteRow.user_id == user_id,
AgentFavoriteRow.agent_id == AgentRow.id,
self._valid_favorite_filter(),
)
.exists()
)
return or_(AgentRow.user_id == user_id, valid_favorite_exists)
if scope == "private":
return AgentRow.is_private.is_(True)
if scope == "builtin":
@@ -437,18 +479,22 @@ class AgentRepository(AgentStore):
# --- Favorites ("收藏") ---
async def add_favorite(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentFavoriteRow.id)
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
return
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
async def add_favorite(self, user_id: str, agent_id: str) -> None:
async with self._sf() as session:
existing = (
await session.execute(
select(AgentFavoriteRow)
.where(AgentFavoriteRow.user_id == user_id, AgentFavoriteRow.agent_id == agent_id)
.limit(1)
)
).scalar_one_or_none()
if existing is not None:
if existing.origin != "user" or existing.position_id is not None:
existing.origin = "user"
existing.position_id = None
await session.commit()
return
session.add(AgentFavoriteRow(id=uuid4().hex, user_id=user_id, agent_id=agent_id))
try:
await session.commit()
except IntegrityError:
@@ -467,12 +513,17 @@ class AgentRepository(AgentStore):
await session.commit()
return (result.rowcount or 0) > 0
async def list_favorite_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentFavoriteRow.agent_id).where(AgentFavoriteRow.user_id == user_id)
)
return {row[0] for row in result.all()}
async def list_favorite_ids(self, user_id: str) -> set[str]:
async with self._sf() as session:
result = await session.execute(
select(AgentFavoriteRow.agent_id)
.join(AgentRow, AgentRow.id == AgentFavoriteRow.agent_id)
.where(
AgentFavoriteRow.user_id == user_id,
self._valid_favorite_filter(),
)
)
return {row[0] for row in result.all()}
async def delete_favorites_for_agent(self, agent_id: str) -> None:
async with self._sf() as session:
@@ -0,0 +1,90 @@
"""Stop position-granted favorites from collecting built-ins/chain seats.
Built-in agents and agents used as business-chain seats should only enter
"mine" when a user explicitly clicks the favorite action. This migration
removes historical automatic ``origin='position'`` rows for those agents while
preserving user-created favorites (``origin='user'``) and ordinary
position-granted agent favorites.
Revision ID: 20260701_01
Revises: 20260625_04
Create Date: 2026-07-01
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260701_01"
down_revision = "20260625_04"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def upgrade() -> None:
if not (
_has_table("agent_favorites")
and _has_column("agent_favorites", "origin")
and _has_column("agent_favorites", "agent_id")
):
return
favorites = sa.table(
"agent_favorites",
sa.column("agent_id", sa.String(64)),
sa.column("origin", sa.String(16)),
)
cleanup_conditions = []
if _has_table("agents") and _has_column("agents", "user_id"):
agents = sa.table(
"agents",
sa.column("id", sa.String(64)),
sa.column("user_id", sa.String(64)),
)
cleanup_conditions.append(
favorites.c.agent_id.in_(sa.select(agents.c.id).where(agents.c.user_id.is_(None)))
)
if _has_table("roundtable_chains") and _has_column("roundtable_chains", "seats"):
chains = sa.table(
"roundtable_chains",
sa.column("seats", sa.Text()),
)
cleanup_conditions.append(
sa.select(sa.literal(1))
.select_from(chains)
.where(
sa.or_(
chains.c.seats.like(sa.literal('%"agent_id": "') + favorites.c.agent_id + sa.literal('"%')),
chains.c.seats.like(sa.literal('%"agent_id":"') + favorites.c.agent_id + sa.literal('"%')),
)
)
.exists()
)
if not cleanup_conditions:
return
op.get_bind().execute(
favorites.delete().where(
favorites.c.origin == "position",
sa.or_(*cleanup_conditions),
)
)
def downgrade() -> None:
# Data cleanup only; removed auto-grants are not reconstructable.
pass
@@ -12,7 +12,8 @@ Resolution rule (per resource type):
* For a resource type **with** bound tags → the union of resources carrying any
of those tags.
* For a resource type **without** bound tags → the *default set*:
- ``agent`` → all built-in agents (``user_id IS NULL``)
- ``agent`` -> built-ins first, then filtered out of
position-granted favorites
- ``scheduled_task`` → empty (no built-in tasks)
- ``skill`` → empty (the 广场 still shows every visible skill;
a position only *adds* its tagged skills to the member's "我的技能")
@@ -24,9 +25,12 @@ Resolution rule (per resource type):
Materialization (``sync_user``) only ever touches rows it created itself
(``origin='position'``), so a user's own favorites / subscriptions are never
clobbered. ``skill`` is materialized into ``skill_favorites`` exactly like
``agent`` → ``agent_favorites``: the tagged skills are *added* to the user's
"我的技能" while the browse-all 广场 stays complete (it is **not** restricted).
clobbered. ``agent`` keeps the original position-granted behavior for ordinary
tagged agents, but filters out built-ins and business-chain seats; those only
enter "mine" after an explicit user favorite. ``skill`` is still materialized
into ``skill_favorites`` so tagged skills are added to the user's "mine"
collection while the browse-all square stays complete (it is **not**
restricted).
Only recommended-question visibility is computed live by :meth:`resolve_visible`
and is *not* materialized. (The admin-configured prompts shown to users ARE the
recommended questions — there is no separate prompt-template store.)
@@ -34,6 +38,7 @@ recommended questions — there is no separate prompt-template store.)
from __future__ import annotations
import json
import logging
import uuid
from datetime import UTC, datetime
@@ -43,13 +48,15 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.positions.base import PositionStore
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.scheduled_tasks.model import ScheduledTaskSubscriptionRow
from deerflow.persistence.skills.model import SkillFavoriteRow
from deerflow.persistence.tags.base import TagStore
logger = logging.getLogger(__name__)
# Materialized into "我的" tables by sync_user.
# Reconciled by sync_user. ``agent`` preserves ordinary position-granted rows,
# but filters built-ins and business-chain seats.
_MATERIALIZED_TYPES = ("agent", "skill", "scheduled_task")
# Resolved live at query time (visibility filters), never materialized.
_VISIBILITY_TYPES = ("recommended_question",)
@@ -93,8 +100,12 @@ class PositionSyncService:
bindings = await self._positions.get_tag_bindings(pid) if pid else {}
tag_ids = bindings.get(resource_type) or []
if tag_ids:
return set(await self._tags.list_targets_for_tags(resource_type, tag_ids))
return await self._default_set(resource_type, session)
target = set(await self._tags.list_targets_for_tags(resource_type, tag_ids))
else:
target = await self._default_set(resource_type, session)
if resource_type == "agent":
return await self._filter_materializable_agents(target, session)
return target
async def _default_set(self, resource_type: str, session: AsyncSession) -> set[str]:
if resource_type == "agent":
@@ -103,6 +114,37 @@ class PositionSyncService:
# scheduled_task (and any other materialized type) defaults to empty.
return set()
async def _filter_materializable_agents(self, agent_ids: set[str], session: AsyncSession) -> set[str]:
if not agent_ids:
return set()
rows = (
await session.execute(
select(AgentRow.id).where(
AgentRow.id.in_(agent_ids),
AgentRow.user_id.is_not(None),
)
)
).all()
candidate_ids = {r[0] for r in rows}
return candidate_ids - await self._business_chain_agent_ids(session)
async def _business_chain_agent_ids(self, session: AsyncSession) -> set[str]:
rows = (await session.execute(select(RoundtableChainRow.seats))).scalars()
agent_ids: set[str] = set()
for raw in rows:
if not raw:
continue
try:
seats = json.loads(raw) if isinstance(raw, str) else raw
except Exception:
continue
if not isinstance(seats, list):
continue
for seat in seats:
if isinstance(seat, dict) and isinstance(seat.get("agent_id"), str):
agent_ids.add(seat["agent_id"])
return agent_ids
# --------------------------------------------------------- materialize
async def sync_user(self, user_id: str) -> None:
@@ -1,33 +1,131 @@
from __future__ import annotations
import pytest
import yaml
from app.gateway.routers.agents import (
_backfill_listing_cache,
_load_config_extras,
_write_agent_config,
)
from app.gateway.services import build_run_config
from deerflow.config import paths as paths_module
from deerflow.config.agents_config import AgentConfig
from deerflow.persistence.agents import MemoryAgentStore
from __future__ import annotations
import json
from uuid import uuid4
import pytest
import pytest_asyncio
import yaml
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers.agents import (
_backfill_listing_cache,
_load_config_extras,
_write_agent_config,
)
from app.gateway.services import build_run_config
from deerflow.config import paths as paths_module
from deerflow.config.agents_config import AgentConfig
from deerflow.persistence.agents import AgentRepository, MemoryAgentStore
from deerflow.persistence.agents.model import AgentFavoriteRow, AgentRow
from deerflow.persistence.base import Base
from deerflow.persistence.positions.memory import MemoryPositionStore
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
from deerflow.persistence.tags.memory import MemoryTagStore
from deerflow.services.position_sync import PositionSyncService
@pytest.fixture
def isolated_paths(tmp_path, monkeypatch):
"""Redirect get_paths() at a fresh temp directory for the duration of one test."""
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
monkeypatch.setattr(paths_module, "_paths", None)
yield tmp_path
monkeypatch.setattr(paths_module, "_paths", None)
def isolated_paths(tmp_path, monkeypatch):
"""Redirect get_paths() at a fresh temp directory for the duration of one test."""
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
monkeypatch.setattr(paths_module, "_paths", None)
yield tmp_path
monkeypatch.setattr(paths_module, "_paths", None)
@pytest_asyncio.fixture
async def sql_session_factory(tmp_path):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'agent-tests.db'}")
async with engine.begin() as conn:
# Importing PositionSyncService above registers the agent / skill /
# scheduled-task rows it materializes into.
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
try:
yield sf
finally:
await engine.dispose()
async def _add_sql_agent(sf, *, agent_id: str, user_id: str | None) -> None:
async with sf() as session:
session.add(
AgentRow(
id=agent_id,
name=agent_id,
description="",
user_id=user_id,
published=True,
is_private=False,
square_id="",
)
)
await session.commit()
async def _add_sql_favorite(sf, *, user_id: str, agent_id: str, origin: str) -> None:
async with sf() as session:
session.add(
AgentFavoriteRow(
id=uuid4().hex,
user_id=user_id,
agent_id=agent_id,
origin=origin,
position_id="default" if origin == "position" else None,
)
)
await session.commit()
async def _add_sql_chain(sf, *, chain_id: str, agent_ids: list[str]) -> None:
async with sf() as session:
session.add(
RoundtableChainRow(
id=chain_id,
user_id="chain-owner",
title=chain_id,
description="",
seats=json.dumps(
[{"agent_id": agent_id, "name": agent_id, "description": ""} for agent_id in agent_ids],
ensure_ascii=False,
),
)
)
await session.commit()
async def _sql_favorite_rows(sf, user_id: str) -> set[tuple[str, str]]:
async with sf() as session:
rows = (
await session.execute(
select(AgentFavoriteRow.agent_id, AgentFavoriteRow.origin).where(
AgentFavoriteRow.user_id == user_id
)
)
).all()
return {(agent_id, origin) for agent_id, origin in rows}
async def _default_positions() -> MemoryPositionStore:
positions = MemoryPositionStore()
await positions.create_position(
{
"id": "default",
"name": "Default",
"description": "",
"is_default": True,
}
)
return positions
@pytest.mark.asyncio
async def test_agent_store_allows_duplicate_chinese_names_and_filters_visibility() -> None:
store = MemoryAgentStore()
chinese_name = "\u4e2d\u6587\u52a9\u624b"
async def test_agent_store_allows_duplicate_chinese_names_and_filters_visibility() -> None:
store = MemoryAgentStore()
chinese_name = "\u4e2d\u6587\u52a9\u624b"
await store.ensure_builtin({"id": "builtin-agent", "name": chinese_name, "description": "built in"})
await store.create({"id": "private-a", "name": chinese_name, "description": "private", "user_id": "user-a"})
await store.create({"id": "published-b", "name": chinese_name, "description": "published", "user_id": "user-b", "published": True})
@@ -35,14 +133,144 @@ async def test_agent_store_allows_duplicate_chinese_names_and_filters_visibility
user_a_ids = {agent["id"] for agent in await store.list_visible("user-a")}
assert user_a_ids == {"builtin-agent", "private-a", "published-b"}
user_c_ids = {agent["id"] for agent in await store.list_visible("user-c")}
assert user_c_ids == {"builtin-agent", "published-b"}
@pytest.mark.asyncio
async def test_agent_store_only_owner_can_update_or_delete() -> None:
store = MemoryAgentStore()
user_c_ids = {agent["id"] for agent in await store.list_visible("user-c")}
assert user_c_ids == {"builtin-agent", "published-b"}
@pytest.mark.asyncio
async def test_position_sync_default_does_not_collect_builtin_agents(sql_session_factory) -> None:
await _add_sql_agent(sql_session_factory, agent_id="builtin-auto", user_id=None)
await _add_sql_agent(sql_session_factory, agent_id="builtin-manual", user_id=None)
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="builtin-auto",
origin="position",
)
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="builtin-manual",
origin="user",
)
sync = PositionSyncService(
sql_session_factory,
await _default_positions(),
MemoryTagStore(),
)
await sync.sync_user("user-a")
assert await _sql_favorite_rows(sql_session_factory, "user-a") == {
("builtin-manual", "user")
}
@pytest.mark.asyncio
async def test_position_sync_tagged_agents_are_not_auto_favorited(sql_session_factory) -> None:
await _add_sql_agent(sql_session_factory, agent_id="builtin-tagged", user_id=None)
await _add_sql_agent(sql_session_factory, agent_id="custom-tagged", user_id="owner")
await _add_sql_agent(sql_session_factory, agent_id="chain-tagged", user_id="owner")
await _add_sql_chain(sql_session_factory, chain_id="chain-1", agent_ids=["chain-tagged"])
positions = await _default_positions()
tags = MemoryTagStore()
await tags.create_tag({"id": "agent-tag", "name": "Agent tag", "tag_type": "agent"})
await tags.set_target_tags("agent", "builtin-tagged", ["agent-tag"])
await tags.set_target_tags("agent", "custom-tagged", ["agent-tag"])
await tags.set_target_tags("agent", "chain-tagged", ["agent-tag"])
await positions.set_tag_bindings("default", {"agent": ["agent-tag"]})
sync = PositionSyncService(sql_session_factory, positions, tags)
await sync.sync_user("user-a")
assert await _sql_favorite_rows(sql_session_factory, "user-a") == {
("custom-tagged", "position")
}
@pytest.mark.asyncio
async def test_mine_scope_ignores_auto_agent_favorites_but_keeps_manual_builtin(sql_session_factory) -> None:
await _add_sql_agent(sql_session_factory, agent_id="builtin-auto", user_id=None)
await _add_sql_agent(sql_session_factory, agent_id="builtin-manual", user_id=None)
await _add_sql_agent(sql_session_factory, agent_id="custom-auto", user_id="owner")
await _add_sql_agent(sql_session_factory, agent_id="chain-auto", user_id="owner")
await _add_sql_chain(sql_session_factory, chain_id="chain-1", agent_ids=["chain-auto"])
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="builtin-auto",
origin="position",
)
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="builtin-manual",
origin="user",
)
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="custom-auto",
origin="position",
)
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="chain-auto",
origin="position",
)
store = AgentRepository(sql_session_factory)
rows, total = await store.list_paginated(
user_id="user-a",
scope="mine",
search=None,
tag_ids=None,
page=1,
page_size=20,
)
assert total == 2
assert {row["id"] for row in rows} == {"builtin-manual", "custom-auto"}
assert await store.list_favorite_ids("user-a") == {"builtin-manual", "custom-auto"}
@pytest.mark.asyncio
async def test_manual_favorite_upgrades_existing_auto_builtin_favorite(sql_session_factory) -> None:
await _add_sql_agent(sql_session_factory, agent_id="builtin-agent", user_id=None)
await _add_sql_favorite(
sql_session_factory,
user_id="user-a",
agent_id="builtin-agent",
origin="position",
)
store = AgentRepository(sql_session_factory)
assert await store.list_favorite_ids("user-a") == set()
await store.add_favorite("user-a", "builtin-agent")
assert await _sql_favorite_rows(sql_session_factory, "user-a") == {
("builtin-agent", "user")
}
assert await store.list_favorite_ids("user-a") == {"builtin-agent"}
rows, total = await store.list_paginated(
user_id="user-a",
scope="mine",
search=None,
tag_ids=None,
page=1,
page_size=20,
)
assert total == 1
assert [row["id"] for row in rows] == ["builtin-agent"]
@pytest.mark.asyncio
async def test_agent_store_only_owner_can_update_or_delete() -> None:
store = MemoryAgentStore()
await store.ensure_builtin({"id": "builtin-agent", "name": "Built in"})
await store.create({"id": "private-a", "name": "Owned", "user_id": "user-a"})
@@ -10,7 +10,9 @@ from types import SimpleNamespace
import pytest
from deerflow.agents.ai_writing.nodes import researcher as R
from deerflow.agents.ai_writing.search import get_search_provider
from deerflow.agents.ai_writing.search.base import SearchResult
from deerflow.agents.ai_writing.search.intranet_stub import IntranetSearchProvider
class _FakeIntranet:
@@ -21,6 +23,7 @@ class _FakeIntranet:
self.raw_responses = []
self.calls = []
self.kw = kw
self.verify_ssl = kw.get("verify_ssl", False)
_FakeIntranet.instances.append(self)
async def search(self, query, max_results=5, **_):
@@ -55,12 +58,24 @@ class _SkillProviderShouldNotRun:
raise AssertionError("knowledge_base 模式不应调用通用 provider / 技能检索")
def _patch(monkeypatch, intranet_url="http://es/api", intranet_cls=_FakeIntranet):
class _FakeResponse:
status_code = 200
text = '[{"title":"内网标题","content":"内网正文","url":"https://intranet/doc"}]'
def raise_for_status(self) -> None:
return None
def json(self):
return [{"title": "内网标题", "content": "内网正文", "url": "https://intranet/doc"}]
def _patch(monkeypatch, intranet_url="http://es/api", intranet_cls=_FakeIntranet, intranet_verify_ssl=False):
_FakeIntranet.instances = []
cfg = {
"max_materials": 20,
"intranet_search_url": intranet_url,
"intranet_knowledge_base": "samples",
"intranet_verify_ssl": intranet_verify_ssl,
}
monkeypatch.setattr(R, "get_ai_writing_config", lambda: cfg)
monkeypatch.setattr(R, "use_builtin_agent", lambda *_: False)
@@ -125,6 +140,15 @@ async def test_knowledge_base_no_url_falls_back_to_default_endpoint(monkeypatch)
assert _FakeIntranet.instances[0].kw.get("base_url") == "", "空地址应原样下传,由 provider 回退默认端点"
@pytest.mark.asyncio
async def test_knowledge_base_passes_verify_ssl_config(monkeypatch):
_patch(monkeypatch, intranet_verify_ssl=True)
out = await R.researcher_node(_state(), {"configurable": {}})
assert _FakeIntranet.instances[0].kw.get("verify_ssl") is True
assert out["material_package"]["intranet_debug"]["verify_ssl"] is True
@pytest.mark.asyncio
async def test_knowledge_base_does_not_orchestrate_skills(monkeypatch):
"""knowledge_base 不编排技能:通用 provider 一旦被调用就会 raise。"""
@@ -163,3 +187,44 @@ async def test_knowledge_base_failure_is_visible(monkeypatch):
steps = pkg["skill_search_steps"]
assert any(step["type"] == "skill_error" for step in steps)
assert any(event["type"] == "notice" and event.get("raw") for event in out["progress_events"])
@pytest.mark.asyncio
async def test_intranet_search_disables_ssl_verification_for_internal_https(monkeypatch):
import httpx
seen: dict = {}
class _FakeAsyncClient:
def __init__(self, **kwargs):
seen.update(kwargs)
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return None
async def post(self, endpoint, json):
seen["endpoint"] = endpoint
seen["payload"] = json
return _FakeResponse()
monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient)
provider = IntranetSearchProvider(base_url="https://intranet.example/search", knowledge_base_name="kb", timeout=300)
results = await provider.search("测试", max_results=1)
assert results
assert seen["verify"] is False
assert seen["trust_env"] is False
assert seen["endpoint"] == "https://intranet.example/search"
assert seen["payload"]["knowledge_base_name"] == "kb"
assert provider.calls[0]["verify_ssl"] is False
def test_intranet_verify_ssl_can_be_reenabled_from_config():
provider = get_search_provider({"search_provider": "intranet", "intranet_verify_ssl": "true"})
assert isinstance(provider, IntranetSearchProvider)
assert provider.verify_ssl is True
+2 -1
View File
@@ -194,7 +194,8 @@ Step1 受众目标分析完成:当前已入库受众 X 个,本步骤新增
- 单个受众、账号、传播节点、脆弱点或策略失败,不中断整个流程。
- 保存接口必须返回 `id`,否则不会进入下游步骤。
- 受众目标默认入库 3 个,关键传播节点默认入库 3 个。
- 下游脆弱点/策略默认只选择前 2 个受众和前 2 个传播节点继续,可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- 下游受众脆弱点/策略只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
- 下游传播节点脆弱点/策略默认只选择前 2 个已入库传播节点继续,可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- 每个被选中的对象默认最多生成 3 个脆弱点、3 个策略。
- 模型流式 `answer` 会自动拼接。
- 流式模型接口第一次返回空响应、`y模型没有响应`、`模型没有响应` 或超时类字符串时,会自动再调用一次;仍失败才写入 `errors`。
@@ -86,7 +86,8 @@ python scripts/xdfx_flow.py --mode full --input xdfx-input.json --dry-run
- 受众目标默认入库最优 3 个,可用 `--max-audiences` 调整。
- 关键传播节点全局去重后默认入库最优 3 个,可用 `--max-pathways` 调整。
- 下游脆弱点/策略默认只从已入库结果中选择前 2 个受众、前 2 个传播节点继续,可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- 下游受众脆弱点/策略只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
- 下游传播节点脆弱点/策略默认从已入库传播节点中选择前 2 个继续;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- 每个被选中的受众/传播节点默认生成最多 3 个脆弱点,可用 `--max-vulns` 调整。
- 每个被选中的受众/传播节点默认生成最多 3 个策略,可用 `--max-strategies` 调整。
- 每个保存接口必须拿到返回 `id` 才进入下游。
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
+3 -2
View File
@@ -77,8 +77,9 @@ python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-
- Step1 的受众目标抽取结果里,`用户编号列表` / `编号id` / `id` 必须对应上一轮 `zhanghao` 账号数组的索引值;脚本会按这个索引依次找到账号并保存受众账号关系。
- Step2 成功后会写入 `topPathways`。
- Step1 默认入库最优 3 个受众;Step2 默认入库最优 3 个传播节点。
- Step3/Step4 默认只选择前 2 个已入库受众/传播节点继续,每个生成最多 3 个脆弱点;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- Step5/Step6 默认对上述选中的每个受众/传播节点生成最多 3 个策略。
- Step3/Step5 的受众下游分析只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
- Step4/Step6 的传播节点下游分析默认从已入库传播节点中选择前 2 个继续;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- Step3-Step6 对上述选中的每个受众/传播节点默认生成最多 3 个脆弱点或策略。
- 各步骤会同步前端 `task/task-action.step`;需要禁用时每条命令追加 `--no-step-status`。
- 策略步骤会同步写入 channel、narrate、message、策略与受众/传播节点关系表,保证原前端策略表格/编排页可继续读取。
- 策略步骤保存 `信息形式/模态` 时,必须使用脚本读取 `sys/sys-dict-data` 的 `modelType` 字典并把文字转成 value 逗号串,不能把“图片,视频”等文字直接写入 `task-action-message.modelType`。
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
+3 -2
View File
@@ -134,8 +134,9 @@ python scripts/xdfx_run_full_steps.py --input xdfx-input.json --state xdfx-flow-
- Step1 入库最优 3 个受众。
- Step2 入库最优 3 个传播节点。
- Step3/Step4 只从已入库结果中默认选择前 2 个受众/传播节点继续,每个生成最多 3 个脆弱点;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- Step5/Step6 对上述选中的每个受众/传播节点默认生成最多 3 个策略。
- Step3/Step5 的受众下游分析只能从“已在 Step2 分析出并关联关键传播节点”的受众中选择,默认取前 2 个;没有产生传播节点的受众不得继续分析脆弱点和策略。
- Step4/Step6 的传播节点下游分析默认从已入库传播节点中选择前 2 个继续;可用 `--downstream-audiences` / `--downstream-pathways` 调成 1 或 2。
- Step3-Step6 对上述选中的每个受众/传播节点默认生成最多 3 个脆弱点或策略。
- Step5/Step6 的策略模态必须按前端 `modelType` 字典保存为 value 逗号串;脚本会自动把“图片,视频/文本,图片”等文字转成 `task-action-message.modelType` 需要的值。
总控反馈用户时建议使用这种格式:
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None:
@@ -368,6 +368,7 @@ def default_state() -> dict[str, Any]:
"topAudiences": [],
"pathwayCandidates": [],
"topPathways": [],
"audiencesWithPathways": [],
"audienceVulns": {},
"pathwayVulns": {},
"audienceStrategies": [],
@@ -2055,6 +2056,88 @@ def selected_accounts_for_item(item: dict[str, Any], *, audience: bool) -> list[
return [item.get("account") or item]
def id_ref_key(value: Any) -> str:
number = as_int(value)
if number is not None:
return str(number)
return str(value or "").strip()
def audience_name_key(value: Any) -> str:
return re.sub(r"\s+", "", str(value or "")).lower()
def compact_pathway_ref(pathway: dict[str, Any]) -> dict[str, Any]:
return {
"pathwayAccountId": pathway.get("pathwayAccountId") or pathway.get("pathwayId"),
"accountId": pathway.get("accountId"),
"accountName": pathway.get("accountName"),
}
def add_unique_pathway(target: list[dict[str, Any]], pathway: dict[str, Any]) -> None:
key = id_ref_key(pathway.get("pathwayAccountId") or pathway.get("pathwayId") or pathway.get("accountId") or pathway.get("accountName"))
if not key:
return
seen = {
id_ref_key(item.get("pathwayAccountId") or item.get("pathwayId") or item.get("accountId") or item.get("accountName"))
for item in target
}
if key not in seen:
target.append(pathway)
def audience_pathway_pairs(state: dict[str, Any]) -> list[tuple[dict[str, Any], list[dict[str, Any]]]]:
"""Return only audiences that produced at least one saved key pathway."""
id_links: dict[str, list[dict[str, Any]]] = {}
name_links: dict[str, list[dict[str, Any]]] = {}
for pathway in state.get("topPathways") or []:
if not isinstance(pathway, dict):
continue
for design_id in pathway.get("designIds") or []:
key = id_ref_key(design_id)
if key:
id_links.setdefault(key, []).append(pathway)
for name in pathway.get("relatedAudiences") or []:
key = audience_name_key(name)
if key:
name_links.setdefault(key, []).append(pathway)
pairs: list[tuple[dict[str, Any], list[dict[str, Any]]]] = []
for audience_item in state.get("topAudiences") or []:
if not isinstance(audience_item, dict):
continue
linked: list[dict[str, Any]] = []
design_key = id_ref_key(audience_item.get("designId"))
for pathway in id_links.get(design_key, []):
add_unique_pathway(linked, pathway)
name_key = audience_name_key(audience_item.get("targetAudience"))
for pathway in name_links.get(name_key, []):
add_unique_pathway(linked, pathway)
if linked:
item = {**audience_item, "downstreamPathways": [compact_pathway_ref(pathway) for pathway in linked]}
pairs.append((item, linked))
return pairs
def audience_pathway_summary(state: dict[str, Any]) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for audience_item, pathways in audience_pathway_pairs(state):
summary.append(
{
"designId": audience_item.get("designId"),
"targetAudience": audience_item.get("targetAudience"),
"pathwayCount": len(pathways),
"pathways": [compact_pathway_ref(pathway) for pathway in pathways],
}
)
return summary
def has_audiences_with_pathways(state: dict[str, Any]) -> bool:
return bool(audience_pathway_pairs(state))
def downstream_limit(ctx: FlowContext, *, audience: bool) -> int:
"""Limit expensive downstream analysis while keeping 3 upstream rows stored."""
raw = ctx.downstream_audiences if audience else ctx.downstream_pathways
@@ -2066,22 +2149,35 @@ def downstream_items(ctx: FlowContext, state: dict[str, Any], stage: str, *, aud
source_key = "topAudiences" if audience else "topPathways"
id_key = "designId" if audience else "pathwayAccountId"
selected_key = "selectedDownstreamAudiences" if audience else "selectedDownstreamPathways"
items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
all_items = [item for item in (state.get(source_key) or []) if isinstance(item, dict)]
if audience:
state["audiencesWithPathways"] = audience_pathway_summary(state)
items = [item for item, _ in audience_pathway_pairs(state)]
if not items:
state[selected_key] = []
record_error(state, stage, "没有已分析出关键传播节点的受众,不能继续分析受众脆弱点或策略;请先重试关键传播节点分析")
return []
else:
items = all_items
limit = downstream_limit(ctx, audience=audience)
selected = items[:limit]
state[selected_key] = [
{
selected_rows: list[dict[str, Any]] = []
for item in selected:
row = {
"id": item.get(id_key),
"name": item.get("targetAudience") if audience else item.get("accountName"),
}
for item in selected
]
if items:
if audience:
row["pathwayCount"] = len(item.get("downstreamPathways") or [])
row["pathways"] = item.get("downstreamPathways") or []
selected_rows.append(row)
state[selected_key] = selected_rows
if selected:
record_event(
state,
stage,
f"下游分析选择 {len(selected)}/{len(items)} 个{'受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key]},
f"下游分析选择 {len(selected)}/{len(items)} 个{'已产生传播节点的受众' if audience else '传播节点'}",
{"limit": limit, "selected": state[selected_key], "candidateCount": len(items), "totalStoredCount": len(all_items)},
)
return selected
@@ -2183,6 +2279,7 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
record_error(state, stage, "没有已入库受众,无法分析传播节点")
return
if state.get("topPathways") and not ctx.dry_run:
state["audiencesWithPathways"] = audience_pathway_summary(state)
record_event(state, stage, "topPathways 已存在,跳过。如需重跑请清空 state 中该字段。")
return
all_candidates: list[dict[str, Any]] = []
@@ -2278,9 +2375,10 @@ def run_step2(ctx: FlowContext, state: dict[str, Any]) -> None:
http_post_json(ctx, state, "save_relation", relation_payload, stage + "/save-relation")
saved.append(item)
state["topPathways"] = saved
state["audiencesWithPathways"] = audience_pathway_summary(state)
if saved:
mark_step_status(ctx, state, ["step-analysis-keypoint", "step-analysis-weakpoint"], stage + "/step-status")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个")
record_event(state, stage, f"完成,保存传播节点 {len(saved)} 个,可继续下游分析的受众 {len(state.get('audiencesWithPathways') or [])} 个")
def run_step3(ctx: FlowContext, state: dict[str, Any]) -> None: