代码提交
This commit is contained in:
1 parent
b101ee7fc9
commit
1938c80061
42 files changed
+5552
-278
No files matched your search
@@ -17,7 +17,7 @@ SERPER_API_KEY=your-serper-api-key
|
||||
# AUTH_JWT_SECRET must stay stable across restarts, or existing logins expire.
|
||||
# GATEWAY_CORS_ORIGINS must include the exact frontend origins, separated by commas.
|
||||
AUTH_JWT_SECRET=i3_BiBK44sYHiFIkChhDs46y57ZewJKO3diQDw01anI
|
||||
GATEWAY_CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173,http://localhost:5174,http://127.0.0.1:5174,http://localhost:4200
|
||||
GATEWAY_CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173,http://localhost:5183,http://localhost:5174,http://127.0.0.1:5174,http://localhost:4200
|
||||
GATEWAY_CORS_ALLOW_ALL=0
|
||||
GATEWAY_ENABLE_DOCS=true
|
||||
GATEWAY_WORKERS=2
|
||||
|
||||
@@ -0,0 +1,760 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv export --no-hashes --no-emit-project --no-dev -o _reqs.txt
|
||||
-e ./packages/harness
|
||||
# via deer-flow
|
||||
agent-client-protocol==0.10.1
|
||||
# via deerflow-harness
|
||||
agent-sandbox==0.0.30
|
||||
# via deerflow-harness
|
||||
aiohappyeyeballs==2.6.2
|
||||
# via aiohttp
|
||||
aiohttp==3.14.1
|
||||
# via
|
||||
# aiohttp-retry
|
||||
# dingtalk-stream
|
||||
# firecrawl-py
|
||||
# hindsight-client
|
||||
# kubernetes
|
||||
# wecom-aibot-python-sdk
|
||||
aiohttp-retry==2.9.1
|
||||
# via hindsight-client
|
||||
aiosignal==1.4.0
|
||||
# via aiohttp
|
||||
aiosqlite==0.22.1
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langgraph-checkpoint-sqlite
|
||||
alembic==1.18.4
|
||||
# via deerflow-harness
|
||||
annotated-doc==0.0.4
|
||||
# via fastapi
|
||||
annotated-types==0.7.0
|
||||
# via pydantic
|
||||
anthropic==0.107.1
|
||||
# via langchain-anthropic
|
||||
anyio==4.13.0
|
||||
# via
|
||||
# anthropic
|
||||
# google-genai
|
||||
# httpx
|
||||
# mcp
|
||||
# openai
|
||||
# sse-starlette
|
||||
# starlette
|
||||
# watchfiles
|
||||
asyncmy==0.2.11
|
||||
# via deer-flow
|
||||
attrs==26.1.0
|
||||
# via
|
||||
# aiohttp
|
||||
# jsonschema
|
||||
# referencing
|
||||
audioop-lts==0.2.2 ; python_full_version >= '3.13'
|
||||
# via
|
||||
# speechrecognition
|
||||
# standard-aifc
|
||||
azure-ai-documentintelligence==1.0.2
|
||||
# via markitdown
|
||||
azure-core==1.41.0
|
||||
# via
|
||||
# azure-ai-documentintelligence
|
||||
# azure-identity
|
||||
azure-identity==1.25.3
|
||||
# via markitdown
|
||||
backoff==2.2.1
|
||||
# via langfuse
|
||||
bcrypt==5.0.0
|
||||
# via deer-flow
|
||||
beautifulsoup4==4.15.0
|
||||
# via
|
||||
# markdownify
|
||||
# markitdown
|
||||
# readabilipy
|
||||
blockbuster==1.5.26
|
||||
# via langgraph-runtime-inmem
|
||||
brotli==1.2.0 ; platform_python_implementation == 'CPython'
|
||||
# via httpx
|
||||
brotlicffi==1.2.0.1 ; platform_python_implementation != 'CPython'
|
||||
# via httpx
|
||||
certifi==2026.5.20
|
||||
# via
|
||||
# httpcore
|
||||
# httpx
|
||||
# kubernetes
|
||||
# requests
|
||||
# volcengine-python-sdk
|
||||
# wecom-aibot-python-sdk
|
||||
cffi==2.0.0
|
||||
# via
|
||||
# brotlicffi
|
||||
# cryptography
|
||||
charset-normalizer==3.4.7
|
||||
# via
|
||||
# markitdown
|
||||
# pdfminer-six
|
||||
# requests
|
||||
click==8.4.1
|
||||
# via
|
||||
# ddgs
|
||||
# langgraph-cli
|
||||
# magika
|
||||
# uvicorn
|
||||
cloudpickle==3.1.2
|
||||
# via langgraph-api
|
||||
cobble==0.1.4
|
||||
# via mammoth
|
||||
colorama==0.4.6 ; sys_platform == 'win32'
|
||||
# via
|
||||
# click
|
||||
# tqdm
|
||||
# uvicorn
|
||||
coloredlogs==15.0.1
|
||||
# via onnxruntime
|
||||
croniter==6.2.2
|
||||
# via
|
||||
# deer-flow
|
||||
# langgraph-runtime-inmem
|
||||
cryptography==48.0.0
|
||||
# via
|
||||
# azure-identity
|
||||
# google-auth
|
||||
# langgraph-api
|
||||
# msal
|
||||
# pdfminer-six
|
||||
# pyjwt
|
||||
# wecom-aibot-python-sdk
|
||||
ddgs==9.14.4
|
||||
# via deerflow-harness
|
||||
defusedxml==0.7.1
|
||||
# via
|
||||
# markitdown
|
||||
# youtube-transcript-api
|
||||
dingtalk-stream==0.24.3
|
||||
# via deer-flow
|
||||
distro==1.9.0
|
||||
# via
|
||||
# anthropic
|
||||
# google-genai
|
||||
# openai
|
||||
dnspython==2.8.0
|
||||
# via email-validator
|
||||
docstring-parser==0.18.0
|
||||
# via anthropic
|
||||
dotenv==0.9.9
|
||||
# via deerflow-harness
|
||||
duckdb==1.5.3
|
||||
# via deerflow-harness
|
||||
durationpy==0.10
|
||||
# via kubernetes
|
||||
email-validator==2.3.0
|
||||
# via deer-flow
|
||||
et-xmlfile==2.0.0
|
||||
# via openpyxl
|
||||
exa-py==2.13.0
|
||||
# via deerflow-harness
|
||||
fake-useragent==2.2.0
|
||||
# via ddgs
|
||||
fastapi==0.136.3
|
||||
# via deer-flow
|
||||
filetype==1.2.0
|
||||
# via langchain-google-genai
|
||||
firecrawl-py==4.28.2
|
||||
# via deerflow-harness
|
||||
flatbuffers==25.12.19
|
||||
# via onnxruntime
|
||||
forbiddenfruit==0.1.4 ; implementation_name == 'cpython'
|
||||
# via blockbuster
|
||||
frozenlist==1.8.0
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
google-auth==2.53.0
|
||||
# via google-genai
|
||||
google-genai==2.8.0
|
||||
# via langchain-google-genai
|
||||
googleapis-common-protos==1.75.0
|
||||
# via opentelemetry-exporter-otlp-proto-http
|
||||
greenlet==3.5.1
|
||||
# via sqlalchemy
|
||||
grpcio==1.80.0
|
||||
# via
|
||||
# grpcio-health-checking
|
||||
# grpcio-tools
|
||||
# langgraph-api
|
||||
grpcio-health-checking==1.80.0
|
||||
# via langgraph-api
|
||||
grpcio-tools==1.80.0
|
||||
# via langgraph-api
|
||||
h11==0.16.0
|
||||
# via
|
||||
# httpcore
|
||||
# uvicorn
|
||||
h2==4.3.0
|
||||
# via httpx
|
||||
hindsight-client==0.7.2
|
||||
# via deer-flow
|
||||
hpack==4.1.0
|
||||
# via h2
|
||||
html5lib==1.1
|
||||
# via readabilipy
|
||||
httpcore==1.0.9
|
||||
# via
|
||||
# exa-py
|
||||
# httpx
|
||||
# python-telegram-bot
|
||||
httptools==0.8.0
|
||||
# via
|
||||
# langgraph-api
|
||||
# uvicorn
|
||||
httpx==0.28.1
|
||||
# via
|
||||
# agent-sandbox
|
||||
# anthropic
|
||||
# ddgs
|
||||
# deer-flow
|
||||
# deerflow-harness
|
||||
# exa-py
|
||||
# firecrawl-py
|
||||
# google-genai
|
||||
# langfuse
|
||||
# langgraph-api
|
||||
# langgraph-cli
|
||||
# langgraph-sdk
|
||||
# langsmith
|
||||
# lark-oapi
|
||||
# mcp
|
||||
# openai
|
||||
# python-telegram-bot
|
||||
# tavily-python
|
||||
httpx-sse==0.4.3
|
||||
# via mcp
|
||||
humanfriendly==10.0
|
||||
# via coloredlogs
|
||||
hyperframe==6.1.0
|
||||
# via h2
|
||||
idna==3.18
|
||||
# via
|
||||
# anyio
|
||||
# email-validator
|
||||
# httpx
|
||||
# requests
|
||||
# yarl
|
||||
isodate==0.7.2
|
||||
# via azure-ai-documentintelligence
|
||||
jiter==0.15.0
|
||||
# via
|
||||
# anthropic
|
||||
# openai
|
||||
jsonpatch==1.33
|
||||
# via langchain-core
|
||||
jsonpointer==3.1.1
|
||||
# via jsonpatch
|
||||
jsonschema==4.26.0
|
||||
# via mcp
|
||||
jsonschema-rs==0.44.1
|
||||
# via langgraph-api
|
||||
jsonschema-specifications==2025.9.1
|
||||
# via jsonschema
|
||||
kubernetes==36.0.2
|
||||
# via deerflow-harness
|
||||
langchain==1.3.4
|
||||
# via deerflow-harness
|
||||
langchain-anthropic==1.4.4
|
||||
# via deerflow-harness
|
||||
langchain-core==1.4.1
|
||||
# via
|
||||
# langchain
|
||||
# langchain-anthropic
|
||||
# langchain-deepseek
|
||||
# langchain-google-genai
|
||||
# langchain-mcp-adapters
|
||||
# langchain-openai
|
||||
# langgraph
|
||||
# langgraph-api
|
||||
# langgraph-checkpoint
|
||||
# langgraph-prebuilt
|
||||
# langgraph-sdk
|
||||
langchain-deepseek==1.1.0
|
||||
# via deerflow-harness
|
||||
langchain-google-genai==4.2.4
|
||||
# via deerflow-harness
|
||||
langchain-mcp-adapters==0.2.2
|
||||
# via deerflow-harness
|
||||
langchain-openai==1.2.2
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langchain-deepseek
|
||||
langchain-protocol==0.0.16
|
||||
# via
|
||||
# langchain-core
|
||||
# langgraph-api
|
||||
# langgraph-sdk
|
||||
langfuse==4.7.1
|
||||
# via deerflow-harness
|
||||
langgraph==1.2.4
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langchain
|
||||
# langgraph-api
|
||||
# langgraph-runtime-inmem
|
||||
langgraph-api==0.9.0
|
||||
# via deerflow-harness
|
||||
langgraph-checkpoint==4.1.1
|
||||
# via
|
||||
# langgraph
|
||||
# langgraph-api
|
||||
# langgraph-checkpoint-postgres
|
||||
# langgraph-checkpoint-sqlite
|
||||
# langgraph-prebuilt
|
||||
# langgraph-runtime-inmem
|
||||
langgraph-checkpoint-postgres==3.1.0
|
||||
# via deerflow-harness
|
||||
langgraph-checkpoint-sqlite==3.1.0
|
||||
# via deerflow-harness
|
||||
langgraph-cli==0.4.27
|
||||
# via deerflow-harness
|
||||
langgraph-prebuilt==1.1.0
|
||||
# via langgraph
|
||||
langgraph-runtime-inmem==0.29.0
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langgraph-api
|
||||
langgraph-sdk==0.4.2
|
||||
# via
|
||||
# deer-flow
|
||||
# deerflow-harness
|
||||
# langgraph
|
||||
# langgraph-api
|
||||
# langgraph-cli
|
||||
langsmith==0.8.9
|
||||
# via
|
||||
# langchain-core
|
||||
# langgraph-api
|
||||
lark-oapi==1.6.8
|
||||
# via deer-flow
|
||||
lxml==6.1.1
|
||||
# via
|
||||
# ddgs
|
||||
# markitdown
|
||||
# python-pptx
|
||||
# readabilipy
|
||||
magika==0.6.3
|
||||
# via markitdown
|
||||
mako==1.3.12
|
||||
# via alembic
|
||||
mammoth==1.11.0
|
||||
# via markitdown
|
||||
markdown-to-mrkdwn==0.3.3
|
||||
# via deer-flow
|
||||
markdownify==1.2.2
|
||||
# via
|
||||
# deerflow-harness
|
||||
# markitdown
|
||||
markitdown==0.1.5
|
||||
# via deerflow-harness
|
||||
markupsafe==3.0.3
|
||||
# via mako
|
||||
mcp==1.27.2
|
||||
# via langchain-mcp-adapters
|
||||
mpmath==1.3.0
|
||||
# via sympy
|
||||
msal==1.37.0
|
||||
# via
|
||||
# azure-identity
|
||||
# msal-extensions
|
||||
msal-extensions==1.3.1
|
||||
# via azure-identity
|
||||
multidict==6.7.1
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
nest-asyncio==1.6.0
|
||||
# via firecrawl-py
|
||||
numpy==2.4.6
|
||||
# via
|
||||
# magika
|
||||
# onnxruntime
|
||||
# pandas
|
||||
oauthlib==3.3.1
|
||||
# via requests-oauthlib
|
||||
olefile==0.47
|
||||
# via markitdown
|
||||
onnxruntime==1.20.1
|
||||
# via magika
|
||||
openai==2.41.0
|
||||
# via
|
||||
# exa-py
|
||||
# langchain-openai
|
||||
openpyxl==3.1.5
|
||||
# via markitdown
|
||||
opentelemetry-api==1.42.1
|
||||
# via
|
||||
# langfuse
|
||||
# langgraph-api
|
||||
# langsmith
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
# opentelemetry-sdk
|
||||
# opentelemetry-semantic-conventions
|
||||
opentelemetry-exporter-otlp-proto-common==1.42.1
|
||||
# via opentelemetry-exporter-otlp-proto-http
|
||||
opentelemetry-exporter-otlp-proto-http==1.42.1
|
||||
# via
|
||||
# langfuse
|
||||
# langgraph-api
|
||||
# langsmith
|
||||
opentelemetry-proto==1.42.1
|
||||
# via
|
||||
# opentelemetry-exporter-otlp-proto-common
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
opentelemetry-sdk==1.42.1
|
||||
# via
|
||||
# langfuse
|
||||
# langgraph-api
|
||||
# langsmith
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
opentelemetry-semantic-conventions==0.63b1
|
||||
# via opentelemetry-sdk
|
||||
orjson==3.11.9
|
||||
# via
|
||||
# langgraph-api
|
||||
# langgraph-checkpoint-postgres
|
||||
# langgraph-sdk
|
||||
# langsmith
|
||||
ormsgpack==1.12.2
|
||||
# via langgraph-checkpoint
|
||||
packaging==26.2
|
||||
# via
|
||||
# langchain-core
|
||||
# langfuse
|
||||
# langsmith
|
||||
# onnxruntime
|
||||
pandas==3.0.3
|
||||
# via markitdown
|
||||
pathspec==1.1.1
|
||||
# via langgraph-cli
|
||||
pdfminer-six==20251230
|
||||
# via
|
||||
# markitdown
|
||||
# pdfplumber
|
||||
pdfplumber==0.11.9
|
||||
# via markitdown
|
||||
pillow==12.2.0
|
||||
# via
|
||||
# pdfplumber
|
||||
# python-pptx
|
||||
primp==1.3.1
|
||||
# via ddgs
|
||||
propcache==0.5.2
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
protobuf==6.33.6
|
||||
# via
|
||||
# googleapis-common-protos
|
||||
# grpcio-health-checking
|
||||
# grpcio-tools
|
||||
# langgraph-api
|
||||
# onnxruntime
|
||||
# opentelemetry-proto
|
||||
psycopg==3.3.4
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langgraph-checkpoint-postgres
|
||||
psycopg-binary==3.3.4 ; implementation_name != 'pypy'
|
||||
# via psycopg
|
||||
psycopg-pool==3.3.1
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langgraph-checkpoint-postgres
|
||||
pyasn1==0.6.3
|
||||
# via pyasn1-modules
|
||||
pyasn1-modules==0.4.2
|
||||
# via google-auth
|
||||
pycparser==3.0 ; implementation_name != 'PyPy'
|
||||
# via cffi
|
||||
pycryptodome==3.23.0
|
||||
# via lark-oapi
|
||||
pydantic==2.13.4
|
||||
# via
|
||||
# agent-client-protocol
|
||||
# agent-sandbox
|
||||
# anthropic
|
||||
# deerflow-harness
|
||||
# exa-py
|
||||
# fastapi
|
||||
# firecrawl-py
|
||||
# google-genai
|
||||
# hindsight-client
|
||||
# langchain
|
||||
# langchain-anthropic
|
||||
# langchain-core
|
||||
# langchain-google-genai
|
||||
# langfuse
|
||||
# langgraph
|
||||
# langsmith
|
||||
# mcp
|
||||
# openai
|
||||
# pydantic-settings
|
||||
pydantic-core==2.46.4
|
||||
# via pydantic
|
||||
pydantic-settings==2.14.1
|
||||
# via mcp
|
||||
pydub==0.25.1
|
||||
# via markitdown
|
||||
pyee==13.0.1
|
||||
# via wecom-aibot-python-sdk
|
||||
pyjwt==2.13.0
|
||||
# via
|
||||
# deer-flow
|
||||
# langgraph-api
|
||||
# mcp
|
||||
# msal
|
||||
pypdfium2==5.9.0
|
||||
# via pdfplumber
|
||||
pyreadline3==3.5.6 ; sys_platform == 'win32'
|
||||
# via humanfriendly
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# croniter
|
||||
# hindsight-client
|
||||
# kubernetes
|
||||
# pandas
|
||||
# volcengine-python-sdk
|
||||
python-dotenv==1.2.2
|
||||
# via
|
||||
# dotenv
|
||||
# exa-py
|
||||
# firecrawl-py
|
||||
# langgraph-cli
|
||||
# magika
|
||||
# pydantic-settings
|
||||
# uvicorn
|
||||
python-multipart==0.0.32
|
||||
# via
|
||||
# deer-flow
|
||||
# mcp
|
||||
python-pptx==1.0.2
|
||||
# via markitdown
|
||||
python-telegram-bot==22.7
|
||||
# via deer-flow
|
||||
pywin32==312 ; sys_platform == 'win32'
|
||||
# via mcp
|
||||
pyyaml==6.0.3
|
||||
# via
|
||||
# deerflow-harness
|
||||
# kubernetes
|
||||
# langchain-core
|
||||
# uvicorn
|
||||
readabilipy==0.3.0
|
||||
# via deerflow-harness
|
||||
referencing==0.37.0
|
||||
# via
|
||||
# jsonschema
|
||||
# jsonschema-specifications
|
||||
regex==2026.5.9
|
||||
# via
|
||||
# readabilipy
|
||||
# tiktoken
|
||||
requests==2.34.2
|
||||
# via
|
||||
# azure-core
|
||||
# dingtalk-stream
|
||||
# exa-py
|
||||
# firecrawl-py
|
||||
# google-auth
|
||||
# google-genai
|
||||
# kubernetes
|
||||
# langsmith
|
||||
# lark-oapi
|
||||
# markitdown
|
||||
# msal
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
# requests-oauthlib
|
||||
# requests-toolbelt
|
||||
# tavily-python
|
||||
# tiktoken
|
||||
# youtube-transcript-api
|
||||
requests-oauthlib==2.0.0
|
||||
# via kubernetes
|
||||
requests-toolbelt==1.0.0
|
||||
# via
|
||||
# langsmith
|
||||
# lark-oapi
|
||||
rpds-py==2026.5.1
|
||||
# via
|
||||
# jsonschema
|
||||
# referencing
|
||||
setuptools==82.0.1
|
||||
# via grpcio-tools
|
||||
six==1.17.0
|
||||
# via
|
||||
# html5lib
|
||||
# kubernetes
|
||||
# markdownify
|
||||
# python-dateutil
|
||||
# volcengine-python-sdk
|
||||
slack-sdk==3.42.0
|
||||
# via deer-flow
|
||||
sniffio==1.3.1
|
||||
# via
|
||||
# anthropic
|
||||
# google-genai
|
||||
# openai
|
||||
socksio==1.0.0
|
||||
# via httpx
|
||||
soupsieve==2.8.4
|
||||
# via beautifulsoup4
|
||||
speechrecognition==3.16.1
|
||||
# via markitdown
|
||||
sqlalchemy==2.0.50
|
||||
# via
|
||||
# alembic
|
||||
# deerflow-harness
|
||||
sqlite-vec==0.1.9
|
||||
# via langgraph-checkpoint-sqlite
|
||||
sse-starlette==3.3.4
|
||||
# via
|
||||
# deer-flow
|
||||
# langgraph-api
|
||||
# langgraph-runtime-inmem
|
||||
# mcp
|
||||
standard-aifc==3.13.0 ; python_full_version >= '3.13'
|
||||
# via speechrecognition
|
||||
standard-chunk==3.13.0 ; python_full_version >= '3.13'
|
||||
# via standard-aifc
|
||||
starlette==1.2.1
|
||||
# via
|
||||
# fastapi
|
||||
# langgraph-api
|
||||
# langgraph-runtime-inmem
|
||||
# mcp
|
||||
# sse-starlette
|
||||
structlog==25.5.0
|
||||
# via
|
||||
# langgraph-api
|
||||
# langgraph-runtime-inmem
|
||||
sympy==1.14.0
|
||||
# via onnxruntime
|
||||
tavily-python==0.7.25
|
||||
# via deerflow-harness
|
||||
tenacity==9.1.4
|
||||
# via
|
||||
# google-genai
|
||||
# langchain-core
|
||||
# langgraph-api
|
||||
tiktoken==0.13.0
|
||||
# via
|
||||
# deerflow-harness
|
||||
# langchain-openai
|
||||
# tavily-python
|
||||
tqdm==4.68.1
|
||||
# via openai
|
||||
truststore==0.10.4
|
||||
# via langgraph-api
|
||||
typing-extensions==4.15.0
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
# alembic
|
||||
# anthropic
|
||||
# anyio
|
||||
# azure-ai-documentintelligence
|
||||
# azure-core
|
||||
# azure-identity
|
||||
# beautifulsoup4
|
||||
# exa-py
|
||||
# fastapi
|
||||
# google-genai
|
||||
# grpcio
|
||||
# hindsight-client
|
||||
# langchain-core
|
||||
# langchain-mcp-adapters
|
||||
# langchain-protocol
|
||||
# mcp
|
||||
# openai
|
||||
# opentelemetry-api
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
# opentelemetry-sdk
|
||||
# opentelemetry-semantic-conventions
|
||||
# psycopg
|
||||
# psycopg-pool
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# pyee
|
||||
# python-pptx
|
||||
# referencing
|
||||
# speechrecognition
|
||||
# sqlalchemy
|
||||
# starlette
|
||||
# typing-inspection
|
||||
typing-inspection==0.4.2
|
||||
# via
|
||||
# fastapi
|
||||
# mcp
|
||||
# pydantic
|
||||
# pydantic-settings
|
||||
tzdata==2026.2 ; sys_platform == 'emscripten' or sys_platform == 'win32'
|
||||
# via
|
||||
# pandas
|
||||
# psycopg
|
||||
urllib3==2.7.0
|
||||
# via
|
||||
# hindsight-client
|
||||
# kubernetes
|
||||
# requests
|
||||
# volcengine-python-sdk
|
||||
uuid-utils==0.16.0
|
||||
# via
|
||||
# langchain-core
|
||||
# langgraph-api
|
||||
# langsmith
|
||||
uvicorn==0.49.0
|
||||
# via
|
||||
# deer-flow
|
||||
# langgraph-api
|
||||
# mcp
|
||||
uvloop==0.22.1 ; sys_platform != 'win32'
|
||||
# via
|
||||
# langgraph-api
|
||||
# uvicorn
|
||||
volcengine-python-sdk==5.0.33
|
||||
# via agent-sandbox
|
||||
watchfiles==1.2.0
|
||||
# via
|
||||
# langgraph-api
|
||||
# uvicorn
|
||||
webencodings==0.5.1
|
||||
# via html5lib
|
||||
websocket-client==1.9.0
|
||||
# via kubernetes
|
||||
websockets==15.0.1
|
||||
# via
|
||||
# dingtalk-stream
|
||||
# firecrawl-py
|
||||
# google-genai
|
||||
# langgraph-api
|
||||
# langgraph-sdk
|
||||
# langsmith
|
||||
# lark-oapi
|
||||
# uvicorn
|
||||
# wecom-aibot-python-sdk
|
||||
wecom-aibot-python-sdk==1.0.2
|
||||
# via deer-flow
|
||||
wrapt==1.17.3
|
||||
# via langfuse
|
||||
xlrd==2.0.2
|
||||
# via markitdown
|
||||
xlsxwriter==3.2.9
|
||||
# via python-pptx
|
||||
xxhash==3.7.0
|
||||
# via
|
||||
# langgraph
|
||||
# langsmith
|
||||
yarl==1.24.2
|
||||
# via aiohttp
|
||||
youtube-transcript-api==1.0.3
|
||||
# via markitdown
|
||||
zstandard==0.25.0
|
||||
# via
|
||||
# langgraph-api
|
||||
# langsmith
|
||||
@@ -30,9 +30,9 @@ from app.gateway.routers import (
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
feedback,
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
feedback,
|
||||
html_page_favorites,
|
||||
intent,
|
||||
llm_metrics,
|
||||
@@ -41,6 +41,8 @@ from app.gateway.routers import (
|
||||
models,
|
||||
multi_agent,
|
||||
notifications,
|
||||
parallel_agent_tasks,
|
||||
parallel_agents,
|
||||
public_embed,
|
||||
public_scheduled_tasks,
|
||||
recommend,
|
||||
@@ -378,7 +380,7 @@ def _release_singleton_lock(handle) -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _curator_scheduler_loop() -> None:
|
||||
async def _curator_scheduler_loop() -> None:
|
||||
"""常驻后台任务:每隔 _CURATOR_CHECK_INTERVAL_SECONDS 检查一次需不需要进化。
|
||||
|
||||
多 worker 部署下用文件锁做 leader 选举 —— 只有抢到锁的那个 worker 进程真正
|
||||
@@ -404,248 +406,248 @@ async def _curator_scheduler_loop() -> None:
|
||||
except Exception:
|
||||
logger.exception("Curator scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
return raw.strip().lower() not in {"0", "false", "no", "off"}
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, *, alias: str | None = None, minimum: int = 1) -> int:
|
||||
raw = os.getenv(name)
|
||||
if raw is None and alias:
|
||||
raw = os.getenv(alias)
|
||||
if raw is None:
|
||||
raw = str(default)
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
def _env_float(name: str, default: float, *, alias: str | None = None, minimum: float = 0.0) -> float:
|
||||
raw = os.getenv(name)
|
||||
if raw is None and alias:
|
||||
raw = os.getenv(alias)
|
||||
if raw is None:
|
||||
raw = str(default)
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
_LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED = _env_bool("ADMIN_LEADERBOARD_BACKGROUND_ENABLED", True)
|
||||
_LEADERBOARD_SNAPSHOT_PREWARM_DAYS = _env_int(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_PREWARM_DAYS",
|
||||
30,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_BATCH_SIZE = _env_int(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_BATCH_SIZE",
|
||||
1,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS",
|
||||
300.0,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_SLEEP_SECONDS",
|
||||
2.0,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS = _env_int(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_RUNNING_TTL_SECONDS",
|
||||
1800,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
|
||||
60.0,
|
||||
)
|
||||
|
||||
|
||||
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
today = datetime.now(BEIJING_TZ).date()
|
||||
start = today - timedelta(days=max(1, days) - 1)
|
||||
return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
|
||||
|
||||
|
||||
async def _leaderboard_snapshot_due_dates(
|
||||
session_factory,
|
||||
*,
|
||||
settings_hash: str,
|
||||
limit: int,
|
||||
running_ttl_seconds: int,
|
||||
) -> list[str]:
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
|
||||
async with session_factory() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AdminLeaderboardDailyStatRow.stat_date)
|
||||
.where(
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
or_(
|
||||
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
|
||||
and_(
|
||||
AdminLeaderboardDailyStatRow.status == "running",
|
||||
AdminLeaderboardDailyStatRow.updated_at < cutoff,
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
).all()
|
||||
return [str(row[0]) for row in rows]
|
||||
|
||||
|
||||
async def _claim_leaderboard_snapshot(
|
||||
session_factory,
|
||||
*,
|
||||
stat_date: str,
|
||||
settings_hash: str,
|
||||
running_ttl_seconds: int,
|
||||
) -> bool:
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.gateway.routers.admin_users import _snapshot_is_fresh
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
async with session_factory() as session:
|
||||
row = await session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": stat_date, "settings_hash": settings_hash},
|
||||
)
|
||||
if row is None or _snapshot_is_fresh(row, stat_date):
|
||||
return False
|
||||
if row.status == "running":
|
||||
updated_at = row.updated_at
|
||||
if isinstance(updated_at, datetime):
|
||||
if updated_at.tzinfo is None:
|
||||
updated_at = updated_at.replace(tzinfo=BEIJING_TZ)
|
||||
age = (datetime.now(BEIJING_TZ) - updated_at.astimezone(BEIJING_TZ)).total_seconds()
|
||||
if age < running_ttl_seconds:
|
||||
return False
|
||||
row.status = "running"
|
||||
row.message = "排行榜后台统计任务已认领。"
|
||||
row.error = None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
|
||||
from app.gateway.routers.admin_users import (
|
||||
_daily_job_key,
|
||||
_queue_daily_leaderboard_snapshots,
|
||||
_refresh_daily_leaderboard_snapshot,
|
||||
_resolve_leaderboard_snapshot_scope,
|
||||
)
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
return 0, 0
|
||||
|
||||
settings = load_system_settings().leaderboard
|
||||
excluded, include_scheduled, include_admins, include_failed, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
|
||||
stat_dates = _recent_leaderboard_stat_dates(_LEADERBOARD_SNAPSHOT_PREWARM_DAYS)
|
||||
queued_dates = await _queue_daily_leaderboard_snapshots(
|
||||
session_factory,
|
||||
stat_dates,
|
||||
settings_hash,
|
||||
message="排行榜后台统计任务定期预热/补算队列。",
|
||||
)
|
||||
|
||||
due_dates = await _leaderboard_snapshot_due_dates(
|
||||
session_factory,
|
||||
settings_hash=settings_hash,
|
||||
limit=_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
|
||||
running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
|
||||
)
|
||||
processed = 0
|
||||
for stat_date in due_dates:
|
||||
claimed = await _claim_leaderboard_snapshot(
|
||||
session_factory,
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
|
||||
)
|
||||
if not claimed:
|
||||
continue
|
||||
logger.info("Leaderboard snapshot scheduler: building date=%s hash=%s", stat_date, settings_hash[:10])
|
||||
await _refresh_daily_leaderboard_snapshot(
|
||||
_daily_job_key(stat_date, settings_hash),
|
||||
session_factory=session_factory,
|
||||
settings=settings,
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
excluded=excluded,
|
||||
include_scheduled=include_scheduled,
|
||||
include_admins=include_admins,
|
||||
include_failed=include_failed,
|
||||
)
|
||||
processed += 1
|
||||
if _LEADERBOARD_SNAPSHOT_SLEEP_SECONDS > 0 and processed < len(due_dates):
|
||||
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS)
|
||||
return processed, len(queued_dates)
|
||||
|
||||
|
||||
async def _leaderboard_snapshot_scheduler_loop() -> None:
|
||||
"""Background task inside the existing gateway process for daily snapshots."""
|
||||
if not _LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED:
|
||||
logger.info("Leaderboard snapshot scheduler disabled by ADMIN_LEADERBOARD_BACKGROUND_ENABLED")
|
||||
return
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".leaderboard_snapshot_scheduler.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Leaderboard snapshot scheduler: another gateway worker holds the lock; this worker skips")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Leaderboard snapshot scheduler started: prewarm_days=%d batch_size=%d interval=%.1fs",
|
||||
_LEADERBOARD_SNAPSHOT_PREWARM_DAYS,
|
||||
_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
|
||||
_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS,
|
||||
)
|
||||
try:
|
||||
if _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS > 0:
|
||||
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
processed, queued = await _run_leaderboard_snapshot_tick()
|
||||
if processed or queued:
|
||||
logger.info(
|
||||
"Leaderboard snapshot scheduler tick: queued=%d processed=%d",
|
||||
queued,
|
||||
processed,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS)
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
return raw.strip().lower() not in {"0", "false", "no", "off"}
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, *, alias: str | None = None, minimum: int = 1) -> int:
|
||||
raw = os.getenv(name)
|
||||
if raw is None and alias:
|
||||
raw = os.getenv(alias)
|
||||
if raw is None:
|
||||
raw = str(default)
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
def _env_float(name: str, default: float, *, alias: str | None = None, minimum: float = 0.0) -> float:
|
||||
raw = os.getenv(name)
|
||||
if raw is None and alias:
|
||||
raw = os.getenv(alias)
|
||||
if raw is None:
|
||||
raw = str(default)
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
_LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED = _env_bool("ADMIN_LEADERBOARD_BACKGROUND_ENABLED", True)
|
||||
_LEADERBOARD_SNAPSHOT_PREWARM_DAYS = _env_int(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_PREWARM_DAYS",
|
||||
30,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_BATCH_SIZE = _env_int(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_BATCH_SIZE",
|
||||
1,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS",
|
||||
300.0,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_SLEEP_SECONDS",
|
||||
2.0,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS = _env_int(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_RUNNING_TTL_SECONDS",
|
||||
1800,
|
||||
)
|
||||
_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
|
||||
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
|
||||
60.0,
|
||||
)
|
||||
|
||||
|
||||
def _recent_leaderboard_stat_dates(days: int) -> list[str]:
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
today = datetime.now(BEIJING_TZ).date()
|
||||
start = today - timedelta(days=max(1, days) - 1)
|
||||
return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
|
||||
|
||||
|
||||
async def _leaderboard_snapshot_due_dates(
|
||||
session_factory,
|
||||
*,
|
||||
settings_hash: str,
|
||||
limit: int,
|
||||
running_ttl_seconds: int,
|
||||
) -> list[str]:
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
|
||||
async with session_factory() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AdminLeaderboardDailyStatRow.stat_date)
|
||||
.where(
|
||||
AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
|
||||
or_(
|
||||
AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
|
||||
and_(
|
||||
AdminLeaderboardDailyStatRow.status == "running",
|
||||
AdminLeaderboardDailyStatRow.updated_at < cutoff,
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
).all()
|
||||
return [str(row[0]) for row in rows]
|
||||
|
||||
|
||||
async def _claim_leaderboard_snapshot(
|
||||
session_factory,
|
||||
*,
|
||||
stat_date: str,
|
||||
settings_hash: str,
|
||||
running_ttl_seconds: int,
|
||||
) -> bool:
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.gateway.routers.admin_users import _snapshot_is_fresh
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
async with session_factory() as session:
|
||||
row = await session.get(
|
||||
AdminLeaderboardDailyStatRow,
|
||||
{"stat_date": stat_date, "settings_hash": settings_hash},
|
||||
)
|
||||
if row is None or _snapshot_is_fresh(row, stat_date):
|
||||
return False
|
||||
if row.status == "running":
|
||||
updated_at = row.updated_at
|
||||
if isinstance(updated_at, datetime):
|
||||
if updated_at.tzinfo is None:
|
||||
updated_at = updated_at.replace(tzinfo=BEIJING_TZ)
|
||||
age = (datetime.now(BEIJING_TZ) - updated_at.astimezone(BEIJING_TZ)).total_seconds()
|
||||
if age < running_ttl_seconds:
|
||||
return False
|
||||
row.status = "running"
|
||||
row.message = "排行榜后台统计任务已认领。"
|
||||
row.error = None
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
|
||||
from app.gateway.routers.admin_users import (
|
||||
_daily_job_key,
|
||||
_queue_daily_leaderboard_snapshots,
|
||||
_refresh_daily_leaderboard_snapshot,
|
||||
_resolve_leaderboard_snapshot_scope,
|
||||
)
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
return 0, 0
|
||||
|
||||
settings = load_system_settings().leaderboard
|
||||
excluded, include_scheduled, include_admins, include_failed, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
|
||||
stat_dates = _recent_leaderboard_stat_dates(_LEADERBOARD_SNAPSHOT_PREWARM_DAYS)
|
||||
queued_dates = await _queue_daily_leaderboard_snapshots(
|
||||
session_factory,
|
||||
stat_dates,
|
||||
settings_hash,
|
||||
message="排行榜后台统计任务定期预热/补算队列。",
|
||||
)
|
||||
|
||||
due_dates = await _leaderboard_snapshot_due_dates(
|
||||
session_factory,
|
||||
settings_hash=settings_hash,
|
||||
limit=_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
|
||||
running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
|
||||
)
|
||||
processed = 0
|
||||
for stat_date in due_dates:
|
||||
claimed = await _claim_leaderboard_snapshot(
|
||||
session_factory,
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
|
||||
)
|
||||
if not claimed:
|
||||
continue
|
||||
logger.info("Leaderboard snapshot scheduler: building date=%s hash=%s", stat_date, settings_hash[:10])
|
||||
await _refresh_daily_leaderboard_snapshot(
|
||||
_daily_job_key(stat_date, settings_hash),
|
||||
session_factory=session_factory,
|
||||
settings=settings,
|
||||
stat_date=stat_date,
|
||||
settings_hash=settings_hash,
|
||||
excluded=excluded,
|
||||
include_scheduled=include_scheduled,
|
||||
include_admins=include_admins,
|
||||
include_failed=include_failed,
|
||||
)
|
||||
processed += 1
|
||||
if _LEADERBOARD_SNAPSHOT_SLEEP_SECONDS > 0 and processed < len(due_dates):
|
||||
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS)
|
||||
return processed, len(queued_dates)
|
||||
|
||||
|
||||
async def _leaderboard_snapshot_scheduler_loop() -> None:
|
||||
"""Background task inside the existing gateway process for daily snapshots."""
|
||||
if not _LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED:
|
||||
logger.info("Leaderboard snapshot scheduler disabled by ADMIN_LEADERBOARD_BACKGROUND_ENABLED")
|
||||
return
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
lock_handle = _acquire_singleton_lock(runtime_home() / ".leaderboard_snapshot_scheduler.lock")
|
||||
if lock_handle is None:
|
||||
logger.info("Leaderboard snapshot scheduler: another gateway worker holds the lock; this worker skips")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Leaderboard snapshot scheduler started: prewarm_days=%d batch_size=%d interval=%.1fs",
|
||||
_LEADERBOARD_SNAPSHOT_PREWARM_DAYS,
|
||||
_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
|
||||
_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS,
|
||||
)
|
||||
try:
|
||||
if _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS > 0:
|
||||
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
processed, queued = await _run_leaderboard_snapshot_tick()
|
||||
if processed or queued:
|
||||
logger.info(
|
||||
"Leaderboard snapshot scheduler tick: queued=%d processed=%d",
|
||||
queued,
|
||||
processed,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler tick failed (non-fatal)")
|
||||
await asyncio.sleep(_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS)
|
||||
finally:
|
||||
_release_singleton_lock(lock_handle)
|
||||
|
||||
|
||||
def _warn_if_multi_worker() -> None:
|
||||
"""检测 uvicorn / gunicorn 多 worker 部署并打 ERROR 日志。
|
||||
|
||||
主 agent / canvas / AI 写作流式响应在 Gateway 进程内通过 ``StreamBridge``
|
||||
@@ -799,21 +801,21 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
# 默认每天凌晨 3 点扫一次,删 retention_days(默认 7 天)以前的会话,联动
|
||||
# 清理 MySQL 业务表 + SQLite checkpointer 里的 thread 状态。多 worker
|
||||
# 下用文件锁选 leader,只有抢到锁的 worker 真正跑清理。
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
|
||||
|
||||
try:
|
||||
app.state.leaderboard_snapshot_scheduler_task = asyncio.create_task(_leaderboard_snapshot_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
try:
|
||||
from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
|
||||
|
||||
app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
|
||||
|
||||
try:
|
||||
app.state.leaderboard_snapshot_scheduler_task = asyncio.create_task(_leaderboard_snapshot_scheduler_loop())
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
|
||||
channel_service = await start_channel_service(app.state.config)
|
||||
logger.info("Channel service started: %s", channel_service.get_status())
|
||||
@@ -857,17 +859,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("AI writing cleanup scheduler shutdown error")
|
||||
|
||||
leaderboard_snapshot_task = getattr(app.state, "leaderboard_snapshot_scheduler_task", None)
|
||||
if leaderboard_snapshot_task is not None:
|
||||
leaderboard_snapshot_task.cancel()
|
||||
try:
|
||||
await leaderboard_snapshot_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler shutdown error")
|
||||
logger.exception("AI writing cleanup scheduler shutdown error")
|
||||
|
||||
leaderboard_snapshot_task = getattr(app.state, "leaderboard_snapshot_scheduler_task", None)
|
||||
if leaderboard_snapshot_task is not None:
|
||||
leaderboard_snapshot_task.cancel()
|
||||
try:
|
||||
await leaderboard_snapshot_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Leaderboard snapshot scheduler shutdown error")
|
||||
|
||||
# 阶段 7 删冗余:AI 写作 graph 不再自管 checkpointer / in-flight task;
|
||||
# thread 状态由 LangGraph Server 主 checkpointer 接管,lifespan 出口由
|
||||
@@ -1122,6 +1124,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Roundtable-planning business chains at /api/roundtable-chains
|
||||
app.include_router(roundtable_chains.router)
|
||||
|
||||
# Standalone parallel multi-agent panel: fan-out streaming + saved tasks
|
||||
app.include_router(parallel_agents.router)
|
||||
app.include_router(parallel_agent_tasks.router)
|
||||
|
||||
# Roundtable-planning background jobs at /api/roundtable-jobs
|
||||
app.include_router(roundtable_jobs.router)
|
||||
|
||||
@@ -1159,10 +1165,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(tool_metrics.router)
|
||||
|
||||
# Channels API is mounted at /api/channels
|
||||
app.include_router(channels.router)
|
||||
|
||||
# Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
|
||||
app.include_router(checkpoint_migration.router)
|
||||
app.include_router(channels.router)
|
||||
|
||||
# Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
|
||||
app.include_router(checkpoint_migration.router)
|
||||
|
||||
# Assistants compatibility API (LangGraph Platform stub)
|
||||
app.include_router(assistants_compat.router)
|
||||
|
||||
@@ -179,6 +179,10 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.roundtable_chain_store = make_roundtable_chain_store(sf)
|
||||
|
||||
from deerflow.persistence.parallel_agent_tasks import make_parallel_agent_task_store
|
||||
|
||||
app.state.parallel_agent_task_store = make_parallel_agent_task_store(sf)
|
||||
|
||||
from deerflow.persistence.roundtable_jobs import (
|
||||
make_roundtable_job_store,
|
||||
register_default_store as _register_roundtable_job_default_store,
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""CRUD API for parallel multi-agent task records (并行多智能体任务).
|
||||
|
||||
A task = one saved run of the standalone parallel multi-agent panel: a task name
|
||||
+ brief dispatched to several agents in parallel, plus the resulting message list.
|
||||
Bound to the requesting user (not the browser), so it follows the account across
|
||||
devices/sessions — same model as roundtable drafts.
|
||||
|
||||
Routes (prefix ``/api/parallel-agent-tasks``):
|
||||
GET "" list current user's tasks (lightweight meta, no messages)
|
||||
POST "" create a task
|
||||
GET "/{task_id}" one task (full, incl. messages)
|
||||
PUT "/{task_id}" partial update (title / brief / agents / messages)
|
||||
DELETE "/{task_id}" delete
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/parallel-agent-tasks", tags=["parallel-agent-tasks"])
|
||||
|
||||
|
||||
# ── schemas ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TaskAgentSchema(BaseModel):
|
||||
agent_id: str = Field(min_length=1, max_length=128)
|
||||
name: str = ""
|
||||
avatarType: str | None = None
|
||||
|
||||
|
||||
class TaskResponse(BaseModel):
|
||||
id: str
|
||||
title: str = ""
|
||||
brief: str | None = None
|
||||
agents: list[TaskAgentSchema] = []
|
||||
# messages is an opaque dialogue list (mirrors the frontend Dialogue[] shape);
|
||||
# absent on list responses (lightweight), present on get.
|
||||
messages: list[dict[str, Any]] | None = None
|
||||
created_at: datetime | str | None = None
|
||||
updated_at: datetime | str | None = None
|
||||
|
||||
|
||||
class TaskListResponse(BaseModel):
|
||||
tasks: list[TaskResponse]
|
||||
|
||||
|
||||
class TaskCreateRequest(BaseModel):
|
||||
id: str | None = Field(default=None, max_length=64)
|
||||
title: str = Field(default="", max_length=512)
|
||||
brief: str | None = None
|
||||
agents: list[TaskAgentSchema] = []
|
||||
messages: list[dict[str, Any]] = []
|
||||
|
||||
|
||||
class TaskUpdateRequest(BaseModel):
|
||||
# All optional → callers send only what changed.
|
||||
title: str | None = Field(default=None, max_length=512)
|
||||
brief: str | None = None
|
||||
agents: list[TaskAgentSchema] | None = None
|
||||
messages: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _current_user_id(request: Request) -> str:
|
||||
user = getattr(request.state, "user", None)
|
||||
if user is not None:
|
||||
return str(user.id)
|
||||
return get_effective_user_id()
|
||||
|
||||
|
||||
def _get_store(request: Request):
|
||||
store = getattr(request.app.state, "parallel_agent_task_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Parallel agent task store not available")
|
||||
return store
|
||||
|
||||
|
||||
# ── routes ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("", response_model=TaskListResponse)
|
||||
async def list_tasks(request: Request) -> TaskListResponse:
|
||||
store = _get_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
rows = await store.list_tasks(user_id)
|
||||
return TaskListResponse(tasks=[TaskResponse(**r) for r in rows])
|
||||
|
||||
|
||||
@router.post("", response_model=TaskResponse, status_code=201)
|
||||
async def create_task(request: Request, body: TaskCreateRequest) -> TaskResponse:
|
||||
store = _get_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
row = await store.create_task(
|
||||
user_id,
|
||||
{
|
||||
"id": (body.id or uuid4().hex),
|
||||
"title": body.title.strip(),
|
||||
"brief": body.brief,
|
||||
"agents": [a.model_dump() for a in body.agents],
|
||||
"messages": body.messages,
|
||||
},
|
||||
)
|
||||
return TaskResponse(**row)
|
||||
|
||||
|
||||
@router.get("/{task_id}", response_model=TaskResponse)
|
||||
async def get_task(request: Request, task_id: str) -> TaskResponse:
|
||||
store = _get_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
row = await store.get_task(task_id, user_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
return TaskResponse(**row)
|
||||
|
||||
|
||||
@router.put("/{task_id}", response_model=TaskResponse)
|
||||
async def update_task(request: Request, task_id: str, body: TaskUpdateRequest) -> TaskResponse:
|
||||
store = _get_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
fields = body.model_dump(exclude_unset=True)
|
||||
# Normalize agents (pydantic models) to plain dicts for JSON storage.
|
||||
if "agents" in fields and fields["agents"] is not None:
|
||||
fields["agents"] = [a if isinstance(a, dict) else a.model_dump() for a in body.agents or []]
|
||||
row = await store.update_task(task_id, user_id, **fields)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
return TaskResponse(**row)
|
||||
|
||||
|
||||
@router.delete("/{task_id}", status_code=204)
|
||||
async def delete_task(request: Request, task_id: str) -> None:
|
||||
store = _get_store(request)
|
||||
user_id = _current_user_id(request)
|
||||
deleted = await store.delete_task(task_id, user_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Server-side parallel fan-out for the standalone parallel multi-agent panel.
|
||||
|
||||
The Vue embeddable panel sends a single request — ``{task_name, brief, agent_ids}``
|
||||
— and this endpoint **fans out** to every agent in parallel (file writes disabled),
|
||||
then **merges** their LangGraph streams into ONE SSE tagged per ``agent_id``, so the
|
||||
client only consumes one stream and never parses LangGraph internals.
|
||||
|
||||
It reuses the roundtable single-agent machinery from ``multi_agent``:
|
||||
``_special_run(..., no_file=True)`` runs one seat under the ``seat_parallel`` run
|
||||
policy (which excludes ``write_file`` / ``str_replace`` / ``bash`` / ``web_search``
|
||||
/ ``agent_orchestration`` …), so parallel agents can't write files or dispatch.
|
||||
|
||||
Unified event protocol (one JSON object per ``data:`` frame)::
|
||||
|
||||
{"type": "agent_start", "agent_id": "..."}
|
||||
{"type": "delta", "agent_id": "...", "text": "..."} # AI text increment
|
||||
{"type": "step", "agent_id": "...", "key": "...", "label": "...", "args": {...}}
|
||||
{"type": "agent_done", "agent_id": "...", "content": "..."}
|
||||
{"type": "agent_error", "agent_id": "...", "message": "..."}
|
||||
{"type": "all_done"}
|
||||
|
||||
The frame-normalisation (LangGraph ``messages-tuple`` → delta / tool step) mirrors
|
||||
the frontend parser in ``frontend-web/src/roundtable-planning/api/multi-agent.ts``
|
||||
(``extractAiTextDelta`` + ``tool_call_chunks`` accumulation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import APIRouter, Body, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.gateway.routers.multi_agent import (
|
||||
_ROUNDTABLE_THREAD_METADATA,
|
||||
_auth_headers,
|
||||
_create_thread,
|
||||
_loopback_base,
|
||||
_make_loopback_client,
|
||||
_special_run,
|
||||
_validate_agent_id,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/parallel-agents", tags=["parallel-agents"])
|
||||
|
||||
# Soft cap on parallel degree — protects remote model concurrency + sandbox.
|
||||
MAX_PARALLEL_AGENTS = 8
|
||||
|
||||
|
||||
# ── frame normalisation (mirrors multi-agent.ts) ─────────────────────────────
|
||||
|
||||
|
||||
def _extract_ai_text(chunk: dict[str, Any]) -> str:
|
||||
"""AI text delta from a ``messages-tuple`` chunk obj; "" when not AI text."""
|
||||
type_ = str(chunk.get("type") or "").lower()
|
||||
if type_ not in ("ai", "aimessagechunk", "aimessage"):
|
||||
return ""
|
||||
content = chunk.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
out: list[str] = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
out.append(item)
|
||||
elif isinstance(item, dict):
|
||||
t = item.get("text")
|
||||
if isinstance(t, str):
|
||||
out.append(t)
|
||||
return "".join(out)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_tool_steps(chunk: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""Tool-call steps from a chunk obj — both complete ``tool_calls`` and
|
||||
incremental ``tool_call_chunks`` (we only need the name to surface a step)."""
|
||||
steps: list[dict[str, Any]] = []
|
||||
tool_calls = chunk.get("tool_calls")
|
||||
if isinstance(tool_calls, list):
|
||||
for tc in tool_calls:
|
||||
if not isinstance(tc, dict):
|
||||
continue
|
||||
name = tc.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
continue
|
||||
ident = tc.get("id") or name
|
||||
args = tc.get("args") if isinstance(tc.get("args"), dict) else None
|
||||
steps.append({"key": f"tool:{ident}", "label": f"调用工具 {name}", "args": args})
|
||||
tcc = chunk.get("tool_call_chunks")
|
||||
if isinstance(tcc, list):
|
||||
for c in tcc:
|
||||
if not isinstance(c, dict):
|
||||
continue
|
||||
name = c.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
continue
|
||||
idx = c.get("index") if isinstance(c.get("index"), int) else 0
|
||||
ident = c.get("id") or f"{name}:{idx}"
|
||||
steps.append({"key": f"tool:{ident}", "label": f"调用工具 {name}", "args": None})
|
||||
return steps
|
||||
|
||||
|
||||
def _build_task_message(task_name: str, brief: str) -> str:
|
||||
parts: list[str] = []
|
||||
if task_name.strip():
|
||||
parts.append(f"任务名称:{task_name.strip()}")
|
||||
if brief.strip():
|
||||
parts.append(f"任务简介:\n{brief.strip()}")
|
||||
parts.append(
|
||||
"请你从你的专业职责出发,针对以上任务给出你的分析与交付。"
|
||||
"本轮为**并行**研讨:请把你的产出**直接写在回复正文里**,"
|
||||
"不要写文件、不要执行脚本/命令。"
|
||||
)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
# ── endpoint ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/run/stream")
|
||||
async def run_parallel_agents_stream(request: Request, payload: dict = Body(...)) -> StreamingResponse:
|
||||
"""并行扇出 N 个智能体,合并成一条带 ``agent_id`` 标记的 SSE。文件写入固定禁用。"""
|
||||
task_name: str = str(payload.get("task_name") or "").strip()
|
||||
brief: str = str(payload.get("brief") or "").strip()
|
||||
raw_ids = payload.get("agent_ids") or []
|
||||
model_name: str = str(payload.get("model") or "")
|
||||
|
||||
if not isinstance(raw_ids, list) or not raw_ids:
|
||||
raise HTTPException(status_code=400, detail="agent_ids 不能为空")
|
||||
agent_ids: list[str] = []
|
||||
for aid in raw_ids:
|
||||
sid = str(aid or "").strip().lower()
|
||||
_validate_agent_id(sid, "agent_ids")
|
||||
if sid not in agent_ids:
|
||||
agent_ids.append(sid)
|
||||
if len(agent_ids) > MAX_PARALLEL_AGENTS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"并行智能体数量上限 {MAX_PARALLEL_AGENTS}(当前 {len(agent_ids)})",
|
||||
)
|
||||
if not task_name and not brief:
|
||||
raise HTTPException(status_code=400, detail="任务名称与简介至少填写其一")
|
||||
|
||||
base = _loopback_base(request)
|
||||
headers = _auth_headers(request)
|
||||
new_message = _build_task_message(task_name, brief)
|
||||
|
||||
async def _consume_agent(
|
||||
client: Any,
|
||||
agent_id: str,
|
||||
queue: asyncio.Queue[dict[str, Any]],
|
||||
) -> None:
|
||||
await queue.put({"type": "agent_start", "agent_id": agent_id})
|
||||
last_event = ""
|
||||
seen_steps: set[str] = set()
|
||||
buffer: list[str] = []
|
||||
done_sent = False
|
||||
try:
|
||||
thread_id = await _create_thread(client, base, headers, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
gen = _special_run(
|
||||
client,
|
||||
base,
|
||||
headers,
|
||||
{agent_id: thread_id},
|
||||
agent_id,
|
||||
new_message,
|
||||
[],
|
||||
model_name,
|
||||
display_name="",
|
||||
no_file=True, # seat_parallel policy → 禁 write_file/bash/str_replace…
|
||||
)
|
||||
async for chunk in gen:
|
||||
if isinstance(chunk, dict):
|
||||
# 终态帧 {status, content, agent_name}
|
||||
content = str(chunk.get("content") or "")
|
||||
await queue.put({"type": "agent_done", "agent_id": agent_id, "content": content})
|
||||
done_sent = True
|
||||
continue
|
||||
line = chunk.strip() if isinstance(chunk, str) else ""
|
||||
if line.startswith("event:"):
|
||||
last_event = line[6:].strip()
|
||||
continue
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
raw = line[5:].strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except Exception:
|
||||
continue
|
||||
if last_event == "messages" and isinstance(parsed, list) and parsed:
|
||||
obj = parsed[0]
|
||||
if isinstance(obj, dict):
|
||||
text = _extract_ai_text(obj)
|
||||
if text:
|
||||
buffer.append(text)
|
||||
await queue.put({"type": "delta", "agent_id": agent_id, "text": text})
|
||||
for step in _extract_tool_steps(obj):
|
||||
key = step["key"]
|
||||
if key in seen_steps:
|
||||
continue
|
||||
seen_steps.add(key)
|
||||
await queue.put({"type": "step", "agent_id": agent_id, **step})
|
||||
if not done_sent:
|
||||
# 流被抢占/截断,没收到终态帧 —— 用已累积正文兜底收口本席位。
|
||||
await queue.put(
|
||||
{"type": "agent_done", "agent_id": agent_id, "content": "".join(buffer)}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — 单个 agent 失败不拖垮其它
|
||||
logger.warning("[parallel-agents] %s failed: %r", agent_id, exc)
|
||||
await queue.put({"type": "agent_error", "agent_id": agent_id, "message": str(exc)})
|
||||
finally:
|
||||
await queue.put({"type": "__agent_end__", "agent_id": agent_id})
|
||||
|
||||
async def stream_generator() -> AsyncIterator[str]:
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
|
||||
async with _make_loopback_client(request) as client:
|
||||
tasks = [asyncio.create_task(_consume_agent(client, aid, queue)) for aid in agent_ids]
|
||||
ended = 0
|
||||
try:
|
||||
while ended < len(agent_ids):
|
||||
event = await queue.get()
|
||||
if event.get("type") == "__agent_end__":
|
||||
ended += 1
|
||||
continue
|
||||
yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
|
||||
yield f"data: {json.dumps({'type': 'all_done'}, ensure_ascii=False)}\n\n"
|
||||
finally:
|
||||
for t in tasks:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
return StreamingResponse(
|
||||
stream_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""Create parallel_agent_tasks table.
|
||||
|
||||
Saved runs of the standalone parallel multi-agent panel (Vue embeddable
|
||||
component): task name + brief + agents snapshot + the resulting message list,
|
||||
bound per user. ``brief`` / ``agents`` / ``messages`` are LONGTEXT on MySQL
|
||||
(mirror ``roundtable_drafts`` headroom) and plain TEXT elsewhere — matching
|
||||
``PortableLongText``. ``create_all`` builds the table for brand-new DBs; this
|
||||
revision patches DBs that already exist.
|
||||
|
||||
Chained after ``20260608_04`` (chain stage_goals) so the revision graph stays
|
||||
single-headed — otherwise ``alembic upgrade head`` aborts with "multiple heads"
|
||||
and the startup auto-migrate silently falls back to the create_all schema.
|
||||
|
||||
Revision ID: 20260608_05
|
||||
Revises: 20260608_04
|
||||
Create Date: 2026-06-08
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.dialects.mysql import LONGTEXT
|
||||
|
||||
|
||||
revision = "20260608_05"
|
||||
down_revision = "20260608_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("parallel_agent_tasks"):
|
||||
return
|
||||
long_text = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text()
|
||||
op.create_table(
|
||||
"parallel_agent_tasks",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True, nullable=False),
|
||||
sa.Column("user_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("title", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("brief", long_text, nullable=True),
|
||||
sa.Column("agents", long_text, nullable=True),
|
||||
sa.Column("messages", long_text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_parallel_agent_tasks_user_id", "parallel_agent_tasks", ["user_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("parallel_agent_tasks"):
|
||||
op.drop_index("ix_parallel_agent_tasks_user_id", table_name="parallel_agent_tasks")
|
||||
op.drop_table("parallel_agent_tasks")
|
||||
+2
@@ -27,6 +27,7 @@ from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
|
||||
from deerflow.persistence.roundtable_chains.model import RoundtableChainRow
|
||||
from deerflow.persistence.roundtable_draft_shares.model import RoundtableDraftShareRow
|
||||
@@ -62,6 +63,7 @@ __all__ = [
|
||||
"AdminLeaderboardDailyStatRow",
|
||||
"LlmCallMetricRow",
|
||||
"NotificationRow",
|
||||
"ParallelAgentTaskRow",
|
||||
"RecommendedQuestionRow",
|
||||
"RoundtableChainRow",
|
||||
"RoundtableDraftRow",
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Parallel multi-agent task persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
from deerflow.persistence.parallel_agent_tasks.sql import ParallelAgentTaskRepository
|
||||
|
||||
__all__ = [
|
||||
"ParallelAgentTaskRepository",
|
||||
"ParallelAgentTaskRow",
|
||||
"make_parallel_agent_task_store",
|
||||
]
|
||||
|
||||
|
||||
def make_parallel_agent_task_store(session_factory):
|
||||
if session_factory is None:
|
||||
return None
|
||||
return ParallelAgentTaskRepository(session_factory)
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
"""ORM model for parallel multi-agent task records.
|
||||
|
||||
A "parallel-agent task" is one saved run of the standalone *parallel multi-agent*
|
||||
panel (the Vue3 embeddable component): a task name + brief dispatched to several
|
||||
agents in parallel, plus the resulting chat/message list. Persisted per user so it
|
||||
follows the account across devices/sessions — same model as ``roundtable_drafts``.
|
||||
|
||||
``agents`` and ``messages`` are stored as JSON serialized into ``PortableLongText``
|
||||
(MySQL ``LONGTEXT``) rather than ``PortableJSON`` (MySQL ``TEXT``, 64 KB cap),
|
||||
because the parallel transcript can carry several agents' full deliveries that
|
||||
easily exceed 64 KB — mirroring ``roundtable_drafts``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class ParallelAgentTaskRow(Base):
|
||||
__tablename__ = "parallel_agent_tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# Task name (用户输入的任务名称) shown on the history list.
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# Task brief / 简介 — the free-text task description.
|
||||
brief: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# JSON: snapshot of the agents dispatched, [{"agent_id", "name", "avatarType"?}].
|
||||
agents: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# JSON: the full message/dialogue list (mirrors the frontend Dialogue[] shape).
|
||||
messages: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
"""SQLAlchemy-backed store for parallel multi-agent task records.
|
||||
|
||||
``agents`` / ``messages`` JSON columns are stored as serialized strings in
|
||||
``PortableLongText`` and round-tripped here with ``json.dumps`` / ``json.loads``
|
||||
— same approach as ``roundtable_drafts`` / ``roundtable_chains``.
|
||||
|
||||
All write/read paths are scoped by ``user_id`` for ownership isolation; a row
|
||||
that belongs to another user is treated as not found.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
class ParallelAgentTaskRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: ParallelAgentTaskRow, *, with_messages: bool = True) -> dict[str, Any]:
|
||||
data: dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"brief": row.brief,
|
||||
"agents": _loads(row.agents) or [],
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
if with_messages:
|
||||
data["messages"] = _loads(row.messages) or []
|
||||
return data
|
||||
|
||||
async def list_tasks(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
"""列出当前用户的任务(轻量元信息,不含 messages,供历史列表)。"""
|
||||
stmt = (
|
||||
select(ParallelAgentTaskRow)
|
||||
.where(ParallelAgentTaskRow.user_id == user_id)
|
||||
.order_by(ParallelAgentTaskRow.updated_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._to_dict(row, with_messages=False) for row in result.scalars()]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ParallelAgentTaskRow, task_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
return self._to_dict(row)
|
||||
|
||||
async def create_task(self, user_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = ParallelAgentTaskRow(
|
||||
id=data["id"],
|
||||
user_id=user_id,
|
||||
title=(data.get("title") or "").strip(),
|
||||
brief=(data.get("brief") or None),
|
||||
agents=_dumps(data.get("agents") or []),
|
||||
messages=_dumps(data.get("messages") or []),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def update_task(
|
||||
self,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
title=_UNSET,
|
||||
brief=_UNSET,
|
||||
agents=_UNSET,
|
||||
messages=_UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ParallelAgentTaskRow, task_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return None
|
||||
if title is not _UNSET and title is not None:
|
||||
row.title = str(title).strip()
|
||||
if brief is not _UNSET:
|
||||
row.brief = brief or None
|
||||
if agents is not _UNSET:
|
||||
row.agents = _dumps(agents)
|
||||
if messages is not _UNSET:
|
||||
row.messages = _dumps(messages)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
async def delete_task(self, task_id: str, user_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ParallelAgentTaskRow, task_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
Reference in new issue
Block a user