Merge branch 'gzy' into 'master'
Gzy See merge request knowledge-base-bj/deerflow-server!149
This commit is contained in:
commit
fb103b6ad9
5 files changed
+105
-14
No files matched your search
@@ -27,7 +27,7 @@ from app.gateway.routers.thread_runs import RunCreateRequest
|
|||||||
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
|
from app.gateway.routers.threads import ThreadCreateRequest, create_thread
|
||||||
from app.gateway.routers.uploads import _make_file_sandbox_writable
|
from app.gateway.routers.uploads import _make_file_sandbox_writable
|
||||||
from app.gateway.services import start_run
|
from app.gateway.services import start_run
|
||||||
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks
|
from deerflow.browser_context.tasks import claim_next_browser_task, complete_browser_task, prune_browser_tasks, tasks_dir
|
||||||
from deerflow.config.app_config import AppConfig
|
from deerflow.config.app_config import AppConfig
|
||||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||||
from deerflow.runtime import serialize_channel_values
|
from deerflow.runtime import serialize_channel_values
|
||||||
@@ -307,10 +307,19 @@ async def claim_browser_context_task(
|
|||||||
config: AppConfig = Depends(get_config),
|
config: AppConfig = Depends(get_config),
|
||||||
) -> BrowserContextTaskNextResponse:
|
) -> BrowserContextTaskNextResponse:
|
||||||
settings = _require_enabled(config)
|
settings = _require_enabled(config)
|
||||||
|
effective_user_id = get_effective_user_id()
|
||||||
|
effective_worker_id = worker_id or "chrome-extension"
|
||||||
|
logger.info(
|
||||||
|
"Browser task next poll user_id=%s worker_id=%s tasks_dir=%s client=%s",
|
||||||
|
effective_user_id,
|
||||||
|
effective_worker_id,
|
||||||
|
tasks_dir(),
|
||||||
|
getattr(request.client, "host", None),
|
||||||
|
)
|
||||||
prune_browser_tasks()
|
prune_browser_tasks()
|
||||||
task = claim_next_browser_task(
|
task = claim_next_browser_task(
|
||||||
user_id=get_effective_user_id(),
|
user_id=effective_user_id,
|
||||||
worker_id=worker_id or "chrome-extension",
|
worker_id=effective_worker_id,
|
||||||
lease_seconds=settings.task_lease_seconds,
|
lease_seconds=settings.task_lease_seconds,
|
||||||
)
|
)
|
||||||
if task is None:
|
if task is None:
|
||||||
|
|||||||
@@ -171,6 +171,11 @@ vision:
|
|||||||
# prompt: 请详细描述这张图片的内容。
|
# prompt: 请详细描述这张图片的内容。
|
||||||
# max_tokens: 2048
|
# max_tokens: 2048
|
||||||
|
|
||||||
|
# browser_act 相关配置
|
||||||
|
browser_context:
|
||||||
|
enabled: true
|
||||||
|
actions_enabled: true
|
||||||
|
|
||||||
# 上传文件识别配置。
|
# 上传文件识别配置。
|
||||||
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
|
# - auto_convert_documents: true 时,上传的 PDF / Word / Excel / PPT 会在服务端自动转换为
|
||||||
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。
|
# Markdown(.md),agent 通过 read_file / grep 读取其内容 —— 即“识别”文档内容。
|
||||||
@@ -268,6 +273,16 @@ tools:
|
|||||||
group: bash
|
group: bash
|
||||||
use: deerflow.sandbox.tools:bash_tool
|
use: deerflow.sandbox.tools:bash_tool
|
||||||
|
|
||||||
|
# browser_act 相关配置
|
||||||
|
- name: browser_fetch_page
|
||||||
|
group: web
|
||||||
|
use: deerflow.tools.builtins.browser_context_tool:browser_fetch_page_tool
|
||||||
|
|
||||||
|
# browser_act 相关配置
|
||||||
|
- name: browser_act
|
||||||
|
group: web
|
||||||
|
use: deerflow.tools.builtins.browser_act_tool:browser_act_tool
|
||||||
|
|
||||||
sandbox:
|
sandbox:
|
||||||
use: deerflow.sandbox.local:LocalSandboxProvider
|
use: deerflow.sandbox.local:LocalSandboxProvider
|
||||||
allow_host_bash: true
|
allow_host_bash: true
|
||||||
|
|||||||
@@ -1,14 +1,35 @@
|
|||||||
{
|
{
|
||||||
"mcpServers": {},
|
"mcpServers": {},
|
||||||
"skills": {
|
"skills": {
|
||||||
"web-research": { "enabled": true },
|
"web-research": {
|
||||||
"deep-search": { "enabled": true },
|
"enabled": true
|
||||||
"code-retrieval": { "enabled": true },
|
},
|
||||||
"news-briefing": { "enabled": true },
|
"deep-search": {
|
||||||
"workspace-search": { "enabled": true },
|
"enabled": true
|
||||||
"policy-knowledge": { "enabled": false },
|
},
|
||||||
"html-page-builder": { "enabled": true },
|
"code-retrieval": {
|
||||||
"html-charts": { "enabled": true },
|
"enabled": true
|
||||||
"html-multipage": { "enabled": true }
|
},
|
||||||
|
"news-briefing": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"workspace-search": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"policy-knowledge": {
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
|
"html-page-builder": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"html-charts": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"html-multipage": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"browser_act": {
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+31
-2
@@ -193,16 +193,22 @@ def claim_next_browser_task(
|
|||||||
user_id = str(user_id or "default")
|
user_id = str(user_id or "default")
|
||||||
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
|
worker_id = (worker_id or "chrome-extension").strip()[:128] or "chrome-extension"
|
||||||
lease_seconds = max(10, int(lease_seconds))
|
lease_seconds = max(10, int(lease_seconds))
|
||||||
|
queue_dir = tasks_dir()
|
||||||
|
|
||||||
with _LOCK:
|
with _LOCK:
|
||||||
paths = sorted(tasks_dir().glob("*.json"), key=lambda p: p.stat().st_mtime)
|
paths = sorted(queue_dir.glob("*.json"), key=lambda p: p.stat().st_mtime)
|
||||||
|
matching_user = 0
|
||||||
|
matching_status_counts: dict[str, int] = {}
|
||||||
for path in paths:
|
for path in paths:
|
||||||
data = _read_task(path)
|
data = _read_task(path)
|
||||||
if not data or str(data.get("user_id") or "") != user_id:
|
if not data or str(data.get("user_id") or "") != user_id:
|
||||||
continue
|
continue
|
||||||
|
matching_user += 1
|
||||||
|
|
||||||
data = _expire_if_needed(path, data, now)
|
data = _expire_if_needed(path, data, now)
|
||||||
status = data.get("status")
|
status = data.get("status")
|
||||||
|
status_key = str(status or "unknown")
|
||||||
|
matching_status_counts[status_key] = matching_status_counts.get(status_key, 0) + 1
|
||||||
if status == "running":
|
if status == "running":
|
||||||
updated_at = _parse_iso(data.get("updated_at")) or now
|
updated_at = _parse_iso(data.get("updated_at")) or now
|
||||||
if (now - updated_at).total_seconds() <= lease_seconds:
|
if (now - updated_at).total_seconds() <= lease_seconds:
|
||||||
@@ -220,8 +226,32 @@ def claim_next_browser_task(
|
|||||||
data["claimed_at"] = _iso(now)
|
data["claimed_at"] = _iso(now)
|
||||||
data["updated_at"] = _iso(now)
|
data["updated_at"] = _iso(now)
|
||||||
_write_task(path, data)
|
_write_task(path, data)
|
||||||
|
logger.info(
|
||||||
|
"Browser task claim scan user_id=%s worker_id=%s tasks_dir=%s total_files=%d matching_user=%d "
|
||||||
|
"matching_statuses=%s claimed_task_id=%s claimed_kind=%s",
|
||||||
|
user_id,
|
||||||
|
worker_id,
|
||||||
|
queue_dir,
|
||||||
|
len(paths),
|
||||||
|
matching_user,
|
||||||
|
matching_status_counts,
|
||||||
|
data.get("id"),
|
||||||
|
data.get("kind") or "fetch",
|
||||||
|
)
|
||||||
return _public_task(data)
|
return _public_task(data)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Browser task claim scan user_id=%s worker_id=%s tasks_dir=%s total_files=%d matching_user=%d "
|
||||||
|
"matching_statuses=%s claimed_task_id=%s claimed_kind=%s",
|
||||||
|
user_id,
|
||||||
|
worker_id,
|
||||||
|
queue_dir,
|
||||||
|
len(paths),
|
||||||
|
matching_user,
|
||||||
|
matching_status_counts,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -300,4 +330,3 @@ async def wait_for_browser_task_result(
|
|||||||
if time.monotonic() >= deadline:
|
if time.monotonic() >= deadline:
|
||||||
return task
|
return task
|
||||||
await asyncio.sleep(interval)
|
await asyncio.sleep(interval)
|
||||||
|
|
||||||
@@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -76,6 +77,22 @@ def test_fetch_task_default_kind_is_fetch():
|
|||||||
assert claimed["action"] is None
|
assert claimed["action"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_next_browser_task_logs_claim_diagnostics(caplog):
|
||||||
|
task_queue.create_browser_task(user_id="u1", url="https://example.com/1")
|
||||||
|
task_queue.create_browser_task(user_id="u2", url="https://example.com/2")
|
||||||
|
|
||||||
|
with caplog.at_level(logging.INFO, logger=task_queue.__name__):
|
||||||
|
claimed = task_queue.claim_next_browser_task(user_id="u1", worker_id="worker-1")
|
||||||
|
|
||||||
|
assert claimed is not None
|
||||||
|
assert "Browser task claim scan" in caplog.text
|
||||||
|
assert "user_id=u1" in caplog.text
|
||||||
|
assert "worker_id=worker-1" in caplog.text
|
||||||
|
assert "total_files=2" in caplog.text
|
||||||
|
assert "matching_user=1" in caplog.text
|
||||||
|
assert "claimed_task_id=" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
# --- tool validation --------------------------------------------------------
|
# --- tool validation --------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user