新增用户意图确认功能

This commit is contained in:
likun committed 2026-05-13 14:36:44 +08:00
1 parent cf2e4bdd9c
commit b8926f00d7
574 files changed
+97664 -97208

No files matched your search

@@ -1,15 +1,15 @@
from .aio_sandbox import AioSandbox
from .aio_sandbox_provider import AioSandboxProvider
from .backend import SandboxBackend
from .local_backend import LocalContainerBackend
from .remote_backend import RemoteSandboxBackend
from .sandbox_info import SandboxInfo
__all__ = [
"AioSandbox",
"AioSandboxProvider",
"LocalContainerBackend",
"RemoteSandboxBackend",
"SandboxBackend",
"SandboxInfo",
]
from .aio_sandbox import AioSandbox
from .aio_sandbox_provider import AioSandboxProvider
from .backend import SandboxBackend
from .local_backend import LocalContainerBackend
from .remote_backend import RemoteSandboxBackend
from .sandbox_info import SandboxInfo
__all__ = [
"AioSandbox",
"AioSandboxProvider",
"LocalContainerBackend",
"RemoteSandboxBackend",
"SandboxBackend",
"SandboxInfo",
]
@@ -1,238 +1,238 @@
import base64
import logging
import shlex
import threading
import uuid
from agent_sandbox import Sandbox as AioSandboxClient
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.search import GrepMatch, path_matches, should_ignore_path, truncate_line
logger = logging.getLogger(__name__)
_ERROR_OBSERVATION_SIGNATURE = "'ErrorObservation' object has no attribute 'exit_code'"
class AioSandbox(Sandbox):
"""Sandbox implementation using the agent-infra/sandbox Docker container.
This sandbox connects to a running AIO sandbox container via HTTP API.
A threading lock serializes shell commands to prevent concurrent requests
from corrupting the container's single persistent session (see #1433).
"""
def __init__(self, id: str, base_url: str, home_dir: str | None = None):
"""Initialize the AIO sandbox.
Args:
id: Unique identifier for this sandbox instance.
base_url: URL of the sandbox API (e.g., http://localhost:8080).
home_dir: Home directory inside the sandbox. If None, will be fetched from the sandbox.
"""
super().__init__(id)
self._base_url = base_url
self._client = AioSandboxClient(base_url=base_url, timeout=600)
self._home_dir = home_dir
self._lock = threading.Lock()
@property
def base_url(self) -> str:
return self._base_url
@property
def home_dir(self) -> str:
"""Get the home directory inside the sandbox."""
if self._home_dir is None:
context = self._client.sandbox.get_context()
self._home_dir = context.home_dir
return self._home_dir
# Default no_change_timeout for exec_command (seconds). Matches the
# client-level timeout so that long-running commands which produce no
# output are not prematurely terminated by the sandbox's built-in 120 s
# default.
_DEFAULT_NO_CHANGE_TIMEOUT = 600
def execute_command(self, command: str) -> str:
"""Execute a shell command in the sandbox.
Uses a lock to serialize concurrent requests. The AIO sandbox
container maintains a single persistent shell session that
corrupts when hit with concurrent exec_command calls (returns
``ErrorObservation`` instead of real output). If corruption is
detected despite the lock (e.g. multiple processes sharing a
sandbox), the command is retried on a fresh session.
Args:
command: The command to execute.
Returns:
The output of the command.
"""
with self._lock:
try:
result = self._client.shell.exec_command(command=command, no_change_timeout=self._DEFAULT_NO_CHANGE_TIMEOUT)
output = result.data.output if result.data else ""
if output and _ERROR_OBSERVATION_SIGNATURE in output:
logger.warning("ErrorObservation detected in sandbox output, retrying with a fresh session")
fresh_id = str(uuid.uuid4())
result = self._client.shell.exec_command(command=command, id=fresh_id, no_change_timeout=self._DEFAULT_NO_CHANGE_TIMEOUT)
output = result.data.output if result.data else ""
return output if output else "(no output)"
except Exception as e:
logger.error(f"Failed to execute command in sandbox: {e}")
return f"Error: {e}"
def read_file(self, path: str) -> str:
"""Read the content of a file in the sandbox.
Args:
path: The absolute path of the file to read.
Returns:
The content of the file.
"""
try:
result = self._client.file.read_file(file=path)
return result.data.content if result.data else ""
except Exception as e:
logger.error(f"Failed to read file in sandbox: {e}")
return f"Error: {e}"
def list_dir(self, path: str, max_depth: int = 2) -> list[str]:
"""List the contents of a directory in the sandbox.
Args:
path: The absolute path of the directory to list.
max_depth: The maximum depth to traverse. Default is 2.
Returns:
The contents of the directory.
"""
with self._lock:
try:
result = self._client.shell.exec_command(command=f"find {shlex.quote(path)} -maxdepth {max_depth} -type f -o -type d 2>/dev/null | head -500", no_change_timeout=self._DEFAULT_NO_CHANGE_TIMEOUT)
output = result.data.output if result.data else ""
if output:
return [line.strip() for line in output.strip().split("\n") if line.strip()]
return []
except Exception as e:
logger.error(f"Failed to list directory in sandbox: {e}")
return []
def write_file(self, path: str, content: str, append: bool = False) -> None:
"""Write content to a file in the sandbox.
Args:
path: The absolute path of the file to write to.
content: The text content to write to the file.
append: Whether to append the content to the file.
"""
with self._lock:
try:
if append:
existing = self.read_file(path)
if not existing.startswith("Error:"):
content = existing + content
self._client.file.write_file(file=path, content=content)
except Exception as e:
logger.error(f"Failed to write file in sandbox: {e}")
raise
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
if not include_dirs:
result = self._client.file.find_files(path=path, glob=pattern)
files = result.data.files if result.data and result.data.files else []
filtered = [file_path for file_path in files if not should_ignore_path(file_path)]
truncated = len(filtered) > max_results
return filtered[:max_results], truncated
result = self._client.file.list_path(path=path, recursive=True, show_hidden=False)
entries = result.data.files if result.data and result.data.files else []
matches: list[str] = []
root_path = path.rstrip("/") or "/"
root_prefix = root_path if root_path == "/" else f"{root_path}/"
for entry in entries:
if entry.path != root_path and not entry.path.startswith(root_prefix):
continue
if should_ignore_path(entry.path):
continue
rel_path = entry.path[len(root_path) :].lstrip("/")
if path_matches(pattern, rel_path):
matches.append(entry.path)
if len(matches) >= max_results:
return matches, True
return matches, False
def grep(
self,
path: str,
pattern: str,
*,
glob: str | None = None,
literal: bool = False,
case_sensitive: bool = False,
max_results: int = 100,
) -> tuple[list[GrepMatch], bool]:
import re as _re
regex_source = _re.escape(pattern) if literal else pattern
# Validate the pattern locally so an invalid regex raises re.error
# (caught by grep_tool's except re.error handler) rather than a
# generic remote API error.
_re.compile(regex_source, 0 if case_sensitive else _re.IGNORECASE)
regex = regex_source if case_sensitive else f"(?i){regex_source}"
if glob is not None:
find_result = self._client.file.find_files(path=path, glob=glob)
candidate_paths = find_result.data.files if find_result.data and find_result.data.files else []
else:
list_result = self._client.file.list_path(path=path, recursive=True, show_hidden=False)
entries = list_result.data.files if list_result.data and list_result.data.files else []
candidate_paths = [entry.path for entry in entries if not entry.is_directory]
matches: list[GrepMatch] = []
truncated = False
for file_path in candidate_paths:
if should_ignore_path(file_path):
continue
search_result = self._client.file.search_in_file(file=file_path, regex=regex)
data = search_result.data
if data is None:
continue
line_numbers = data.line_numbers or []
matched_lines = data.matches or []
for line_number, line in zip(line_numbers, matched_lines):
matches.append(
GrepMatch(
path=file_path,
line_number=line_number if isinstance(line_number, int) else 0,
line=truncate_line(line),
)
)
if len(matches) >= max_results:
truncated = True
return matches, truncated
return matches, truncated
def update_file(self, path: str, content: bytes) -> None:
"""Update a file with binary content in the sandbox.
Args:
path: The absolute path of the file to update.
content: The binary content to write to the file.
"""
with self._lock:
try:
base64_content = base64.b64encode(content).decode("utf-8")
self._client.file.write_file(file=path, content=base64_content, encoding="base64")
except Exception as e:
logger.error(f"Failed to update file in sandbox: {e}")
raise
import base64
import logging
import shlex
import threading
import uuid
from agent_sandbox import Sandbox as AioSandboxClient
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.search import GrepMatch, path_matches, should_ignore_path, truncate_line
logger = logging.getLogger(__name__)
_ERROR_OBSERVATION_SIGNATURE = "'ErrorObservation' object has no attribute 'exit_code'"
class AioSandbox(Sandbox):
"""Sandbox implementation using the agent-infra/sandbox Docker container.
This sandbox connects to a running AIO sandbox container via HTTP API.
A threading lock serializes shell commands to prevent concurrent requests
from corrupting the container's single persistent session (see #1433).
"""
def __init__(self, id: str, base_url: str, home_dir: str | None = None):
"""Initialize the AIO sandbox.
Args:
id: Unique identifier for this sandbox instance.
base_url: URL of the sandbox API (e.g., http://localhost:8080).
home_dir: Home directory inside the sandbox. If None, will be fetched from the sandbox.
"""
super().__init__(id)
self._base_url = base_url
self._client = AioSandboxClient(base_url=base_url, timeout=600)
self._home_dir = home_dir
self._lock = threading.Lock()
@property
def base_url(self) -> str:
return self._base_url
@property
def home_dir(self) -> str:
"""Get the home directory inside the sandbox."""
if self._home_dir is None:
context = self._client.sandbox.get_context()
self._home_dir = context.home_dir
return self._home_dir
# Default no_change_timeout for exec_command (seconds). Matches the
# client-level timeout so that long-running commands which produce no
# output are not prematurely terminated by the sandbox's built-in 120 s
# default.
_DEFAULT_NO_CHANGE_TIMEOUT = 600
def execute_command(self, command: str) -> str:
"""Execute a shell command in the sandbox.
Uses a lock to serialize concurrent requests. The AIO sandbox
container maintains a single persistent shell session that
corrupts when hit with concurrent exec_command calls (returns
``ErrorObservation`` instead of real output). If corruption is
detected despite the lock (e.g. multiple processes sharing a
sandbox), the command is retried on a fresh session.
Args:
command: The command to execute.
Returns:
The output of the command.
"""
with self._lock:
try:
result = self._client.shell.exec_command(command=command, no_change_timeout=self._DEFAULT_NO_CHANGE_TIMEOUT)
output = result.data.output if result.data else ""
if output and _ERROR_OBSERVATION_SIGNATURE in output:
logger.warning("ErrorObservation detected in sandbox output, retrying with a fresh session")
fresh_id = str(uuid.uuid4())
result = self._client.shell.exec_command(command=command, id=fresh_id, no_change_timeout=self._DEFAULT_NO_CHANGE_TIMEOUT)
output = result.data.output if result.data else ""
return output if output else "(no output)"
except Exception as e:
logger.error(f"Failed to execute command in sandbox: {e}")
return f"Error: {e}"
def read_file(self, path: str) -> str:
"""Read the content of a file in the sandbox.
Args:
path: The absolute path of the file to read.
Returns:
The content of the file.
"""
try:
result = self._client.file.read_file(file=path)
return result.data.content if result.data else ""
except Exception as e:
logger.error(f"Failed to read file in sandbox: {e}")
return f"Error: {e}"
def list_dir(self, path: str, max_depth: int = 2) -> list[str]:
"""List the contents of a directory in the sandbox.
Args:
path: The absolute path of the directory to list.
max_depth: The maximum depth to traverse. Default is 2.
Returns:
The contents of the directory.
"""
with self._lock:
try:
result = self._client.shell.exec_command(command=f"find {shlex.quote(path)} -maxdepth {max_depth} -type f -o -type d 2>/dev/null | head -500", no_change_timeout=self._DEFAULT_NO_CHANGE_TIMEOUT)
output = result.data.output if result.data else ""
if output:
return [line.strip() for line in output.strip().split("\n") if line.strip()]
return []
except Exception as e:
logger.error(f"Failed to list directory in sandbox: {e}")
return []
def write_file(self, path: str, content: str, append: bool = False) -> None:
"""Write content to a file in the sandbox.
Args:
path: The absolute path of the file to write to.
content: The text content to write to the file.
append: Whether to append the content to the file.
"""
with self._lock:
try:
if append:
existing = self.read_file(path)
if not existing.startswith("Error:"):
content = existing + content
self._client.file.write_file(file=path, content=content)
except Exception as e:
logger.error(f"Failed to write file in sandbox: {e}")
raise
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
if not include_dirs:
result = self._client.file.find_files(path=path, glob=pattern)
files = result.data.files if result.data and result.data.files else []
filtered = [file_path for file_path in files if not should_ignore_path(file_path)]
truncated = len(filtered) > max_results
return filtered[:max_results], truncated
result = self._client.file.list_path(path=path, recursive=True, show_hidden=False)
entries = result.data.files if result.data and result.data.files else []
matches: list[str] = []
root_path = path.rstrip("/") or "/"
root_prefix = root_path if root_path == "/" else f"{root_path}/"
for entry in entries:
if entry.path != root_path and not entry.path.startswith(root_prefix):
continue
if should_ignore_path(entry.path):
continue
rel_path = entry.path[len(root_path) :].lstrip("/")
if path_matches(pattern, rel_path):
matches.append(entry.path)
if len(matches) >= max_results:
return matches, True
return matches, False
def grep(
self,
path: str,
pattern: str,
*,
glob: str | None = None,
literal: bool = False,
case_sensitive: bool = False,
max_results: int = 100,
) -> tuple[list[GrepMatch], bool]:
import re as _re
regex_source = _re.escape(pattern) if literal else pattern
# Validate the pattern locally so an invalid regex raises re.error
# (caught by grep_tool's except re.error handler) rather than a
# generic remote API error.
_re.compile(regex_source, 0 if case_sensitive else _re.IGNORECASE)
regex = regex_source if case_sensitive else f"(?i){regex_source}"
if glob is not None:
find_result = self._client.file.find_files(path=path, glob=glob)
candidate_paths = find_result.data.files if find_result.data and find_result.data.files else []
else:
list_result = self._client.file.list_path(path=path, recursive=True, show_hidden=False)
entries = list_result.data.files if list_result.data and list_result.data.files else []
candidate_paths = [entry.path for entry in entries if not entry.is_directory]
matches: list[GrepMatch] = []
truncated = False
for file_path in candidate_paths:
if should_ignore_path(file_path):
continue
search_result = self._client.file.search_in_file(file=file_path, regex=regex)
data = search_result.data
if data is None:
continue
line_numbers = data.line_numbers or []
matched_lines = data.matches or []
for line_number, line in zip(line_numbers, matched_lines):
matches.append(
GrepMatch(
path=file_path,
line_number=line_number if isinstance(line_number, int) else 0,
line=truncate_line(line),
)
)
if len(matches) >= max_results:
truncated = True
return matches, truncated
return matches, truncated
def update_file(self, path: str, content: bytes) -> None:
"""Update a file with binary content in the sandbox.
Args:
path: The absolute path of the file to update.
content: The binary content to write to the file.
"""
with self._lock:
try:
base64_content = base64.b64encode(content).decode("utf-8")
self._client.file.write_file(file=path, content=base64_content, encoding="base64")
except Exception as e:
logger.error(f"Failed to update file in sandbox: {e}")
raise
@@ -1,114 +1,114 @@
"""Abstract base class for sandbox provisioning backends."""
from __future__ import annotations
import logging
import time
from abc import ABC, abstractmethod
import requests
from .sandbox_info import SandboxInfo
logger = logging.getLogger(__name__)
def wait_for_sandbox_ready(sandbox_url: str, timeout: int = 30) -> bool:
"""Poll sandbox health endpoint until ready or timeout.
Args:
sandbox_url: URL of the sandbox (e.g. http://k3s:30001).
timeout: Maximum time to wait in seconds.
Returns:
True if sandbox is ready, False otherwise.
"""
start_time = time.time()
while time.time() - start_time < timeout:
try:
response = requests.get(f"{sandbox_url}/v1/sandbox", timeout=5)
if response.status_code == 200:
return True
except requests.exceptions.RequestException:
pass
time.sleep(1)
return False
class SandboxBackend(ABC):
"""Abstract base for sandbox provisioning backends.
Two implementations:
- LocalContainerBackend: starts Docker/Apple Container locally, manages ports
- RemoteSandboxBackend: connects to a pre-existing URL (K8s service, external)
"""
@abstractmethod
def create(self, thread_id: str, sandbox_id: str, extra_mounts: list[tuple[str, str, bool]] | None = None) -> SandboxInfo:
"""Create/provision a new sandbox.
Args:
thread_id: Thread ID for which the sandbox is being created. Useful for backends that want to organize sandboxes by thread.
sandbox_id: Deterministic sandbox identifier.
extra_mounts: Additional volume mounts as (host_path, container_path, read_only) tuples.
Ignored by backends that don't manage containers (e.g., remote).
Returns:
SandboxInfo with connection details.
"""
...
@abstractmethod
def destroy(self, info: SandboxInfo) -> None:
"""Destroy/cleanup a sandbox and release its resources.
Args:
info: The sandbox metadata to destroy.
"""
...
@abstractmethod
def is_alive(self, info: SandboxInfo) -> bool:
"""Quick check whether a sandbox is still alive.
This should be a lightweight check (e.g., container inspect)
rather than a full health check.
Args:
info: The sandbox metadata to check.
Returns:
True if the sandbox appears to be alive.
"""
...
@abstractmethod
def discover(self, sandbox_id: str) -> SandboxInfo | None:
"""Try to discover an existing sandbox by its deterministic ID.
Used for cross-process recovery: when another process started a sandbox,
this process can discover it by the deterministic container name or URL.
Args:
sandbox_id: The deterministic sandbox ID to look for.
Returns:
SandboxInfo if found and healthy, None otherwise.
"""
...
def list_running(self) -> list[SandboxInfo]:
"""Enumerate all running sandboxes managed by this backend.
Used for startup reconciliation: when the process restarts, it needs
to discover containers started by previous processes so they can be
adopted into the warm pool or destroyed if idle too long.
The default implementation returns an empty list, which is correct
for backends that don't manage local containers (e.g., RemoteSandboxBackend
delegates lifecycle to the provisioner which handles its own cleanup).
Returns:
A list of SandboxInfo for all currently running sandboxes.
"""
return []
"""Abstract base class for sandbox provisioning backends."""
from __future__ import annotations
import logging
import time
from abc import ABC, abstractmethod
import requests
from .sandbox_info import SandboxInfo
logger = logging.getLogger(__name__)
def wait_for_sandbox_ready(sandbox_url: str, timeout: int = 30) -> bool:
"""Poll sandbox health endpoint until ready or timeout.
Args:
sandbox_url: URL of the sandbox (e.g. http://k3s:30001).
timeout: Maximum time to wait in seconds.
Returns:
True if sandbox is ready, False otherwise.
"""
start_time = time.time()
while time.time() - start_time < timeout:
try:
response = requests.get(f"{sandbox_url}/v1/sandbox", timeout=5)
if response.status_code == 200:
return True
except requests.exceptions.RequestException:
pass
time.sleep(1)
return False
class SandboxBackend(ABC):
"""Abstract base for sandbox provisioning backends.
Two implementations:
- LocalContainerBackend: starts Docker/Apple Container locally, manages ports
- RemoteSandboxBackend: connects to a pre-existing URL (K8s service, external)
"""
@abstractmethod
def create(self, thread_id: str, sandbox_id: str, extra_mounts: list[tuple[str, str, bool]] | None = None) -> SandboxInfo:
"""Create/provision a new sandbox.
Args:
thread_id: Thread ID for which the sandbox is being created. Useful for backends that want to organize sandboxes by thread.
sandbox_id: Deterministic sandbox identifier.
extra_mounts: Additional volume mounts as (host_path, container_path, read_only) tuples.
Ignored by backends that don't manage containers (e.g., remote).
Returns:
SandboxInfo with connection details.
"""
...
@abstractmethod
def destroy(self, info: SandboxInfo) -> None:
"""Destroy/cleanup a sandbox and release its resources.
Args:
info: The sandbox metadata to destroy.
"""
...
@abstractmethod
def is_alive(self, info: SandboxInfo) -> bool:
"""Quick check whether a sandbox is still alive.
This should be a lightweight check (e.g., container inspect)
rather than a full health check.
Args:
info: The sandbox metadata to check.
Returns:
True if the sandbox appears to be alive.
"""
...
@abstractmethod
def discover(self, sandbox_id: str) -> SandboxInfo | None:
"""Try to discover an existing sandbox by its deterministic ID.
Used for cross-process recovery: when another process started a sandbox,
this process can discover it by the deterministic container name or URL.
Args:
sandbox_id: The deterministic sandbox ID to look for.
Returns:
SandboxInfo if found and healthy, None otherwise.
"""
...
def list_running(self) -> list[SandboxInfo]:
"""Enumerate all running sandboxes managed by this backend.
Used for startup reconciliation: when the process restarts, it needs
to discover containers started by previous processes so they can be
adopted into the warm pool or destroyed if idle too long.
The default implementation returns an empty list, which is correct
for backends that don't manage local containers (e.g., RemoteSandboxBackend
delegates lifecycle to the provisioner which handles its own cleanup).
Returns:
A list of SandboxInfo for all currently running sandboxes.
"""
return []
@@ -1,156 +1,156 @@
"""Remote sandbox backend — delegates Pod lifecycle to the provisioner service.
The provisioner dynamically creates per-sandbox-id Pods + NodePort Services
in k3s. The backend accesses sandbox pods directly via ``k3s:{NodePort}``.
Architecture:
┌────────────┐ HTTP ┌─────────────┐ K8s API ┌──────────┐
│ this file │ ──────▸ │ provisioner │ ────────▸ │ k3s │
│ (backend) │ │ :8002 │ │ :6443 │
└────────────┘ └─────────────┘ └─────┬────┘
│ creates
┌─────────────┐ ┌─────▼──────┐
│ backend │ ────────▸ │ sandbox │
│ │ direct │ Pod(s) │
└─────────────┘ k3s:NPort └────────────┘
"""
from __future__ import annotations
import logging
import requests
from .backend import SandboxBackend
from .sandbox_info import SandboxInfo
logger = logging.getLogger(__name__)
class RemoteSandboxBackend(SandboxBackend):
"""Backend that delegates sandbox lifecycle to the provisioner service.
All Pod creation, destruction, and discovery are handled by the
provisioner. This backend is a thin HTTP client.
Typical config.yaml::
sandbox:
use: deerflow.community.aio_sandbox:AioSandboxProvider
provisioner_url: http://provisioner:8002
"""
def __init__(self, provisioner_url: str):
"""Initialize with the provisioner service URL.
Args:
provisioner_url: URL of the provisioner service
(e.g., ``http://provisioner:8002``).
"""
self._provisioner_url = provisioner_url.rstrip("/")
@property
def provisioner_url(self) -> str:
return self._provisioner_url
# ── SandboxBackend interface ──────────────────────────────────────────
def create(
self,
thread_id: str,
sandbox_id: str,
extra_mounts: list[tuple[str, str, bool]] | None = None,
) -> SandboxInfo:
"""Create a sandbox Pod + Service via the provisioner.
Calls ``POST /api/sandboxes`` which creates a dedicated Pod +
NodePort Service in k3s.
"""
return self._provisioner_create(thread_id, sandbox_id, extra_mounts)
def destroy(self, info: SandboxInfo) -> None:
"""Destroy a sandbox Pod + Service via the provisioner."""
self._provisioner_destroy(info.sandbox_id)
def is_alive(self, info: SandboxInfo) -> bool:
"""Check whether the sandbox Pod is running."""
return self._provisioner_is_alive(info.sandbox_id)
def discover(self, sandbox_id: str) -> SandboxInfo | None:
"""Discover an existing sandbox via the provisioner.
Calls ``GET /api/sandboxes/{sandbox_id}`` and returns info if
the Pod exists.
"""
return self._provisioner_discover(sandbox_id)
# ── Provisioner API calls ─────────────────────────────────────────────
def _provisioner_create(self, thread_id: str, sandbox_id: str, extra_mounts: list[tuple[str, str, bool]] | None = None) -> SandboxInfo:
"""POST /api/sandboxes → create Pod + Service."""
try:
resp = requests.post(
f"{self._provisioner_url}/api/sandboxes",
json={
"sandbox_id": sandbox_id,
"thread_id": thread_id,
},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
logger.info(f"Provisioner created sandbox {sandbox_id}: sandbox_url={data['sandbox_url']}")
return SandboxInfo(
sandbox_id=sandbox_id,
sandbox_url=data["sandbox_url"],
)
except requests.RequestException as exc:
logger.error(f"Provisioner create failed for {sandbox_id}: {exc}")
raise RuntimeError(f"Provisioner create failed: {exc}") from exc
def _provisioner_destroy(self, sandbox_id: str) -> None:
"""DELETE /api/sandboxes/{sandbox_id} → destroy Pod + Service."""
try:
resp = requests.delete(
f"{self._provisioner_url}/api/sandboxes/{sandbox_id}",
timeout=15,
)
if resp.ok:
logger.info(f"Provisioner destroyed sandbox {sandbox_id}")
else:
logger.warning(f"Provisioner destroy returned {resp.status_code}: {resp.text}")
except requests.RequestException as exc:
logger.warning(f"Provisioner destroy failed for {sandbox_id}: {exc}")
def _provisioner_is_alive(self, sandbox_id: str) -> bool:
"""GET /api/sandboxes/{sandbox_id} → check Pod phase."""
try:
resp = requests.get(
f"{self._provisioner_url}/api/sandboxes/{sandbox_id}",
timeout=10,
)
if resp.ok:
data = resp.json()
return data.get("status") == "Running"
return False
except requests.RequestException:
return False
def _provisioner_discover(self, sandbox_id: str) -> SandboxInfo | None:
"""GET /api/sandboxes/{sandbox_id} → discover existing sandbox."""
try:
resp = requests.get(
f"{self._provisioner_url}/api/sandboxes/{sandbox_id}",
timeout=10,
)
if resp.status_code == 404:
return None
resp.raise_for_status()
data = resp.json()
return SandboxInfo(
sandbox_id=sandbox_id,
sandbox_url=data["sandbox_url"],
)
except requests.RequestException as exc:
logger.debug(f"Provisioner discover failed for {sandbox_id}: {exc}")
return None
"""Remote sandbox backend — delegates Pod lifecycle to the provisioner service.
The provisioner dynamically creates per-sandbox-id Pods + NodePort Services
in k3s. The backend accesses sandbox pods directly via ``k3s:{NodePort}``.
Architecture:
┌────────────┐ HTTP ┌─────────────┐ K8s API ┌──────────┐
│ this file │ ──────▸ │ provisioner │ ────────▸ │ k3s │
│ (backend) │ │ :8002 │ │ :6443 │
└────────────┘ └─────────────┘ └─────┬────┘
│ creates
┌─────────────┐ ┌─────▼──────┐
│ backend │ ────────▸ │ sandbox │
│ │ direct │ Pod(s) │
└─────────────┘ k3s:NPort └────────────┘
"""
from __future__ import annotations
import logging
import requests
from .backend import SandboxBackend
from .sandbox_info import SandboxInfo
logger = logging.getLogger(__name__)
class RemoteSandboxBackend(SandboxBackend):
"""Backend that delegates sandbox lifecycle to the provisioner service.
All Pod creation, destruction, and discovery are handled by the
provisioner. This backend is a thin HTTP client.
Typical config.yaml::
sandbox:
use: deerflow.community.aio_sandbox:AioSandboxProvider
provisioner_url: http://provisioner:8002
"""
def __init__(self, provisioner_url: str):
"""Initialize with the provisioner service URL.
Args:
provisioner_url: URL of the provisioner service
(e.g., ``http://provisioner:8002``).
"""
self._provisioner_url = provisioner_url.rstrip("/")
@property
def provisioner_url(self) -> str:
return self._provisioner_url
# ── SandboxBackend interface ──────────────────────────────────────────
def create(
self,
thread_id: str,
sandbox_id: str,
extra_mounts: list[tuple[str, str, bool]] | None = None,
) -> SandboxInfo:
"""Create a sandbox Pod + Service via the provisioner.
Calls ``POST /api/sandboxes`` which creates a dedicated Pod +
NodePort Service in k3s.
"""
return self._provisioner_create(thread_id, sandbox_id, extra_mounts)
def destroy(self, info: SandboxInfo) -> None:
"""Destroy a sandbox Pod + Service via the provisioner."""
self._provisioner_destroy(info.sandbox_id)
def is_alive(self, info: SandboxInfo) -> bool:
"""Check whether the sandbox Pod is running."""
return self._provisioner_is_alive(info.sandbox_id)
def discover(self, sandbox_id: str) -> SandboxInfo | None:
"""Discover an existing sandbox via the provisioner.
Calls ``GET /api/sandboxes/{sandbox_id}`` and returns info if
the Pod exists.
"""
return self._provisioner_discover(sandbox_id)
# ── Provisioner API calls ─────────────────────────────────────────────
def _provisioner_create(self, thread_id: str, sandbox_id: str, extra_mounts: list[tuple[str, str, bool]] | None = None) -> SandboxInfo:
"""POST /api/sandboxes → create Pod + Service."""
try:
resp = requests.post(
f"{self._provisioner_url}/api/sandboxes",
json={
"sandbox_id": sandbox_id,
"thread_id": thread_id,
},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
logger.info(f"Provisioner created sandbox {sandbox_id}: sandbox_url={data['sandbox_url']}")
return SandboxInfo(
sandbox_id=sandbox_id,
sandbox_url=data["sandbox_url"],
)
except requests.RequestException as exc:
logger.error(f"Provisioner create failed for {sandbox_id}: {exc}")
raise RuntimeError(f"Provisioner create failed: {exc}") from exc
def _provisioner_destroy(self, sandbox_id: str) -> None:
"""DELETE /api/sandboxes/{sandbox_id} → destroy Pod + Service."""
try:
resp = requests.delete(
f"{self._provisioner_url}/api/sandboxes/{sandbox_id}",
timeout=15,
)
if resp.ok:
logger.info(f"Provisioner destroyed sandbox {sandbox_id}")
else:
logger.warning(f"Provisioner destroy returned {resp.status_code}: {resp.text}")
except requests.RequestException as exc:
logger.warning(f"Provisioner destroy failed for {sandbox_id}: {exc}")
def _provisioner_is_alive(self, sandbox_id: str) -> bool:
"""GET /api/sandboxes/{sandbox_id} → check Pod phase."""
try:
resp = requests.get(
f"{self._provisioner_url}/api/sandboxes/{sandbox_id}",
timeout=10,
)
if resp.ok:
data = resp.json()
return data.get("status") == "Running"
return False
except requests.RequestException:
return False
def _provisioner_discover(self, sandbox_id: str) -> SandboxInfo | None:
"""GET /api/sandboxes/{sandbox_id} → discover existing sandbox."""
try:
resp = requests.get(
f"{self._provisioner_url}/api/sandboxes/{sandbox_id}",
timeout=10,
)
if resp.status_code == 404:
return None
resp.raise_for_status()
data = resp.json()
return SandboxInfo(
sandbox_id=sandbox_id,
sandbox_url=data["sandbox_url"],
)
except requests.RequestException as exc:
logger.debug(f"Provisioner discover failed for {sandbox_id}: {exc}")
return None
@@ -1,41 +1,41 @@
"""Sandbox metadata for cross-process discovery and state persistence."""
from __future__ import annotations
import time
from dataclasses import dataclass, field
@dataclass
class SandboxInfo:
"""Persisted sandbox metadata that enables cross-process discovery.
This dataclass holds all the information needed to reconnect to an
existing sandbox from a different process (e.g., gateway vs langgraph,
multiple workers, or across K8s pods with shared storage).
"""
sandbox_id: str
sandbox_url: str # e.g. http://localhost:8080 or http://k3s:30001
container_name: str | None = None # Only for local container backend
container_id: str | None = None # Only for local container backend
created_at: float = field(default_factory=time.time)
def to_dict(self) -> dict:
return {
"sandbox_id": self.sandbox_id,
"sandbox_url": self.sandbox_url,
"container_name": self.container_name,
"container_id": self.container_id,
"created_at": self.created_at,
}
@classmethod
def from_dict(cls, data: dict) -> SandboxInfo:
return cls(
sandbox_id=data["sandbox_id"],
sandbox_url=data.get("sandbox_url", data.get("base_url", "")),
container_name=data.get("container_name"),
container_id=data.get("container_id"),
created_at=data.get("created_at", time.time()),
)
"""Sandbox metadata for cross-process discovery and state persistence."""
from __future__ import annotations
import time
from dataclasses import dataclass, field
@dataclass
class SandboxInfo:
"""Persisted sandbox metadata that enables cross-process discovery.
This dataclass holds all the information needed to reconnect to an
existing sandbox from a different process (e.g., gateway vs langgraph,
multiple workers, or across K8s pods with shared storage).
"""
sandbox_id: str
sandbox_url: str # e.g. http://localhost:8080 or http://k3s:30001
container_name: str | None = None # Only for local container backend
container_id: str | None = None # Only for local container backend
created_at: float = field(default_factory=time.time)
def to_dict(self) -> dict:
return {
"sandbox_id": self.sandbox_id,
"sandbox_url": self.sandbox_url,
"container_name": self.container_name,
"container_id": self.container_id,
"created_at": self.created_at,
}
@classmethod
def from_dict(cls, data: dict) -> SandboxInfo:
return cls(
sandbox_id=data["sandbox_id"],
sandbox_url=data.get("sandbox_url", data.get("base_url", "")),
container_name=data.get("container_name"),
container_id=data.get("container_id"),
created_at=data.get("created_at", time.time()),
)
@@ -1,3 +1,3 @@
from .tools import web_search_tool
__all__ = ["web_search_tool"]
from .tools import web_search_tool
__all__ = ["web_search_tool"]
@@ -1,95 +1,95 @@
"""
Web Search Tool - Search the web using DuckDuckGo (no API key required).
"""
import json
import logging
from langchain.tools import tool
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
def _search_text(
query: str,
max_results: int = 5,
region: str = "wt-wt",
safesearch: str = "moderate",
) -> list[dict]:
"""
Execute text search using DuckDuckGo.
Args:
query: Search keywords
max_results: Maximum number of results
region: Search region
safesearch: Safe search level
Returns:
List of search results
"""
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs library not installed. Run: pip install ddgs")
return []
ddgs = DDGS(timeout=30)
try:
results = ddgs.text(
query,
region=region,
safesearch=safesearch,
max_results=max_results,
)
return list(results) if results else []
except Exception as e:
logger.error(f"Failed to search web: {e}")
return []
@tool("web_search", parse_docstring=True)
def web_search_tool(
query: str,
max_results: int = 5,
) -> str:
"""Search the web for information. Use this tool to find current information, news, articles, and facts from the internet.
Args:
query: Search keywords describing what you want to find. Be specific for better results.
max_results: Maximum number of results to return. Default is 5.
"""
config = get_app_config().get_tool_config("web_search")
# Override max_results from config if set
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results", max_results)
results = _search_text(
query=query,
max_results=max_results,
)
if not results:
return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False)
normalized_results = [
{
"title": r.get("title", ""),
"url": r.get("href", r.get("link", "")),
"content": r.get("body", r.get("snippet", "")),
}
for r in results
]
output = {
"query": query,
"total_results": len(normalized_results),
"results": normalized_results,
}
return json.dumps(output, indent=2, ensure_ascii=False)
"""
Web Search Tool - Search the web using DuckDuckGo (no API key required).
"""
import json
import logging
from langchain.tools import tool
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
def _search_text(
query: str,
max_results: int = 5,
region: str = "wt-wt",
safesearch: str = "moderate",
) -> list[dict]:
"""
Execute text search using DuckDuckGo.
Args:
query: Search keywords
max_results: Maximum number of results
region: Search region
safesearch: Safe search level
Returns:
List of search results
"""
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs library not installed. Run: pip install ddgs")
return []
ddgs = DDGS(timeout=30)
try:
results = ddgs.text(
query,
region=region,
safesearch=safesearch,
max_results=max_results,
)
return list(results) if results else []
except Exception as e:
logger.error(f"Failed to search web: {e}")
return []
@tool("web_search", parse_docstring=True)
def web_search_tool(
query: str,
max_results: int = 5,
) -> str:
"""Search the web for information. Use this tool to find current information, news, articles, and facts from the internet.
Args:
query: Search keywords describing what you want to find. Be specific for better results.
max_results: Maximum number of results to return. Default is 5.
"""
config = get_app_config().get_tool_config("web_search")
# Override max_results from config if set
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results", max_results)
results = _search_text(
query=query,
max_results=max_results,
)
if not results:
return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False)
normalized_results = [
{
"title": r.get("title", ""),
"url": r.get("href", r.get("link", "")),
"content": r.get("body", r.get("snippet", "")),
}
for r in results
]
output = {
"query": query,
"total_results": len(normalized_results),
"results": normalized_results,
}
return json.dumps(output, indent=2, ensure_ascii=False)
@@ -1,79 +1,79 @@
import json
from exa_py import Exa
from langchain.tools import tool
from deerflow.config import get_app_config
def _get_exa_client(tool_name: str = "web_search") -> Exa:
config = get_app_config().get_tool_config(tool_name)
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
return Exa(api_key=api_key)
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
try:
config = get_app_config().get_tool_config("web_search")
max_results = 5
search_type = "auto"
contents_max_characters = 1000
if config is not None:
max_results = config.model_extra.get("max_results", max_results)
search_type = config.model_extra.get("search_type", search_type)
contents_max_characters = config.model_extra.get("contents_max_characters", contents_max_characters)
client = _get_exa_client()
res = client.search(
query,
type=search_type,
num_results=max_results,
contents={"highlights": {"max_characters": contents_max_characters}},
)
normalized_results = [
{
"title": result.title or "",
"url": result.url or "",
"snippet": "\n".join(result.highlights) if result.highlights else "",
}
for result in res.results
]
json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
return json_results
except Exception as e:
return f"Error: {str(e)}"
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
try:
client = _get_exa_client("web_fetch")
res = client.get_contents([url], text={"max_characters": 4096})
if res.results:
result = res.results[0]
title = result.title or "Untitled"
text = result.text or ""
return f"# {title}\n\n{text[:4096]}"
else:
return "Error: No results found"
except Exception as e:
return f"Error: {str(e)}"
import json
from exa_py import Exa
from langchain.tools import tool
from deerflow.config import get_app_config
def _get_exa_client(tool_name: str = "web_search") -> Exa:
config = get_app_config().get_tool_config(tool_name)
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
return Exa(api_key=api_key)
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
try:
config = get_app_config().get_tool_config("web_search")
max_results = 5
search_type = "auto"
contents_max_characters = 1000
if config is not None:
max_results = config.model_extra.get("max_results", max_results)
search_type = config.model_extra.get("search_type", search_type)
contents_max_characters = config.model_extra.get("contents_max_characters", contents_max_characters)
client = _get_exa_client()
res = client.search(
query,
type=search_type,
num_results=max_results,
contents={"highlights": {"max_characters": contents_max_characters}},
)
normalized_results = [
{
"title": result.title or "",
"url": result.url or "",
"snippet": "\n".join(result.highlights) if result.highlights else "",
}
for result in res.results
]
json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
return json_results
except Exception as e:
return f"Error: {str(e)}"
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
try:
client = _get_exa_client("web_fetch")
res = client.get_contents([url], text={"max_characters": 4096})
if res.results:
result = res.results[0]
title = result.title or "Untitled"
text = result.text or ""
return f"# {title}\n\n{text[:4096]}"
else:
return "Error: No results found"
except Exception as e:
return f"Error: {str(e)}"
@@ -1,73 +1,73 @@
import json
from firecrawl import FirecrawlApp
from langchain.tools import tool
from deerflow.config import get_app_config
def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp:
config = get_app_config().get_tool_config(tool_name)
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
return FirecrawlApp(api_key=api_key) # type: ignore[arg-type]
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
try:
config = get_app_config().get_tool_config("web_search")
max_results = 5
if config is not None:
max_results = config.model_extra.get("max_results", max_results)
client = _get_firecrawl_client("web_search")
result = client.search(query, limit=max_results)
# result.web contains list of SearchResultWeb objects
web_results = result.web or []
normalized_results = [
{
"title": getattr(item, "title", "") or "",
"url": getattr(item, "url", "") or "",
"snippet": getattr(item, "description", "") or "",
}
for item in web_results
]
json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
return json_results
except Exception as e:
return f"Error: {str(e)}"
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
try:
client = _get_firecrawl_client("web_fetch")
result = client.scrape(url, formats=["markdown"])
markdown_content = result.markdown or ""
metadata = result.metadata
title = metadata.title if metadata and metadata.title else "Untitled"
if not markdown_content:
return "Error: No content found"
except Exception as e:
return f"Error: {str(e)}"
return f"# {title}\n\n{markdown_content[:4096]}"
import json
from firecrawl import FirecrawlApp
from langchain.tools import tool
from deerflow.config import get_app_config
def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp:
config = get_app_config().get_tool_config(tool_name)
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
return FirecrawlApp(api_key=api_key) # type: ignore[arg-type]
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
try:
config = get_app_config().get_tool_config("web_search")
max_results = 5
if config is not None:
max_results = config.model_extra.get("max_results", max_results)
client = _get_firecrawl_client("web_search")
result = client.search(query, limit=max_results)
# result.web contains list of SearchResultWeb objects
web_results = result.web or []
normalized_results = [
{
"title": getattr(item, "title", "") or "",
"url": getattr(item, "url", "") or "",
"snippet": getattr(item, "description", "") or "",
}
for item in web_results
]
json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
return json_results
except Exception as e:
return f"Error: {str(e)}"
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
try:
client = _get_firecrawl_client("web_fetch")
result = client.scrape(url, formats=["markdown"])
markdown_content = result.markdown or ""
metadata = result.metadata
title = metadata.title if metadata and metadata.title else "Untitled"
if not markdown_content:
return "Error: No content found"
except Exception as e:
return f"Error: {str(e)}"
return f"# {title}\n\n{markdown_content[:4096]}"
@@ -1,3 +1,3 @@
from .tools import image_search_tool
__all__ = ["image_search_tool"]
from .tools import image_search_tool
__all__ = ["image_search_tool"]
@@ -1,135 +1,135 @@
"""
Image Search Tool - Search images using DuckDuckGo for reference in image generation.
"""
import json
import logging
from langchain.tools import tool
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
def _search_images(
query: str,
max_results: int = 5,
region: str = "wt-wt",
safesearch: str = "moderate",
size: str | None = None,
color: str | None = None,
type_image: str | None = None,
layout: str | None = None,
license_image: str | None = None,
) -> list[dict]:
"""
Execute image search using DuckDuckGo.
Args:
query: Search keywords
max_results: Maximum number of results
region: Search region
safesearch: Safe search level
size: Image size (Small/Medium/Large/Wallpaper)
color: Color filter
type_image: Image type (photo/clipart/gif/transparent/line)
layout: Layout (Square/Tall/Wide)
license_image: License filter
Returns:
List of search results
"""
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs library not installed. Run: pip install ddgs")
return []
ddgs = DDGS(timeout=30)
try:
kwargs = {
"region": region,
"safesearch": safesearch,
"max_results": max_results,
}
if size:
kwargs["size"] = size
if color:
kwargs["color"] = color
if type_image:
kwargs["type_image"] = type_image
if layout:
kwargs["layout"] = layout
if license_image:
kwargs["license_image"] = license_image
results = ddgs.images(query, **kwargs)
return list(results) if results else []
except Exception as e:
logger.error(f"Failed to search images: {e}")
return []
@tool("image_search", parse_docstring=True)
def image_search_tool(
query: str,
max_results: int = 5,
size: str | None = None,
type_image: str | None = None,
layout: str | None = None,
) -> str:
"""Search for images online. Use this tool BEFORE image generation to find reference images for characters, portraits, objects, scenes, or any content requiring visual accuracy.
**When to use:**
- Before generating character/portrait images: search for similar poses, expressions, styles
- Before generating specific objects/products: search for accurate visual references
- Before generating scenes/locations: search for architectural or environmental references
- Before generating fashion/clothing: search for style and detail references
The returned image URLs can be used as reference images in image generation to significantly improve quality.
Args:
query: Search keywords describing the images you want to find. Be specific for better results (e.g., "Japanese woman street photography 1990s" instead of just "woman").
max_results: Maximum number of images to return. Default is 5.
size: Image size filter. Options: "Small", "Medium", "Large", "Wallpaper". Use "Large" for reference images.
type_image: Image type filter. Options: "photo", "clipart", "gif", "transparent", "line". Use "photo" for realistic references.
layout: Layout filter. Options: "Square", "Tall", "Wide". Choose based on your generation needs.
"""
config = get_app_config().get_tool_config("image_search")
# Override max_results from config if set
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results", max_results)
results = _search_images(
query=query,
max_results=max_results,
size=size,
type_image=type_image,
layout=layout,
)
if not results:
return json.dumps({"error": "No images found", "query": query}, ensure_ascii=False)
normalized_results = [
{
"title": r.get("title", ""),
"image_url": r.get("thumbnail", ""),
"thumbnail_url": r.get("thumbnail", ""),
}
for r in results
]
output = {
"query": query,
"total_results": len(normalized_results),
"results": normalized_results,
"usage_hint": "Use the 'image_url' values as reference images in image generation. Download them first if needed.",
}
return json.dumps(output, indent=2, ensure_ascii=False)
"""
Image Search Tool - Search images using DuckDuckGo for reference in image generation.
"""
import json
import logging
from langchain.tools import tool
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
def _search_images(
query: str,
max_results: int = 5,
region: str = "wt-wt",
safesearch: str = "moderate",
size: str | None = None,
color: str | None = None,
type_image: str | None = None,
layout: str | None = None,
license_image: str | None = None,
) -> list[dict]:
"""
Execute image search using DuckDuckGo.
Args:
query: Search keywords
max_results: Maximum number of results
region: Search region
safesearch: Safe search level
size: Image size (Small/Medium/Large/Wallpaper)
color: Color filter
type_image: Image type (photo/clipart/gif/transparent/line)
layout: Layout (Square/Tall/Wide)
license_image: License filter
Returns:
List of search results
"""
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs library not installed. Run: pip install ddgs")
return []
ddgs = DDGS(timeout=30)
try:
kwargs = {
"region": region,
"safesearch": safesearch,
"max_results": max_results,
}
if size:
kwargs["size"] = size
if color:
kwargs["color"] = color
if type_image:
kwargs["type_image"] = type_image
if layout:
kwargs["layout"] = layout
if license_image:
kwargs["license_image"] = license_image
results = ddgs.images(query, **kwargs)
return list(results) if results else []
except Exception as e:
logger.error(f"Failed to search images: {e}")
return []
@tool("image_search", parse_docstring=True)
def image_search_tool(
query: str,
max_results: int = 5,
size: str | None = None,
type_image: str | None = None,
layout: str | None = None,
) -> str:
"""Search for images online. Use this tool BEFORE image generation to find reference images for characters, portraits, objects, scenes, or any content requiring visual accuracy.
**When to use:**
- Before generating character/portrait images: search for similar poses, expressions, styles
- Before generating specific objects/products: search for accurate visual references
- Before generating scenes/locations: search for architectural or environmental references
- Before generating fashion/clothing: search for style and detail references
The returned image URLs can be used as reference images in image generation to significantly improve quality.
Args:
query: Search keywords describing the images you want to find. Be specific for better results (e.g., "Japanese woman street photography 1990s" instead of just "woman").
max_results: Maximum number of images to return. Default is 5.
size: Image size filter. Options: "Small", "Medium", "Large", "Wallpaper". Use "Large" for reference images.
type_image: Image type filter. Options: "photo", "clipart", "gif", "transparent", "line". Use "photo" for realistic references.
layout: Layout filter. Options: "Square", "Tall", "Wide". Choose based on your generation needs.
"""
config = get_app_config().get_tool_config("image_search")
# Override max_results from config if set
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results", max_results)
results = _search_images(
query=query,
max_results=max_results,
size=size,
type_image=type_image,
layout=layout,
)
if not results:
return json.dumps({"error": "No images found", "query": query}, ensure_ascii=False)
normalized_results = [
{
"title": r.get("title", ""),
"image_url": r.get("thumbnail", ""),
"thumbnail_url": r.get("thumbnail", ""),
}
for r in results
]
output = {
"query": query,
"total_results": len(normalized_results),
"results": normalized_results,
"usage_hint": "Use the 'image_url' values as reference images in image generation. Download them first if needed.",
}
return json.dumps(output, indent=2, ensure_ascii=False)
@@ -1,404 +1,404 @@
"""Util that calls InfoQuest Search And Fetch API.
In order to set this up, follow instructions at:
https://docs.byteplus.com/en/docs/InfoQuest/What_is_Info_Quest
"""
import json
import logging
import os
from typing import Any
import requests
logger = logging.getLogger(__name__)
class InfoQuestClient:
"""Client for interacting with the InfoQuest web search and fetch API."""
def __init__(self, fetch_time: int = -1, fetch_timeout: int = -1, fetch_navigation_timeout: int = -1, search_time_range: int = -1, image_search_time_range: int = -1, image_size: str = "i"):
logger.info("\n============================================\n🚀 BytePlus InfoQuest Client Initialization 🚀\n============================================")
self.fetch_time = fetch_time
self.fetch_timeout = fetch_timeout
self.fetch_navigation_timeout = fetch_navigation_timeout
self.search_time_range = search_time_range
self.image_search_time_range = image_search_time_range
self.image_size = image_size
self.api_key_set = bool(os.getenv("INFOQUEST_API_KEY"))
if logger.isEnabledFor(logging.DEBUG):
config_details = (
f"\n📋 Configuration Details:\n"
f"├── Fetch time: {fetch_time} {'(Default: No fetch time)' if fetch_time == -1 else '(Custom)'}\n"
f"├── Fetch Timeout: {fetch_timeout} {'(Default: No fetch timeout)' if fetch_timeout == -1 else '(Custom)'}\n"
f"├── Navigation Timeout: {fetch_navigation_timeout} {'(Default: No Navigation Timeout)' if fetch_navigation_timeout == -1 else '(Custom)'}\n"
f"├── Search Time Range: {search_time_range} {'(Default: No Search Time Range)' if search_time_range == -1 else '(Custom)'}\n"
f"├── Image Search Time Range: {image_search_time_range} {'(Default: No Image Search Time Range)' if image_search_time_range == -1 else '(Custom)'}\n"
f"├── Image Size: {image_size} {'(Default: Medium)' if image_size == 'm' else '(Custom)'}\n"
f"└── API Key: {'✅ Configured' if self.api_key_set else '❌ Not set'}"
)
logger.debug(config_details)
logger.debug("\n" + "*" * 70 + "\n")
def fetch(self, url: str, return_format: str = "html") -> str:
if logger.isEnabledFor(logging.DEBUG):
url_truncated = url[:50] + "..." if len(url) > 50 else url
logger.debug(
f"InfoQuest - Fetch API request initiated | "
f"operation=crawl url | "
f"url_truncated={url_truncated} | "
f"has_timeout_filter={self.fetch_timeout > 0} | timeout_filter={self.fetch_timeout} | "
f"has_fetch_time_filter={self.fetch_time > 0} | fetch_time_filter={self.fetch_time} | "
f"has_navigation_timeout_filter={self.fetch_navigation_timeout > 0} | navi_timeout_filter={self.fetch_navigation_timeout} | "
f"request_type=sync"
)
# Prepare headers
headers = self._prepare_headers()
# Prepare request data
data = self._prepare_crawl_request_data(url, return_format)
logger.debug("Sending crawl request to InfoQuest API")
try:
response = requests.post("https://reader.infoquest.bytepluses.com", headers=headers, json=data)
# Check if status code is not 200
if response.status_code != 200:
error_message = f"fetch API returned status {response.status_code}: {response.text}"
logger.debug("InfoQuest Crawler fetch API return status %d: %s for URL: %s", response.status_code, response.text, url)
return f"Error: {error_message}"
# Check for empty response
if not response.text or not response.text.strip():
error_message = "no result found"
logger.debug("InfoQuest Crawler returned empty response for URL: %s", url)
return f"Error: {error_message}"
# Try to parse response as JSON and extract reader_result
try:
response_data = json.loads(response.text)
# Extract reader_result if it exists
if "reader_result" in response_data:
logger.debug("Successfully extracted reader_result from JSON response")
return response_data["reader_result"]
elif "content" in response_data:
# Fallback to content field if reader_result is not available
logger.debug("reader_result missing in JSON response, falling back to content field: %s", response_data["content"])
return response_data["content"]
else:
# If neither field exists, return the original response
logger.warning("Neither reader_result nor content field found in JSON response")
except json.JSONDecodeError:
# If response is not JSON, return the original text
logger.debug("Response is not in JSON format, returning as-is")
return response.text
# Print partial response for debugging
if logger.isEnabledFor(logging.DEBUG):
response_sample = response.text[:200] + ("..." if len(response.text) > 200 else "")
logger.debug("Successfully received response, content length: %d bytes, first 200 chars: %s", len(response.text), response_sample)
return response.text
except Exception as e:
error_message = f"fetch API failed: {str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
@staticmethod
def _prepare_headers() -> dict[str, str]:
"""Prepare request headers."""
headers = {
"Content-Type": "application/json",
}
# Add API key if available
if os.getenv("INFOQUEST_API_KEY"):
headers["Authorization"] = f"Bearer {os.getenv('INFOQUEST_API_KEY')}"
logger.debug("API key added to request headers")
else:
logger.warning("InfoQuest API key is not set. Provide your own key for authentication.")
return headers
def _prepare_crawl_request_data(self, url: str, return_format: str) -> dict[str, Any]:
"""Prepare request data with formatted parameters."""
# Normalize return_format
if return_format and return_format.lower() == "html":
normalized_format = "HTML"
else:
normalized_format = return_format
data = {"url": url, "format": normalized_format}
# Add timeout parameters if set to positive values
timeout_params = {}
if self.fetch_time > 0:
timeout_params["fetch_time"] = self.fetch_time
if self.fetch_timeout > 0:
timeout_params["timeout"] = self.fetch_timeout
if self.fetch_navigation_timeout > 0:
timeout_params["navi_timeout"] = self.fetch_navigation_timeout
# Log applied timeout parameters
if timeout_params:
logger.debug("Applying timeout parameters: %s", timeout_params)
data.update(timeout_params)
return data
def web_search_raw_results(
self,
query: str,
site: str,
output_format: str = "JSON",
) -> dict:
"""Get results from the InfoQuest Web-Search API synchronously."""
headers = self._prepare_headers()
params = {"format": output_format, "query": query}
if self.search_time_range > 0:
params["time_range"] = self.search_time_range
if site != "":
params["site"] = site
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params)
response.raise_for_status()
# Print partial response for debugging
response_json = response.json()
if logger.isEnabledFor(logging.DEBUG):
response_sample = json.dumps(response_json)[:200] + ("..." if len(json.dumps(response_json)) > 200 else "")
logger.debug(f"Search API request completed successfully | service=InfoQuest | status=success | response_sample={response_sample}")
return response_json
@staticmethod
def clean_results(raw_results: list[dict[str, dict[str, dict[str, Any]]]]) -> list[dict]:
"""Clean results from InfoQuest Web-Search API."""
logger.debug("Processing web-search results")
seen_urls = set()
clean_results = []
counts = {"pages": 0, "news": 0}
for content_list in raw_results:
content = content_list["content"]
results = content["results"]
if results.get("organic"):
organic_results = results["organic"]
for result in organic_results:
clean_result = {
"type": "page",
}
if "title" in result:
clean_result["title"] = result["title"]
if "desc" in result:
clean_result["desc"] = result["desc"]
clean_result["snippet"] = result["desc"]
if "url" in result:
clean_result["url"] = result["url"]
url = clean_result["url"]
if isinstance(url, str) and url and url not in seen_urls:
seen_urls.add(url)
clean_results.append(clean_result)
counts["pages"] += 1
if results.get("top_stories"):
news = results["top_stories"]
for obj in news["items"]:
clean_result = {
"type": "news",
}
if "time_frame" in obj:
clean_result["time_frame"] = obj["time_frame"]
if "source" in obj:
clean_result["source"] = obj["source"]
title = obj.get("title")
url = obj.get("url")
if title:
clean_result["title"] = title
if url:
clean_result["url"] = url
if title and isinstance(url, str) and url and url not in seen_urls:
seen_urls.add(url)
clean_results.append(clean_result)
counts["news"] += 1
logger.debug(f"Results processing completed | total_results={len(clean_results)} | pages={counts['pages']} | news_items={counts['news']} | unique_urls={len(seen_urls)}")
return clean_results
def web_search(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> str:
if logger.isEnabledFor(logging.DEBUG):
query_truncated = query[:50] + "..." if len(query) > 50 else query
logger.debug(
f"InfoQuest - Search API request initiated | "
f"operation=search webs | "
f"query_truncated={query_truncated} | "
f"has_time_filter={self.search_time_range > 0} | time_filter={self.search_time_range} | "
f"has_site_filter={bool(site)} | site={site} | "
f"request_type=sync"
)
try:
logger.debug("InfoQuest Web-Search - Executing search with parameters")
raw_results = self.web_search_raw_results(
query,
site,
output_format,
)
if "search_result" in raw_results:
logger.debug("InfoQuest Web-Search - Successfully extracted search_result from JSON response")
results = raw_results["search_result"]
logger.debug("InfoQuest Web-Search - Processing raw search results")
cleaned_results = self.clean_results(results["results"])
result_json = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
logger.debug(f"InfoQuest Web-Search - Search tool execution completed | mode=synchronous | results_count={len(cleaned_results)}")
return result_json
elif "content" in raw_results:
# Fallback to content field if search_result is not available
error_message = "web search API return wrong format"
logger.error("web search API return wrong format, no search_result nor content field found in JSON response, content: %s", raw_results["content"])
return f"Error: {error_message}"
else:
# If neither field exists, return the original response
logger.warning("InfoQuest Web-Search - Neither search_result nor content field found in JSON response")
return json.dumps(raw_results, indent=2, ensure_ascii=False)
except Exception as e:
error_message = f"InfoQuest Web-Search - Search tool execution failed | mode=synchronous | error={str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
@staticmethod
def clean_results_with_image_search(raw_results: list[dict[str, dict[str, dict[str, Any]]]]) -> list[dict]:
"""Clean results from InfoQuest Web-Search API."""
logger.debug("Processing web-search results")
seen_urls = set()
clean_results = []
counts = {"images": 0}
for content_list in raw_results:
content = content_list["content"]
results = content["results"]
if results.get("images_results"):
images_results = results["images_results"]
for result in images_results:
clean_result = {}
if "original" in result:
clean_result["image_url"] = result["original"]
url = clean_result["image_url"]
if isinstance(url, str) and url and url not in seen_urls:
seen_urls.add(url)
clean_results.append(clean_result)
counts["images"] += 1
if "title" in result:
clean_result["title"] = result["title"]
logger.debug(f"Results processing completed | total_results={len(clean_results)} | images={counts['images']} | unique_urls={len(seen_urls)}")
return clean_results
def image_search_raw_results(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> dict:
"""Get image search results from the InfoQuest Web-Search API synchronously."""
headers = self._prepare_headers()
params = {"format": output_format, "query": query, "search_type": "Images"}
# Add time_range filter if specified (1-365)
if 1 <= self.image_search_time_range <= 365:
params["time_range"] = self.image_search_time_range
elif self.image_search_time_range > 0:
logger.warning(f"time_range {self.image_search_time_range} is out of valid range (1-365), ignoring")
# Add site filter if specified
if site:
params["site"] = site
# Add image_size filter if specified
if self.image_size and self.image_size in ["l", "m", "i"]:
params["image_size"] = self.image_size
elif self.image_size:
logger.warning(f"image_size {self.image_size} is not valid, must be 'l', 'm', or 'i'")
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params)
response.raise_for_status()
# Print partial response for debugging
response_json = response.json()
if logger.isEnabledFor(logging.DEBUG):
response_sample = json.dumps(response_json)[:200] + ("..." if len(json.dumps(response_json)) > 200 else "")
logger.debug(f"Image Search API request completed successfully | service=InfoQuest | status=success | response_sample={response_sample}")
return response_json
def image_search(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> str:
if logger.isEnabledFor(logging.DEBUG):
query_truncated = query[:50] + "..." if len(query) > 50 else query
logger.debug(
f"InfoQuest - Image Search API request initiated | "
f"operation=search images | "
f"query_truncated={query_truncated} | "
f"has_site_filter={bool(site)} | site={site} | "
f"image_search_time_range={self.image_search_time_range if self.image_search_time_range >= 1 and self.image_search_time_range <= 365 else 'default'} | "
f"image_size={self.image_size} |"
f"request_type=sync"
)
try:
logger.info("InfoQuest Image Search - Executing search with parameters")
raw_results = self.image_search_raw_results(
query,
site,
output_format,
)
if "search_result" in raw_results:
logger.debug("InfoQuest Image Search - Successfully extracted search_result from JSON response")
results = raw_results["search_result"]
logger.debug(f"InfoQuest Image Search - Processing raw image search results: {results}")
cleaned_results = self.clean_results_with_image_search(results["results"])
result_json = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
logger.debug(f"InfoQuest Image Search - Image search tool execution completed | mode=synchronous | results_count={len(cleaned_results)}")
return result_json
elif "content" in raw_results:
# Fallback to content field if search_result is not available
error_message = "image search API return wrong format"
logger.error("image search API return wrong format, no search_result nor content field found in JSON response, content: %s", raw_results["content"])
return f"Error: {error_message}"
else:
# If neither field exists, return the original response
logger.warning("InfoQuest Image Search - Neither search_result nor content field found in JSON response")
return json.dumps(raw_results, indent=2, ensure_ascii=False)
except Exception as e:
error_message = f"InfoQuest Image Search - Image search tool execution failed | mode=synchronous | error={str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
"""Util that calls InfoQuest Search And Fetch API.
In order to set this up, follow instructions at:
https://docs.byteplus.com/en/docs/InfoQuest/What_is_Info_Quest
"""
import json
import logging
import os
from typing import Any
import requests
logger = logging.getLogger(__name__)
class InfoQuestClient:
"""Client for interacting with the InfoQuest web search and fetch API."""
def __init__(self, fetch_time: int = -1, fetch_timeout: int = -1, fetch_navigation_timeout: int = -1, search_time_range: int = -1, image_search_time_range: int = -1, image_size: str = "i"):
logger.info("\n============================================\n🚀 BytePlus InfoQuest Client Initialization 🚀\n============================================")
self.fetch_time = fetch_time
self.fetch_timeout = fetch_timeout
self.fetch_navigation_timeout = fetch_navigation_timeout
self.search_time_range = search_time_range
self.image_search_time_range = image_search_time_range
self.image_size = image_size
self.api_key_set = bool(os.getenv("INFOQUEST_API_KEY"))
if logger.isEnabledFor(logging.DEBUG):
config_details = (
f"\n📋 Configuration Details:\n"
f"├── Fetch time: {fetch_time} {'(Default: No fetch time)' if fetch_time == -1 else '(Custom)'}\n"
f"├── Fetch Timeout: {fetch_timeout} {'(Default: No fetch timeout)' if fetch_timeout == -1 else '(Custom)'}\n"
f"├── Navigation Timeout: {fetch_navigation_timeout} {'(Default: No Navigation Timeout)' if fetch_navigation_timeout == -1 else '(Custom)'}\n"
f"├── Search Time Range: {search_time_range} {'(Default: No Search Time Range)' if search_time_range == -1 else '(Custom)'}\n"
f"├── Image Search Time Range: {image_search_time_range} {'(Default: No Image Search Time Range)' if image_search_time_range == -1 else '(Custom)'}\n"
f"├── Image Size: {image_size} {'(Default: Medium)' if image_size == 'm' else '(Custom)'}\n"
f"└── API Key: {'✅ Configured' if self.api_key_set else '❌ Not set'}"
)
logger.debug(config_details)
logger.debug("\n" + "*" * 70 + "\n")
def fetch(self, url: str, return_format: str = "html") -> str:
if logger.isEnabledFor(logging.DEBUG):
url_truncated = url[:50] + "..." if len(url) > 50 else url
logger.debug(
f"InfoQuest - Fetch API request initiated | "
f"operation=crawl url | "
f"url_truncated={url_truncated} | "
f"has_timeout_filter={self.fetch_timeout > 0} | timeout_filter={self.fetch_timeout} | "
f"has_fetch_time_filter={self.fetch_time > 0} | fetch_time_filter={self.fetch_time} | "
f"has_navigation_timeout_filter={self.fetch_navigation_timeout > 0} | navi_timeout_filter={self.fetch_navigation_timeout} | "
f"request_type=sync"
)
# Prepare headers
headers = self._prepare_headers()
# Prepare request data
data = self._prepare_crawl_request_data(url, return_format)
logger.debug("Sending crawl request to InfoQuest API")
try:
response = requests.post("https://reader.infoquest.bytepluses.com", headers=headers, json=data)
# Check if status code is not 200
if response.status_code != 200:
error_message = f"fetch API returned status {response.status_code}: {response.text}"
logger.debug("InfoQuest Crawler fetch API return status %d: %s for URL: %s", response.status_code, response.text, url)
return f"Error: {error_message}"
# Check for empty response
if not response.text or not response.text.strip():
error_message = "no result found"
logger.debug("InfoQuest Crawler returned empty response for URL: %s", url)
return f"Error: {error_message}"
# Try to parse response as JSON and extract reader_result
try:
response_data = json.loads(response.text)
# Extract reader_result if it exists
if "reader_result" in response_data:
logger.debug("Successfully extracted reader_result from JSON response")
return response_data["reader_result"]
elif "content" in response_data:
# Fallback to content field if reader_result is not available
logger.debug("reader_result missing in JSON response, falling back to content field: %s", response_data["content"])
return response_data["content"]
else:
# If neither field exists, return the original response
logger.warning("Neither reader_result nor content field found in JSON response")
except json.JSONDecodeError:
# If response is not JSON, return the original text
logger.debug("Response is not in JSON format, returning as-is")
return response.text
# Print partial response for debugging
if logger.isEnabledFor(logging.DEBUG):
response_sample = response.text[:200] + ("..." if len(response.text) > 200 else "")
logger.debug("Successfully received response, content length: %d bytes, first 200 chars: %s", len(response.text), response_sample)
return response.text
except Exception as e:
error_message = f"fetch API failed: {str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
@staticmethod
def _prepare_headers() -> dict[str, str]:
"""Prepare request headers."""
headers = {
"Content-Type": "application/json",
}
# Add API key if available
if os.getenv("INFOQUEST_API_KEY"):
headers["Authorization"] = f"Bearer {os.getenv('INFOQUEST_API_KEY')}"
logger.debug("API key added to request headers")
else:
logger.warning("InfoQuest API key is not set. Provide your own key for authentication.")
return headers
def _prepare_crawl_request_data(self, url: str, return_format: str) -> dict[str, Any]:
"""Prepare request data with formatted parameters."""
# Normalize return_format
if return_format and return_format.lower() == "html":
normalized_format = "HTML"
else:
normalized_format = return_format
data = {"url": url, "format": normalized_format}
# Add timeout parameters if set to positive values
timeout_params = {}
if self.fetch_time > 0:
timeout_params["fetch_time"] = self.fetch_time
if self.fetch_timeout > 0:
timeout_params["timeout"] = self.fetch_timeout
if self.fetch_navigation_timeout > 0:
timeout_params["navi_timeout"] = self.fetch_navigation_timeout
# Log applied timeout parameters
if timeout_params:
logger.debug("Applying timeout parameters: %s", timeout_params)
data.update(timeout_params)
return data
def web_search_raw_results(
self,
query: str,
site: str,
output_format: str = "JSON",
) -> dict:
"""Get results from the InfoQuest Web-Search API synchronously."""
headers = self._prepare_headers()
params = {"format": output_format, "query": query}
if self.search_time_range > 0:
params["time_range"] = self.search_time_range
if site != "":
params["site"] = site
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params)
response.raise_for_status()
# Print partial response for debugging
response_json = response.json()
if logger.isEnabledFor(logging.DEBUG):
response_sample = json.dumps(response_json)[:200] + ("..." if len(json.dumps(response_json)) > 200 else "")
logger.debug(f"Search API request completed successfully | service=InfoQuest | status=success | response_sample={response_sample}")
return response_json
@staticmethod
def clean_results(raw_results: list[dict[str, dict[str, dict[str, Any]]]]) -> list[dict]:
"""Clean results from InfoQuest Web-Search API."""
logger.debug("Processing web-search results")
seen_urls = set()
clean_results = []
counts = {"pages": 0, "news": 0}
for content_list in raw_results:
content = content_list["content"]
results = content["results"]
if results.get("organic"):
organic_results = results["organic"]
for result in organic_results:
clean_result = {
"type": "page",
}
if "title" in result:
clean_result["title"] = result["title"]
if "desc" in result:
clean_result["desc"] = result["desc"]
clean_result["snippet"] = result["desc"]
if "url" in result:
clean_result["url"] = result["url"]
url = clean_result["url"]
if isinstance(url, str) and url and url not in seen_urls:
seen_urls.add(url)
clean_results.append(clean_result)
counts["pages"] += 1
if results.get("top_stories"):
news = results["top_stories"]
for obj in news["items"]:
clean_result = {
"type": "news",
}
if "time_frame" in obj:
clean_result["time_frame"] = obj["time_frame"]
if "source" in obj:
clean_result["source"] = obj["source"]
title = obj.get("title")
url = obj.get("url")
if title:
clean_result["title"] = title
if url:
clean_result["url"] = url
if title and isinstance(url, str) and url and url not in seen_urls:
seen_urls.add(url)
clean_results.append(clean_result)
counts["news"] += 1
logger.debug(f"Results processing completed | total_results={len(clean_results)} | pages={counts['pages']} | news_items={counts['news']} | unique_urls={len(seen_urls)}")
return clean_results
def web_search(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> str:
if logger.isEnabledFor(logging.DEBUG):
query_truncated = query[:50] + "..." if len(query) > 50 else query
logger.debug(
f"InfoQuest - Search API request initiated | "
f"operation=search webs | "
f"query_truncated={query_truncated} | "
f"has_time_filter={self.search_time_range > 0} | time_filter={self.search_time_range} | "
f"has_site_filter={bool(site)} | site={site} | "
f"request_type=sync"
)
try:
logger.debug("InfoQuest Web-Search - Executing search with parameters")
raw_results = self.web_search_raw_results(
query,
site,
output_format,
)
if "search_result" in raw_results:
logger.debug("InfoQuest Web-Search - Successfully extracted search_result from JSON response")
results = raw_results["search_result"]
logger.debug("InfoQuest Web-Search - Processing raw search results")
cleaned_results = self.clean_results(results["results"])
result_json = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
logger.debug(f"InfoQuest Web-Search - Search tool execution completed | mode=synchronous | results_count={len(cleaned_results)}")
return result_json
elif "content" in raw_results:
# Fallback to content field if search_result is not available
error_message = "web search API return wrong format"
logger.error("web search API return wrong format, no search_result nor content field found in JSON response, content: %s", raw_results["content"])
return f"Error: {error_message}"
else:
# If neither field exists, return the original response
logger.warning("InfoQuest Web-Search - Neither search_result nor content field found in JSON response")
return json.dumps(raw_results, indent=2, ensure_ascii=False)
except Exception as e:
error_message = f"InfoQuest Web-Search - Search tool execution failed | mode=synchronous | error={str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
@staticmethod
def clean_results_with_image_search(raw_results: list[dict[str, dict[str, dict[str, Any]]]]) -> list[dict]:
"""Clean results from InfoQuest Web-Search API."""
logger.debug("Processing web-search results")
seen_urls = set()
clean_results = []
counts = {"images": 0}
for content_list in raw_results:
content = content_list["content"]
results = content["results"]
if results.get("images_results"):
images_results = results["images_results"]
for result in images_results:
clean_result = {}
if "original" in result:
clean_result["image_url"] = result["original"]
url = clean_result["image_url"]
if isinstance(url, str) and url and url not in seen_urls:
seen_urls.add(url)
clean_results.append(clean_result)
counts["images"] += 1
if "title" in result:
clean_result["title"] = result["title"]
logger.debug(f"Results processing completed | total_results={len(clean_results)} | images={counts['images']} | unique_urls={len(seen_urls)}")
return clean_results
def image_search_raw_results(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> dict:
"""Get image search results from the InfoQuest Web-Search API synchronously."""
headers = self._prepare_headers()
params = {"format": output_format, "query": query, "search_type": "Images"}
# Add time_range filter if specified (1-365)
if 1 <= self.image_search_time_range <= 365:
params["time_range"] = self.image_search_time_range
elif self.image_search_time_range > 0:
logger.warning(f"time_range {self.image_search_time_range} is out of valid range (1-365), ignoring")
# Add site filter if specified
if site:
params["site"] = site
# Add image_size filter if specified
if self.image_size and self.image_size in ["l", "m", "i"]:
params["image_size"] = self.image_size
elif self.image_size:
logger.warning(f"image_size {self.image_size} is not valid, must be 'l', 'm', or 'i'")
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params)
response.raise_for_status()
# Print partial response for debugging
response_json = response.json()
if logger.isEnabledFor(logging.DEBUG):
response_sample = json.dumps(response_json)[:200] + ("..." if len(json.dumps(response_json)) > 200 else "")
logger.debug(f"Image Search API request completed successfully | service=InfoQuest | status=success | response_sample={response_sample}")
return response_json
def image_search(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> str:
if logger.isEnabledFor(logging.DEBUG):
query_truncated = query[:50] + "..." if len(query) > 50 else query
logger.debug(
f"InfoQuest - Image Search API request initiated | "
f"operation=search images | "
f"query_truncated={query_truncated} | "
f"has_site_filter={bool(site)} | site={site} | "
f"image_search_time_range={self.image_search_time_range if self.image_search_time_range >= 1 and self.image_search_time_range <= 365 else 'default'} | "
f"image_size={self.image_size} |"
f"request_type=sync"
)
try:
logger.info("InfoQuest Image Search - Executing search with parameters")
raw_results = self.image_search_raw_results(
query,
site,
output_format,
)
if "search_result" in raw_results:
logger.debug("InfoQuest Image Search - Successfully extracted search_result from JSON response")
results = raw_results["search_result"]
logger.debug(f"InfoQuest Image Search - Processing raw image search results: {results}")
cleaned_results = self.clean_results_with_image_search(results["results"])
result_json = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
logger.debug(f"InfoQuest Image Search - Image search tool execution completed | mode=synchronous | results_count={len(cleaned_results)}")
return result_json
elif "content" in raw_results:
# Fallback to content field if search_result is not available
error_message = "image search API return wrong format"
logger.error("image search API return wrong format, no search_result nor content field found in JSON response, content: %s", raw_results["content"])
return f"Error: {error_message}"
else:
# If neither field exists, return the original response
logger.warning("InfoQuest Image Search - Neither search_result nor content field found in JSON response")
return json.dumps(raw_results, indent=2, ensure_ascii=False)
except Exception as e:
error_message = f"InfoQuest Image Search - Image search tool execution failed | mode=synchronous | error={str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
@@ -1,93 +1,93 @@
from langchain.tools import tool
from deerflow.config import get_app_config
from deerflow.utils.readability import ReadabilityExtractor
from .infoquest_client import InfoQuestClient
readability_extractor = ReadabilityExtractor()
def _get_infoquest_client() -> InfoQuestClient:
search_config = get_app_config().get_tool_config("web_search")
search_time_range = -1
if search_config is not None and "search_time_range" in search_config.model_extra:
search_time_range = search_config.model_extra.get("search_time_range")
fetch_config = get_app_config().get_tool_config("web_fetch")
fetch_time = -1
if fetch_config is not None and "fetch_time" in fetch_config.model_extra:
fetch_time = fetch_config.model_extra.get("fetch_time")
fetch_timeout = -1
if fetch_config is not None and "timeout" in fetch_config.model_extra:
fetch_timeout = fetch_config.model_extra.get("timeout")
navigation_timeout = -1
if fetch_config is not None and "navigation_timeout" in fetch_config.model_extra:
navigation_timeout = fetch_config.model_extra.get("navigation_timeout")
image_search_config = get_app_config().get_tool_config("image_search")
image_search_time_range = -1
if image_search_config is not None and "image_search_time_range" in image_search_config.model_extra:
image_search_time_range = image_search_config.model_extra.get("image_search_time_range")
image_size = "i"
if image_search_config is not None and "image_size" in image_search_config.model_extra:
image_size = image_search_config.model_extra.get("image_size")
return InfoQuestClient(
search_time_range=search_time_range,
fetch_timeout=fetch_timeout,
fetch_navigation_timeout=navigation_timeout,
fetch_time=fetch_time,
image_search_time_range=image_search_time_range,
image_size=image_size,
)
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
client = _get_infoquest_client()
return client.web_search(query)
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
client = _get_infoquest_client()
result = client.fetch(url)
if result.startswith("Error: "):
return result
article = readability_extractor.extract_article(result)
return article.to_markdown()[:4096]
@tool("image_search", parse_docstring=True)
def image_search_tool(query: str) -> str:
"""Search for images online. Use this tool BEFORE image generation to find reference images for characters, portraits, objects, scenes, or any content requiring visual accuracy.
**When to use:**
- Before generating character/portrait images: search for similar poses, expressions, styles
- Before generating specific objects/products: search for accurate visual references
- Before generating scenes/locations: search for architectural or environmental references
- Before generating fashion/clothing: search for style and detail references
The returned image URLs can be used as reference images in image generation to significantly improve quality.
Args:
query: The query to search for images.
"""
client = _get_infoquest_client()
return client.image_search(query)
from langchain.tools import tool
from deerflow.config import get_app_config
from deerflow.utils.readability import ReadabilityExtractor
from .infoquest_client import InfoQuestClient
readability_extractor = ReadabilityExtractor()
def _get_infoquest_client() -> InfoQuestClient:
search_config = get_app_config().get_tool_config("web_search")
search_time_range = -1
if search_config is not None and "search_time_range" in search_config.model_extra:
search_time_range = search_config.model_extra.get("search_time_range")
fetch_config = get_app_config().get_tool_config("web_fetch")
fetch_time = -1
if fetch_config is not None and "fetch_time" in fetch_config.model_extra:
fetch_time = fetch_config.model_extra.get("fetch_time")
fetch_timeout = -1
if fetch_config is not None and "timeout" in fetch_config.model_extra:
fetch_timeout = fetch_config.model_extra.get("timeout")
navigation_timeout = -1
if fetch_config is not None and "navigation_timeout" in fetch_config.model_extra:
navigation_timeout = fetch_config.model_extra.get("navigation_timeout")
image_search_config = get_app_config().get_tool_config("image_search")
image_search_time_range = -1
if image_search_config is not None and "image_search_time_range" in image_search_config.model_extra:
image_search_time_range = image_search_config.model_extra.get("image_search_time_range")
image_size = "i"
if image_search_config is not None and "image_size" in image_search_config.model_extra:
image_size = image_search_config.model_extra.get("image_size")
return InfoQuestClient(
search_time_range=search_time_range,
fetch_timeout=fetch_timeout,
fetch_navigation_timeout=navigation_timeout,
fetch_time=fetch_time,
image_search_time_range=image_search_time_range,
image_size=image_size,
)
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
client = _get_infoquest_client()
return client.web_search(query)
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
client = _get_infoquest_client()
result = client.fetch(url)
if result.startswith("Error: "):
return result
article = readability_extractor.extract_article(result)
return article.to_markdown()[:4096]
@tool("image_search", parse_docstring=True)
def image_search_tool(query: str) -> str:
"""Search for images online. Use this tool BEFORE image generation to find reference images for characters, portraits, objects, scenes, or any content requiring visual accuracy.
**When to use:**
- Before generating character/portrait images: search for similar poses, expressions, styles
- Before generating specific objects/products: search for accurate visual references
- Before generating scenes/locations: search for architectural or environmental references
- Before generating fashion/clothing: search for style and detail references
The returned image URLs can be used as reference images in image generation to significantly improve quality.
Args:
query: The query to search for images.
"""
client = _get_infoquest_client()
return client.image_search(query)
@@ -1,43 +1,43 @@
import logging
import os
import httpx
logger = logging.getLogger(__name__)
_api_key_warned = False
class JinaClient:
async def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str:
global _api_key_warned
headers = {
"Content-Type": "application/json",
"X-Return-Format": return_format,
"X-Timeout": str(timeout),
}
if os.getenv("JINA_API_KEY"):
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
elif not _api_key_warned:
_api_key_warned = True
logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.")
data = {"url": url}
try:
async with httpx.AsyncClient() as client:
response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=timeout)
if response.status_code != 200:
error_message = f"Jina API returned status {response.status_code}: {response.text}"
logger.error(error_message)
return f"Error: {error_message}"
if not response.text or not response.text.strip():
error_message = "Jina API returned empty response"
logger.error(error_message)
return f"Error: {error_message}"
return response.text
except Exception as e:
error_message = f"Request to Jina API failed: {type(e).__name__}: {e}"
logger.warning(error_message)
return f"Error: {error_message}"
import logging
import os
import httpx
logger = logging.getLogger(__name__)
_api_key_warned = False
class JinaClient:
async def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str:
global _api_key_warned
headers = {
"Content-Type": "application/json",
"X-Return-Format": return_format,
"X-Timeout": str(timeout),
}
if os.getenv("JINA_API_KEY"):
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
elif not _api_key_warned:
_api_key_warned = True
logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.")
data = {"url": url}
try:
async with httpx.AsyncClient() as client:
response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=timeout)
if response.status_code != 200:
error_message = f"Jina API returned status {response.status_code}: {response.text}"
logger.error(error_message)
return f"Error: {error_message}"
if not response.text or not response.text.strip():
error_message = "Jina API returned empty response"
logger.error(error_message)
return f"Error: {error_message}"
return response.text
except Exception as e:
error_message = f"Request to Jina API failed: {type(e).__name__}: {e}"
logger.warning(error_message)
return f"Error: {error_message}"
@@ -1,32 +1,32 @@
import asyncio
from langchain.tools import tool
from deerflow.community.jina_ai.jina_client import JinaClient
from deerflow.config import get_app_config
from deerflow.utils.readability import ReadabilityExtractor
readability_extractor = ReadabilityExtractor()
@tool("web_fetch", parse_docstring=True)
async def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
jina_client = JinaClient()
timeout = 10
config = get_app_config().get_tool_config("web_fetch")
if config is not None and "timeout" in config.model_extra:
timeout = config.model_extra.get("timeout")
html_content = await jina_client.crawl(url, return_format="html", timeout=timeout)
if isinstance(html_content, str) and html_content.startswith("Error:"):
return html_content
article = await asyncio.to_thread(readability_extractor.extract_article, html_content)
return article.to_markdown()[:4096]
import asyncio
from langchain.tools import tool
from deerflow.community.jina_ai.jina_client import JinaClient
from deerflow.config import get_app_config
from deerflow.utils.readability import ReadabilityExtractor
readability_extractor = ReadabilityExtractor()
@tool("web_fetch", parse_docstring=True)
async def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
jina_client = JinaClient()
timeout = 10
config = get_app_config().get_tool_config("web_fetch")
if config is not None and "timeout" in config.model_extra:
timeout = config.model_extra.get("timeout")
html_content = await jina_client.crawl(url, return_format="html", timeout=timeout)
if isinstance(html_content, str) and html_content.startswith("Error:"):
return html_content
article = await asyncio.to_thread(readability_extractor.extract_article, html_content)
return article.to_markdown()[:4096]
@@ -1,3 +1,3 @@
from .tools import web_search_tool
__all__ = ["web_search_tool"]
from .tools import web_search_tool
__all__ = ["web_search_tool"]
@@ -1,95 +1,95 @@
"""
Web Search Tool - Search the web using Serper (Google Search API).
Serper provides real-time Google Search results via a JSON API.
An API key is required. Sign up at https://serper.dev to get one.
"""
import json
import logging
import os
import httpx
from langchain.tools import tool
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
_SERPER_ENDPOINT = "https://google.serper.dev/search"
_api_key_warned = False
def _get_api_key() -> str | None:
config = get_app_config().get_tool_config("web_search")
if config is not None:
api_key = config.model_extra.get("api_key")
if isinstance(api_key, str) and api_key.strip():
return api_key
return os.getenv("SERPER_API_KEY")
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str, max_results: int = 5) -> str:
"""Search the web for information using Google Search via Serper.
Args:
query: Search keywords describing what you want to find. Be specific for better results.
max_results: Maximum number of search results to return. Default is 5.
"""
global _api_key_warned
config = get_app_config().get_tool_config("web_search")
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results", max_results)
api_key = _get_api_key()
if not api_key:
if not _api_key_warned:
_api_key_warned = True
logger.warning("Serper API key is not set. Set SERPER_API_KEY in your environment or provide api_key in config.yaml. Sign up at https://serper.dev")
return json.dumps(
{"error": "SERPER_API_KEY is not configured", "query": query},
ensure_ascii=False,
)
headers = {
"X-API-KEY": api_key,
"Content-Type": "application/json",
}
payload = {"q": query, "num": max_results}
try:
with httpx.Client(timeout=30) as client:
response = client.post(_SERPER_ENDPOINT, headers=headers, json=payload)
response.raise_for_status()
data = response.json()
except httpx.HTTPStatusError as e:
logger.error(f"Serper API returned HTTP {e.response.status_code}: {e.response.text}")
return json.dumps(
{"error": f"Serper API error: HTTP {e.response.status_code}", "query": query},
ensure_ascii=False,
)
except Exception as e:
logger.error(f"Serper search failed: {type(e).__name__}: {e}")
return json.dumps({"error": str(e), "query": query}, ensure_ascii=False)
organic = data.get("organic", [])
if not organic:
return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False)
normalized_results = [
{
"title": r.get("title", ""),
"url": r.get("link", ""),
"content": r.get("snippet", ""),
}
for r in organic[:max_results]
]
output = {
"query": query,
"total_results": len(normalized_results),
"results": normalized_results,
}
return json.dumps(output, indent=2, ensure_ascii=False)
"""
Web Search Tool - Search the web using Serper (Google Search API).
Serper provides real-time Google Search results via a JSON API.
An API key is required. Sign up at https://serper.dev to get one.
"""
import json
import logging
import os
import httpx
from langchain.tools import tool
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
_SERPER_ENDPOINT = "https://google.serper.dev/search"
_api_key_warned = False
def _get_api_key() -> str | None:
config = get_app_config().get_tool_config("web_search")
if config is not None:
api_key = config.model_extra.get("api_key")
if isinstance(api_key, str) and api_key.strip():
return api_key
return os.getenv("SERPER_API_KEY")
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str, max_results: int = 5) -> str:
"""Search the web for information using Google Search via Serper.
Args:
query: Search keywords describing what you want to find. Be specific for better results.
max_results: Maximum number of search results to return. Default is 5.
"""
global _api_key_warned
config = get_app_config().get_tool_config("web_search")
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results", max_results)
api_key = _get_api_key()
if not api_key:
if not _api_key_warned:
_api_key_warned = True
logger.warning("Serper API key is not set. Set SERPER_API_KEY in your environment or provide api_key in config.yaml. Sign up at https://serper.dev")
return json.dumps(
{"error": "SERPER_API_KEY is not configured", "query": query},
ensure_ascii=False,
)
headers = {
"X-API-KEY": api_key,
"Content-Type": "application/json",
}
payload = {"q": query, "num": max_results}
try:
with httpx.Client(timeout=30) as client:
response = client.post(_SERPER_ENDPOINT, headers=headers, json=payload)
response.raise_for_status()
data = response.json()
except httpx.HTTPStatusError as e:
logger.error(f"Serper API returned HTTP {e.response.status_code}: {e.response.text}")
return json.dumps(
{"error": f"Serper API error: HTTP {e.response.status_code}", "query": query},
ensure_ascii=False,
)
except Exception as e:
logger.error(f"Serper search failed: {type(e).__name__}: {e}")
return json.dumps({"error": str(e), "query": query}, ensure_ascii=False)
organic = data.get("organic", [])
if not organic:
return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False)
normalized_results = [
{
"title": r.get("title", ""),
"url": r.get("link", ""),
"content": r.get("snippet", ""),
}
for r in organic[:max_results]
]
output = {
"query": query,
"total_results": len(normalized_results),
"results": normalized_results,
}
return json.dumps(output, indent=2, ensure_ascii=False)
@@ -1,62 +1,62 @@
import json
from langchain.tools import tool
from tavily import TavilyClient
from deerflow.config import get_app_config
def _get_tavily_client() -> TavilyClient:
config = get_app_config().get_tool_config("web_search")
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
return TavilyClient(api_key=api_key)
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
config = get_app_config().get_tool_config("web_search")
max_results = 5
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results")
client = _get_tavily_client()
res = client.search(query, max_results=max_results)
normalized_results = [
{
"title": result["title"],
"url": result["url"],
"snippet": result["content"],
}
for result in res["results"]
]
json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
return json_results
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
client = _get_tavily_client()
res = client.extract([url])
if "failed_results" in res and len(res["failed_results"]) > 0:
return f"Error: {res['failed_results'][0]['error']}"
elif "results" in res and len(res["results"]) > 0:
result = res["results"][0]
return f"# {result['title']}\n\n{result['raw_content'][:4096]}"
else:
return "Error: No results found"
import json
from langchain.tools import tool
from tavily import TavilyClient
from deerflow.config import get_app_config
def _get_tavily_client() -> TavilyClient:
config = get_app_config().get_tool_config("web_search")
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
return TavilyClient(api_key=api_key)
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
"""Search the web.
Args:
query: The query to search for.
"""
config = get_app_config().get_tool_config("web_search")
max_results = 5
if config is not None and "max_results" in config.model_extra:
max_results = config.model_extra.get("max_results")
client = _get_tavily_client()
res = client.search(query, max_results=max_results)
normalized_results = [
{
"title": result["title"],
"url": result["url"],
"snippet": result["content"],
}
for result in res["results"]
]
json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
return json_results
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
Do NOT add www. to URLs that do NOT have them.
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
Args:
url: The URL to fetch the contents of.
"""
client = _get_tavily_client()
res = client.extract([url])
if "failed_results" in res and len(res["failed_results"]) > 0:
return f"Error: {res['failed_results'][0]['error']}"
elif "results" in res and len(res["results"]) > 0:
result = res["results"][0]
return f"# {result['title']}\n\n{result['raw_content'][:4096]}"
else:
return "Error: No results found"