From e446ccc56700716702977c633b582e218138880f Mon Sep 17 00:00:00 2001 From: likun Date: Thu, 21 May 2026 12:32:07 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=9F=A5=E8=AF=86=E6=9B=B4?= =?UTF-8?q?=E6=96=B0bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../maintenance/knowledge_update.cron.example | 13 ++ .../maintenance/knowledge_update_pipeline.py | 167 +++++++++++++++++- .../maintenance/run_daily_knowledge_update.sh | 39 +++- 3 files changed, 209 insertions(+), 10 deletions(-) diff --git a/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update.cron.example b/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update.cron.example index 88bda4c8..29774b0e 100644 --- a/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update.cron.example +++ b/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update.cron.example @@ -6,5 +6,18 @@ # # Logs: # knowledge/knowledge_generate/maintenance/logs/daily_knowledge_update.log +# +# Optional internal model environment variables: +# KNOWLEDGE_UPDATE_MODEL=INTERNAL-GLM +# KNOWLEDGE_UPDATE_API_KEY=env:INTERNAL_LLM_KEY +# KNOWLEDGE_UPDATE_BASE_URL=http://internal-llm.example/v1 +# KNOWLEDGE_UPDATE_MODEL_NAME=your-internal-model-name +# KNOWLEDGE_UPDATE_HEADERS_JSON={"X-Project":"knowledge-update"} +# +# If using env:INTERNAL_LLM_KEY, export INTERNAL_LLM_KEY in the cron environment +# or put it before the command line. 10 12 * * * mkdir -p /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/logs && /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh >> /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/logs/daily_knowledge_update.log 2>&1 + +# Example with internal model overrides: +# 10 12 * * * export INTERNAL_LLM_KEY="***"; KNOWLEDGE_UPDATE_MODEL=INTERNAL-GLM KNOWLEDGE_UPDATE_API_KEY=env:INTERNAL_LLM_KEY KNOWLEDGE_UPDATE_BASE_URL=http://internal-llm.example/v1 KNOWLEDGE_UPDATE_MODEL_NAME=your-internal-model-name /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh >> /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/logs/daily_knowledge_update.log 2>&1 diff --git a/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py b/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py index 1f2f12ab..bbf4f221 100644 --- a/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py +++ b/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse import json +import os import re import shutil import sys @@ -21,18 +22,21 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from deerflow.client import DeerFlowClient -from deerflow.config.paths import get_paths -from deerflow.runtime.user_context import get_effective_user_id +BACKEND_ROOT = Path(__file__).resolve().parents[3] +KNOWLEDGE_PARENT = Path(__file__).resolve().parents[2] +HARNESS_ROOT = BACKEND_ROOT / "packages" / "harness" +for import_root in (BACKEND_ROOT, HARNESS_ROOT): + import_root_str = str(import_root) + if import_root_str not in sys.path: + sys.path.insert(0, import_root_str) - -BACKEND_ROOT = Path(__file__).resolve().parents[2] -KNOWLEDGE_ROOT = BACKEND_ROOT / "knowledge_base" -MAINTENANCE_ROOT = BACKEND_ROOT / "knowledge_generate" / "maintenance" +KNOWLEDGE_ROOT = KNOWLEDGE_PARENT / "knowledge_base" +MAINTENANCE_ROOT = KNOWLEDGE_PARENT / "knowledge_generate" / "maintenance" RUNS_ROOT = MAINTENANCE_ROOT / "runs" DEFAULT_MODEL = "GLM-5" DEFAULT_SKILLS = {"policy-knowledge", "update-knowledge"} +DEFAULT_CONFIG_PATH = BACKEND_ROOT / "config.yaml" class PatchError(RuntimeError): @@ -292,20 +296,147 @@ def event_to_jsonable(event) -> dict: return {"type": event.type, "data": dict(event.data or {})} +def resolve_secret(value: str | None) -> str | None: + """Resolve direct values or env references. + + Supported forms: + - ``sk-xxx``: returned as-is + - ``$ENV_NAME``: read from environment + - ``env:ENV_NAME``: read from environment + """ + if not value: + return None + value = value.strip() + if value.startswith("env:"): + return os.getenv(value[4:]) + if value.startswith("$"): + return os.getenv(value[1:]) + return value + + +def secret_config_value(value: str) -> str: + """Return the value to write into temp config without expanding env refs.""" + value = value.strip() + if value.startswith("env:"): + return f"${value[4:]}" + return value + + +def parse_header_values(values: list[str] | None) -> dict[str, str]: + """Parse repeated ``--header Key=Value`` options.""" + headers: dict[str, str] = {} + for raw in values or []: + if "=" not in raw: + raise ValueError(f"--header must use Key=Value format, got: {raw}") + key, value = raw.split("=", 1) + key = key.strip() + if not key: + raise ValueError(f"--header key cannot be empty: {raw}") + headers[key] = value.strip() + return headers + + +def parse_headers_json(raw: str | None) -> dict[str, str]: + if not raw: + return {} + data = json.loads(raw) + if not isinstance(data, dict): + raise ValueError("--headers-json must be a JSON object") + return {str(k): str(v) for k, v in data.items()} + + +def build_runtime_config( + *, + source_config: Path, + run_dir: Path, + config_model_name: str, + provider_model_name: str | None, + api_key: str | None, + base_url: str | None, + headers: dict[str, str], +) -> Path | None: + """Create a temporary config.yaml with an injected model when overrides are provided. + + Returns ``None`` when no override is needed, so callers can use the normal + DeerFlow config resolution. + """ + if not any([provider_model_name, api_key, base_url, headers]): + return None + + try: + import yaml + except ImportError as exc: + raise RuntimeError("PyYAML is required for model override config generation") from exc + + config = yaml.safe_load(source_config.read_text(encoding="utf-8")) or {} + models = list(config.get("models") or []) + if not models: + raise RuntimeError(f"source config has no models: {source_config}") + + base_model = dict(models[0]) + for model in models: + if model.get("name") == config_model_name: + base_model = dict(model) + break + + base_model["name"] = config_model_name + base_model["display_name"] = config_model_name + base_model["use"] = "langchain_openai:ChatOpenAI" + if provider_model_name: + base_model["model"] = provider_model_name + if api_key: + resolved_api_key = resolve_secret(api_key) + if not resolved_api_key: + raise RuntimeError(f"api key is empty after env resolution: {api_key}") + base_model["api_key"] = secret_config_value(api_key) + if base_url: + base_model["base_url"] = base_url + if headers: + # LangChain ChatOpenAI / OpenAI client uses default_headers. + existing = base_model.get("default_headers") or {} + if not isinstance(existing, dict): + existing = {} + base_model["default_headers"] = {**existing, **headers} + base_model["supports_thinking"] = False + base_model["supports_vision"] = bool(base_model.get("supports_vision", False)) + + replaced = False + new_models = [] + for model in models: + if model.get("name") == config_model_name: + new_models.append(base_model) + replaced = True + else: + new_models.append(model) + if not replaced: + new_models.insert(0, base_model) + config["models"] = new_models + + runtime_config = run_dir / "_runtime_config.yaml" + runtime_config.write_text(yaml.safe_dump(config, allow_unicode=True, sort_keys=False), encoding="utf-8") + return runtime_config + + def generate_patches( *, live_date: str, run_dir: Path, model: str, + config_path: Path | None, limit: int, topic_hint: str | None, keep_log: bool, ) -> list[Path]: + from deerflow.client import DeerFlowClient + from deerflow.config.paths import get_paths + from deerflow.runtime.user_context import get_effective_user_id + thread_id = f"knowledge-update-{live_date}-{int(time.time())}" workspace_rel_dir = f"knowledge_update_runs/{run_dir.name}" prompt = build_prompt(live_date, workspace_rel_dir, limit, topic_hint) client = DeerFlowClient( + config_path=str(config_path) if config_path else None, model_name=model, thinking_enabled=False, subagent_enabled=False, @@ -378,7 +509,13 @@ def create_run_dir(live_date: str, run_name: str | None) -> Path: def main() -> int: parser = argparse.ArgumentParser(description="Generate and apply knowledge patches.") parser.add_argument("--live-date", default="2026-05-18", help="live file date, e.g. 2026-05-18") - parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--model", default=DEFAULT_MODEL, help="DeerFlow runtime model config name") + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH, help="source config.yaml used to build runtime config") + parser.add_argument("--api-key", help="override API key; supports direct value, $ENV_NAME, or env:ENV_NAME") + parser.add_argument("--base-url", help="override OpenAI-compatible base URL") + parser.add_argument("--model-name", help="override provider model name, e.g. Pro/zai-org/GLM-5.1") + parser.add_argument("--header", action="append", help="extra HTTP header, repeatable, format Key=Value") + parser.add_argument("--headers-json", help='extra HTTP headers as JSON object, e.g. \'{"X-Org":"abc"}\'') parser.add_argument("--limit", type=int, default=5, help="maximum number of patches to generate; agent may generate fewer, including 0") parser.add_argument("--topic", help="optional topic/target hint for the agent") parser.add_argument("--run-name", help="run directory name under maintenance/runs") @@ -389,6 +526,19 @@ def main() -> int: run_dir = create_run_dir(args.live_date, args.run_name) try: + headers = { + **parse_headers_json(args.headers_json), + **parse_header_values(args.header), + } + runtime_config = build_runtime_config( + source_config=args.config, + run_dir=run_dir, + config_model_name=args.model, + provider_model_name=args.model_name, + api_key=args.api_key, + base_url=args.base_url, + headers=headers, + ) if args.patch: patch_dest = run_dir / args.patch.name shutil.copy2(args.patch, patch_dest) @@ -398,6 +548,7 @@ def main() -> int: live_date=args.live_date, run_dir=run_dir, model=args.model, + config_path=runtime_config, limit=args.limit, topic_hint=args.topic, keep_log=args.keep_log, diff --git a/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh b/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh index ab20aba1..6205cf80 100644 --- a/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh +++ b/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh @@ -4,6 +4,13 @@ set -euo pipefail BACKEND_DIR="/data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend" RUN_DATE="${1:-$(date +%F)}" MAX_PATCHES="${MAX_PATCHES:-5}" +KNOWLEDGE_UPDATE_MODEL="${KNOWLEDGE_UPDATE_MODEL:-GLM-5}" +KNOWLEDGE_UPDATE_API_KEY="${KNOWLEDGE_UPDATE_API_KEY:-}" +KNOWLEDGE_UPDATE_BASE_URL="${KNOWLEDGE_UPDATE_BASE_URL:-}" +KNOWLEDGE_UPDATE_MODEL_NAME="${KNOWLEDGE_UPDATE_MODEL_NAME:-}" +KNOWLEDGE_UPDATE_HEADERS_JSON="${KNOWLEDGE_UPDATE_HEADERS_JSON:-}" +KNOWLEDGE_UPDATE_TOPIC="${KNOWLEDGE_UPDATE_TOPIC:-}" +KNOWLEDGE_UPDATE_KEEP_LOG="${KNOWLEDGE_UPDATE_KEEP_LOG:-1}" export http_proxy="${http_proxy:-http://192.168.134.44:7897}" export https_proxy="${https_proxy:-http://192.168.134.44:7897}" @@ -22,8 +29,36 @@ if [[ ! -s "$LIVE_FILE" ]]; then exit 1 fi -.venv/bin/python knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py \ - --live-date "$RUN_DATE" \ +PIPELINE_ARGS=( + --live-date "$RUN_DATE" --limit "$MAX_PATCHES" + --model "$KNOWLEDGE_UPDATE_MODEL" +) + +if [[ -n "$KNOWLEDGE_UPDATE_API_KEY" ]]; then + PIPELINE_ARGS+=(--api-key "$KNOWLEDGE_UPDATE_API_KEY") +fi + +if [[ -n "$KNOWLEDGE_UPDATE_BASE_URL" ]]; then + PIPELINE_ARGS+=(--base-url "$KNOWLEDGE_UPDATE_BASE_URL") +fi + +if [[ -n "$KNOWLEDGE_UPDATE_MODEL_NAME" ]]; then + PIPELINE_ARGS+=(--model-name "$KNOWLEDGE_UPDATE_MODEL_NAME") +fi + +if [[ -n "$KNOWLEDGE_UPDATE_HEADERS_JSON" ]]; then + PIPELINE_ARGS+=(--headers-json "$KNOWLEDGE_UPDATE_HEADERS_JSON") +fi + +if [[ -n "$KNOWLEDGE_UPDATE_TOPIC" ]]; then + PIPELINE_ARGS+=(--topic "$KNOWLEDGE_UPDATE_TOPIC") +fi + +if [[ "$KNOWLEDGE_UPDATE_KEEP_LOG" == "1" || "$KNOWLEDGE_UPDATE_KEEP_LOG" == "true" ]]; then + PIPELINE_ARGS+=(--keep-log) +fi + +.venv/bin/python knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py "${PIPELINE_ARGS[@]}" echo "[$(date '+%F %T')] daily knowledge update finished"