修复知识更新bug

offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update.cron.example

@@ -6,5 +6,18 @@
 #
 # Logs:
 #   knowledge/knowledge_generate/maintenance/logs/daily_knowledge_update.log
+#
+# Optional internal model environment variables:
+#   KNOWLEDGE_UPDATE_MODEL=INTERNAL-GLM
+#   KNOWLEDGE_UPDATE_API_KEY=env:INTERNAL_LLM_KEY
+#   KNOWLEDGE_UPDATE_BASE_URL=http://internal-llm.example/v1
+#   KNOWLEDGE_UPDATE_MODEL_NAME=your-internal-model-name
+#   KNOWLEDGE_UPDATE_HEADERS_JSON={"X-Project":"knowledge-update"}
+#
+# If using env:INTERNAL_LLM_KEY, export INTERNAL_LLM_KEY in the cron environment
+# or put it before the command line.
 
 10 12 * * * mkdir -p /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/logs && /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh >> /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/logs/daily_knowledge_update.log 2>&1
+
+# Example with internal model overrides:
+# 10 12 * * * export INTERNAL_LLM_KEY="***"; KNOWLEDGE_UPDATE_MODEL=INTERNAL-GLM KNOWLEDGE_UPDATE_API_KEY=env:INTERNAL_LLM_KEY KNOWLEDGE_UPDATE_BASE_URL=http://internal-llm.example/v1 KNOWLEDGE_UPDATE_MODEL_NAME=your-internal-model-name /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh >> /data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/logs/daily_knowledge_update.log 2>&1

offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import re
 import shutil
 import sys
@@ -21,18 +22,21 @@ from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 
-from deerflow.client import DeerFlowClient
-from deerflow.config.paths import get_paths
-from deerflow.runtime.user_context import get_effective_user_id
+BACKEND_ROOT = Path(__file__).resolve().parents[3]
+KNOWLEDGE_PARENT = Path(__file__).resolve().parents[2]
+HARNESS_ROOT = BACKEND_ROOT / "packages" / "harness"
+for import_root in (BACKEND_ROOT, HARNESS_ROOT):
+    import_root_str = str(import_root)
+    if import_root_str not in sys.path:
+        sys.path.insert(0, import_root_str)
 
-
-BACKEND_ROOT = Path(__file__).resolve().parents[2]
-KNOWLEDGE_ROOT = BACKEND_ROOT / "knowledge_base"
-MAINTENANCE_ROOT = BACKEND_ROOT / "knowledge_generate" / "maintenance"
+KNOWLEDGE_ROOT = KNOWLEDGE_PARENT / "knowledge_base"
+MAINTENANCE_ROOT = KNOWLEDGE_PARENT / "knowledge_generate" / "maintenance"
 RUNS_ROOT = MAINTENANCE_ROOT / "runs"
 
 DEFAULT_MODEL = "GLM-5"
 DEFAULT_SKILLS = {"policy-knowledge", "update-knowledge"}
+DEFAULT_CONFIG_PATH = BACKEND_ROOT / "config.yaml"
 
 
 class PatchError(RuntimeError):
@@ -292,20 +296,147 @@ def event_to_jsonable(event) -> dict:
     return {"type": event.type, "data": dict(event.data or {})}
 
 
+def resolve_secret(value: str | None) -> str | None:
+    """Resolve direct values or env references.
+
+    Supported forms:
+    - ``sk-xxx``: returned as-is
+    - ``$ENV_NAME``: read from environment
+    - ``env:ENV_NAME``: read from environment
+    """
+    if not value:
+        return None
+    value = value.strip()
+    if value.startswith("env:"):
+        return os.getenv(value[4:])
+    if value.startswith("$"):
+        return os.getenv(value[1:])
+    return value
+
+
+def secret_config_value(value: str) -> str:
+    """Return the value to write into temp config without expanding env refs."""
+    value = value.strip()
+    if value.startswith("env:"):
+        return f"${value[4:]}"
+    return value
+
+
+def parse_header_values(values: list[str] | None) -> dict[str, str]:
+    """Parse repeated ``--header Key=Value`` options."""
+    headers: dict[str, str] = {}
+    for raw in values or []:
+        if "=" not in raw:
+            raise ValueError(f"--header must use Key=Value format, got: {raw}")
+        key, value = raw.split("=", 1)
+        key = key.strip()
+        if not key:
+            raise ValueError(f"--header key cannot be empty: {raw}")
+        headers[key] = value.strip()
+    return headers
+
+
+def parse_headers_json(raw: str | None) -> dict[str, str]:
+    if not raw:
+        return {}
+    data = json.loads(raw)
+    if not isinstance(data, dict):
+        raise ValueError("--headers-json must be a JSON object")
+    return {str(k): str(v) for k, v in data.items()}
+
+
+def build_runtime_config(
+    *,
+    source_config: Path,
+    run_dir: Path,
+    config_model_name: str,
+    provider_model_name: str | None,
+    api_key: str | None,
+    base_url: str | None,
+    headers: dict[str, str],
+) -> Path | None:
+    """Create a temporary config.yaml with an injected model when overrides are provided.
+
+    Returns ``None`` when no override is needed, so callers can use the normal
+    DeerFlow config resolution.
+    """
+    if not any([provider_model_name, api_key, base_url, headers]):
+        return None
+
+    try:
+        import yaml
+    except ImportError as exc:
+        raise RuntimeError("PyYAML is required for model override config generation") from exc
+
+    config = yaml.safe_load(source_config.read_text(encoding="utf-8")) or {}
+    models = list(config.get("models") or [])
+    if not models:
+        raise RuntimeError(f"source config has no models: {source_config}")
+
+    base_model = dict(models[0])
+    for model in models:
+        if model.get("name") == config_model_name:
+            base_model = dict(model)
+            break
+
+    base_model["name"] = config_model_name
+    base_model["display_name"] = config_model_name
+    base_model["use"] = "langchain_openai:ChatOpenAI"
+    if provider_model_name:
+        base_model["model"] = provider_model_name
+    if api_key:
+        resolved_api_key = resolve_secret(api_key)
+        if not resolved_api_key:
+            raise RuntimeError(f"api key is empty after env resolution: {api_key}")
+        base_model["api_key"] = secret_config_value(api_key)
+    if base_url:
+        base_model["base_url"] = base_url
+    if headers:
+        # LangChain ChatOpenAI / OpenAI client uses default_headers.
+        existing = base_model.get("default_headers") or {}
+        if not isinstance(existing, dict):
+            existing = {}
+        base_model["default_headers"] = {**existing, **headers}
+    base_model["supports_thinking"] = False
+    base_model["supports_vision"] = bool(base_model.get("supports_vision", False))
+
+    replaced = False
+    new_models = []
+    for model in models:
+        if model.get("name") == config_model_name:
+            new_models.append(base_model)
+            replaced = True
+        else:
+            new_models.append(model)
+    if not replaced:
+        new_models.insert(0, base_model)
+    config["models"] = new_models
+
+    runtime_config = run_dir / "_runtime_config.yaml"
+    runtime_config.write_text(yaml.safe_dump(config, allow_unicode=True, sort_keys=False), encoding="utf-8")
+    return runtime_config
+
+
 def generate_patches(
     *,
     live_date: str,
     run_dir: Path,
     model: str,
+    config_path: Path | None,
     limit: int,
     topic_hint: str | None,
     keep_log: bool,
 ) -> list[Path]:
+    from deerflow.client import DeerFlowClient
+    from deerflow.config.paths import get_paths
+    from deerflow.runtime.user_context import get_effective_user_id
+
     thread_id = f"knowledge-update-{live_date}-{int(time.time())}"
     workspace_rel_dir = f"knowledge_update_runs/{run_dir.name}"
     prompt = build_prompt(live_date, workspace_rel_dir, limit, topic_hint)
 
     client = DeerFlowClient(
+        config_path=str(config_path) if config_path else None,
         model_name=model,
         thinking_enabled=False,
         subagent_enabled=False,
@@ -378,7 +509,13 @@ def create_run_dir(live_date: str, run_name: str | None) -> Path:
 def main() -> int:
     parser = argparse.ArgumentParser(description="Generate and apply knowledge patches.")
     parser.add_argument("--live-date", default="2026-05-18", help="live file date, e.g. 2026-05-18")
-    parser.add_argument("--model", default=DEFAULT_MODEL)
+    parser.add_argument("--model", default=DEFAULT_MODEL, help="DeerFlow runtime model config name")
+    parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_PATH, help="source config.yaml used to build runtime config")
+    parser.add_argument("--api-key", help="override API key; supports direct value, $ENV_NAME, or env:ENV_NAME")
+    parser.add_argument("--base-url", help="override OpenAI-compatible base URL")
+    parser.add_argument("--model-name", help="override provider model name, e.g. Pro/zai-org/GLM-5.1")
+    parser.add_argument("--header", action="append", help="extra HTTP header, repeatable, format Key=Value")
+    parser.add_argument("--headers-json", help='extra HTTP headers as JSON object, e.g. \'{"X-Org":"abc"}\'')
     parser.add_argument("--limit", type=int, default=5, help="maximum number of patches to generate; agent may generate fewer, including 0")
     parser.add_argument("--topic", help="optional topic/target hint for the agent")
     parser.add_argument("--run-name", help="run directory name under maintenance/runs")
@@ -389,6 +526,19 @@ def main() -> int:
 
     run_dir = create_run_dir(args.live_date, args.run_name)
     try:
+        headers = {
+            **parse_headers_json(args.headers_json),
+            **parse_header_values(args.header),
+        }
+        runtime_config = build_runtime_config(
+            source_config=args.config,
+            run_dir=run_dir,
+            config_model_name=args.model,
+            provider_model_name=args.model_name,
+            api_key=args.api_key,
+            base_url=args.base_url,
+            headers=headers,
+        )
         if args.patch:
             patch_dest = run_dir / args.patch.name
             shutil.copy2(args.patch, patch_dest)
@@ -398,6 +548,7 @@ def main() -> int:
                 live_date=args.live_date,
                 run_dir=run_dir,
                 model=args.model,
+                config_path=runtime_config,
                 limit=args.limit,
                 topic_hint=args.topic,
                 keep_log=args.keep_log,

offline-backend-20260512/backend/knowledge/knowledge_generate/maintenance/run_daily_knowledge_update.sh

@@ -4,6 +4,13 @@ set -euo pipefail
 BACKEND_DIR="/data01/likun/deerflow-cmzs/deerflow-server/offline-backend-20260512/backend"
 RUN_DATE="${1:-$(date +%F)}"
 MAX_PATCHES="${MAX_PATCHES:-5}"
+KNOWLEDGE_UPDATE_MODEL="${KNOWLEDGE_UPDATE_MODEL:-GLM-5}"
+KNOWLEDGE_UPDATE_API_KEY="${KNOWLEDGE_UPDATE_API_KEY:-}"
+KNOWLEDGE_UPDATE_BASE_URL="${KNOWLEDGE_UPDATE_BASE_URL:-}"
+KNOWLEDGE_UPDATE_MODEL_NAME="${KNOWLEDGE_UPDATE_MODEL_NAME:-}"
+KNOWLEDGE_UPDATE_HEADERS_JSON="${KNOWLEDGE_UPDATE_HEADERS_JSON:-}"
+KNOWLEDGE_UPDATE_TOPIC="${KNOWLEDGE_UPDATE_TOPIC:-}"
+KNOWLEDGE_UPDATE_KEEP_LOG="${KNOWLEDGE_UPDATE_KEEP_LOG:-1}"
 
 export http_proxy="${http_proxy:-http://192.168.134.44:7897}"
 export https_proxy="${https_proxy:-http://192.168.134.44:7897}"
@@ -22,8 +29,36 @@ if [[ ! -s "$LIVE_FILE" ]]; then
   exit 1
 fi
 
-.venv/bin/python knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py \
-  --live-date "$RUN_DATE" \
+PIPELINE_ARGS=(
+  --live-date "$RUN_DATE"
   --limit "$MAX_PATCHES"
+  --model "$KNOWLEDGE_UPDATE_MODEL"
+)
+
+if [[ -n "$KNOWLEDGE_UPDATE_API_KEY" ]]; then
+  PIPELINE_ARGS+=(--api-key "$KNOWLEDGE_UPDATE_API_KEY")
+fi
+
+if [[ -n "$KNOWLEDGE_UPDATE_BASE_URL" ]]; then
+  PIPELINE_ARGS+=(--base-url "$KNOWLEDGE_UPDATE_BASE_URL")
+fi
+
+if [[ -n "$KNOWLEDGE_UPDATE_MODEL_NAME" ]]; then
+  PIPELINE_ARGS+=(--model-name "$KNOWLEDGE_UPDATE_MODEL_NAME")
+fi
+
+if [[ -n "$KNOWLEDGE_UPDATE_HEADERS_JSON" ]]; then
+  PIPELINE_ARGS+=(--headers-json "$KNOWLEDGE_UPDATE_HEADERS_JSON")
+fi
+
+if [[ -n "$KNOWLEDGE_UPDATE_TOPIC" ]]; then
+  PIPELINE_ARGS+=(--topic "$KNOWLEDGE_UPDATE_TOPIC")
+fi
+
+if [[ "$KNOWLEDGE_UPDATE_KEEP_LOG" == "1" || "$KNOWLEDGE_UPDATE_KEEP_LOG" == "true" ]]; then
+  PIPELINE_ARGS+=(--keep-log)
+fi
+
+.venv/bin/python knowledge/knowledge_generate/maintenance/knowledge_update_pipeline.py "${PIPELINE_ARGS[@]}"
 
 echo "[$(date '+%F %T')] daily knowledge update finished"