feat: MySQL 索引长度兼容 + 启动自动补列 + 未发布徽标与查询加载态
后端: - 索引/唯一键改用 mysql_length 前缀索引,适配 MySQL 767 字节限制(error 1071):knowledge_notes、knowledge_entities、knowledge_graph_blacklist、agent_skills、feedback - engine.py 新增 _ensure_orm_columns_sync:启动时对比 ORM 元数据,自动 ALTER TABLE 补齐缺失的可空/带默认值列及其索引 - 新增 scripts/audit_index_key_lengths.py 审计全部索引键长度 - 更新 backend/CLAUDE.md 迁移说明 前端: - 智能体卡片/技能列表:所有者与管理员可见未发布徽标 - 智能体广场:服务端查询进行中时灰显旧结果并悬浮加载提示 - 排行榜入口从 Header 移到欢迎语旁 - threads/hooks.ts:抽取 updateCachedThreadLists 统一处理数组与分页两种缓存形态 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
da677f87d5
commit
57a87d6322
15 files changed
+330
-102
No files matched your search
@@ -3,6 +3,7 @@
|
||||
import {
|
||||
BotIcon,
|
||||
CopyIcon,
|
||||
EyeOffIcon,
|
||||
GlobeIcon,
|
||||
LockIcon,
|
||||
MessageSquareIcon,
|
||||
@@ -529,7 +530,7 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar
|
||||
来自他人
|
||||
</Badge>
|
||||
)}
|
||||
{isOwned && agent.published && !agent.is_private && (
|
||||
{(isOwned || isAdmin) && !isBuiltin && agent.published && !agent.is_private && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="border-sky-500/30 text-sky-700 dark:text-sky-300 text-xs"
|
||||
@@ -539,6 +540,16 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar
|
||||
已发布
|
||||
</Badge>
|
||||
)}
|
||||
{(isOwned || isAdmin) && !isBuiltin && !agent.published && !agent.is_private && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="border-zinc-400/40 text-zinc-500 dark:text-zinc-400 text-xs"
|
||||
title="该智能体尚未发布,仅所有者可见"
|
||||
>
|
||||
<EyeOffIcon className="size-3" />
|
||||
未发布
|
||||
</Badge>
|
||||
)}
|
||||
{agent.is_private && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import { ArrowDownUpIcon, BotIcon, LockIcon, PlusIcon, ShieldIcon, XIcon } from "lucide-react";
|
||||
import { ArrowDownUpIcon, BotIcon, Loader2, LockIcon, PlusIcon, ShieldIcon, XIcon } from "lucide-react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useEffect, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
@@ -190,16 +190,21 @@ export function AgentGallery() {
|
||||
list: Agent[];
|
||||
total: number;
|
||||
isLoading: boolean;
|
||||
isFetching: boolean;
|
||||
tab: string;
|
||||
emptyDescription: string;
|
||||
showNewButton?: boolean;
|
||||
}) => {
|
||||
const { list, total, isLoading, tab, emptyDescription, showNewButton } = opts;
|
||||
const { list, total, isLoading, isFetching, tab, emptyDescription, showNewButton } =
|
||||
opts;
|
||||
// keepPreviousData keeps the old page visible while a new page loads, so
|
||||
// only show the spinner on the very first load (no rows yet).
|
||||
if (isLoading && list.length === 0) {
|
||||
// this full-height spinner only covers the very first load (no rows yet),
|
||||
// or a filter change while the previous result was already empty (where
|
||||
// isLoading stays false but there is nothing to keep on screen).
|
||||
if ((isLoading || isFetching) && list.length === 0) {
|
||||
return (
|
||||
<div className="text-muted-foreground flex h-40 items-center justify-center text-sm">
|
||||
<div className="text-muted-foreground flex h-40 items-center justify-center gap-2 text-sm">
|
||||
<Loader2 className="size-4 animate-spin" />
|
||||
{t.common.loading}
|
||||
</div>
|
||||
);
|
||||
@@ -208,8 +213,29 @@ export function AgentGallery() {
|
||||
return renderEmpty(emptyDescription, showNewButton);
|
||||
}
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
{renderGrid(list)}
|
||||
<div className="relative flex flex-col gap-4">
|
||||
{/* While a server-side search/tag/page query is in flight the stale
|
||||
keepPreviousData rows stay mounted — dim them and float a spinner
|
||||
on top so a slow backend query never looks like the filter was
|
||||
ignored, and block clicks on results that are about to change. */}
|
||||
<div
|
||||
className={
|
||||
isFetching
|
||||
? "pointer-events-none opacity-40 transition-opacity duration-200"
|
||||
: "transition-opacity duration-200"
|
||||
}
|
||||
aria-busy={isFetching}
|
||||
>
|
||||
{renderGrid(list)}
|
||||
</div>
|
||||
{isFetching && (
|
||||
<div className="absolute inset-0 z-10 flex items-start justify-center pt-24">
|
||||
<div className="bg-background/90 text-muted-foreground flex items-center gap-2 rounded-full border px-4 py-2 text-sm shadow-sm">
|
||||
<Loader2 className="size-4 animate-spin" />
|
||||
正在查询…
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<Pagination
|
||||
page={pages[tab] ?? 1}
|
||||
pageSize={PAGE_SIZE}
|
||||
@@ -315,6 +341,7 @@ export function AgentGallery() {
|
||||
list: privateAgents,
|
||||
total: privateTotal,
|
||||
isLoading: isPrivateLoading,
|
||||
isFetching: isPrivateFetching,
|
||||
tab: "private",
|
||||
emptyDescription: "私密广场暂时还没有智能体。",
|
||||
});
|
||||
@@ -444,6 +471,7 @@ export function AgentGallery() {
|
||||
list: mainAgents,
|
||||
total: mainTotal,
|
||||
isLoading: mainLoading,
|
||||
isFetching: mainFetching,
|
||||
tab: sq.id,
|
||||
emptyDescription: `「${sq.name}」广场暂时还没有智能体。`,
|
||||
})}
|
||||
@@ -455,6 +483,7 @@ export function AgentGallery() {
|
||||
list: mainAgents,
|
||||
total: mainTotal,
|
||||
isLoading: mainLoading,
|
||||
isFetching: mainFetching,
|
||||
tab: "mine",
|
||||
emptyDescription: t.agents.emptyDescription,
|
||||
showNewButton: true,
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
ChevronRightIcon,
|
||||
ClockIcon,
|
||||
CopyIcon,
|
||||
EyeOffIcon,
|
||||
FileArchive,
|
||||
GlobeIcon,
|
||||
InfoIcon,
|
||||
@@ -749,7 +750,7 @@ export function SkillRow({
|
||||
{t.settings.skills.badgeFromOthers}
|
||||
</Badge>
|
||||
)}
|
||||
{isOwned && skill.published && (
|
||||
{(isOwned || isAdmin) && !isBuiltin && skill.published && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="border-sky-500/30 text-sky-700 dark:text-sky-300 text-xs"
|
||||
@@ -759,6 +760,16 @@ export function SkillRow({
|
||||
{t.settings.skills.badgePublished}
|
||||
</Badge>
|
||||
)}
|
||||
{(isOwned || isAdmin) && !isBuiltin && !skill.published && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="border-zinc-400/40 text-zinc-500 dark:text-zinc-400 text-xs"
|
||||
title={t.settings.skills.badgeUnpublishedTitle}
|
||||
>
|
||||
<EyeOffIcon className="size-3" />
|
||||
{t.settings.skills.badgeUnpublished}
|
||||
</Badge>
|
||||
)}
|
||||
{isCustom && !isBuiltin && skill.source === "agent" && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import { Trophy } from "lucide-react";
|
||||
import { useSearchParams } from "next/navigation";
|
||||
import { useEffect, useMemo } from "react";
|
||||
|
||||
@@ -7,6 +8,7 @@ import { useI18n } from "@/core/i18n/hooks";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
import { AuroraText } from "../ui/aurora-text";
|
||||
import { Button } from "../ui/button";
|
||||
|
||||
let waved = false;
|
||||
|
||||
@@ -45,6 +47,20 @@ export function Welcome({
|
||||
{isUltra ? "🚀" : "👋"}
|
||||
</div>
|
||||
<AuroraText colors={colors}>{t.welcome.greeting}</AuroraText>
|
||||
{/* 用户排行榜入口:紧挨欢迎语的"CM助手",新标签打开免登录的公开排行榜页。 */}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() =>
|
||||
window.open("#/public/leaderboard", "_blank", "noopener")
|
||||
}
|
||||
title="打开用户排行榜公开页"
|
||||
className="text-muted-foreground hover:text-foreground h-8 shrink-0 gap-1.5 px-2 text-xs font-medium"
|
||||
>
|
||||
<Trophy className="size-4" />
|
||||
<span>排行榜</span>
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -482,6 +482,8 @@ export const enUS: Translations = {
|
||||
badgeFromOthersTitle: "Owned by user {user_id}",
|
||||
badgePublished: "Published",
|
||||
badgePublishedTitle: "Visible to all users",
|
||||
badgeUnpublished: "Unpublished",
|
||||
badgeUnpublishedTitle: "Not published yet — only visible to its owner",
|
||||
badgeStale: "Stale",
|
||||
badgeStaleTitle: "Idle for a long time — flagged by the curator",
|
||||
badgeArchived: "Archived",
|
||||
|
||||
@@ -415,6 +415,8 @@ export interface Translations {
|
||||
badgeFromOthersTitle: string;
|
||||
badgePublished: string;
|
||||
badgePublishedTitle: string;
|
||||
badgeUnpublished: string;
|
||||
badgeUnpublishedTitle: string;
|
||||
badgeStale: string;
|
||||
badgeStaleTitle: string;
|
||||
badgeArchived: string;
|
||||
|
||||
@@ -462,6 +462,8 @@ export const zhCN: Translations = {
|
||||
badgeFromOthersTitle: "来自用户 {user_id}",
|
||||
badgePublished: "已发布",
|
||||
badgePublishedTitle: "该技能已对所有用户可见",
|
||||
badgeUnpublished: "未发布",
|
||||
badgeUnpublishedTitle: "该技能尚未发布,仅所有者可见",
|
||||
badgeStale: "过时",
|
||||
badgeStaleTitle: "长期未使用,已被策展程序标记",
|
||||
badgeArchived: "归档",
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { AIMessage, Message } from "@langchain/langgraph-sdk";
|
||||
import type { ThreadsClient } from "@langchain/langgraph-sdk/client";
|
||||
import { useStream } from "@langchain/langgraph-sdk/react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import type { QueryClient } from "@tanstack/react-query";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import { toast } from "sonner";
|
||||
@@ -298,25 +299,19 @@ export function useThreadStream({
|
||||
);
|
||||
for (const update of updates) {
|
||||
if (update && "title" in update && update.title) {
|
||||
void queryClient.setQueriesData(
|
||||
{
|
||||
queryKey: ["threads", "search"],
|
||||
exact: false,
|
||||
},
|
||||
(oldData: Array<AgentThread> | undefined) => {
|
||||
return oldData?.map((t) => {
|
||||
if (t.thread_id === threadIdRef.current) {
|
||||
return {
|
||||
const nextTitle = update.title;
|
||||
updateCachedThreadLists(queryClient, (threads) =>
|
||||
threads.map((t) =>
|
||||
t.thread_id === threadIdRef.current
|
||||
? {
|
||||
...t,
|
||||
values: {
|
||||
...t.values,
|
||||
title: update.title,
|
||||
title: nextTitle,
|
||||
},
|
||||
};
|
||||
}
|
||||
return t;
|
||||
});
|
||||
},
|
||||
}
|
||||
: t,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -815,6 +810,31 @@ export interface ThreadsPageResult {
|
||||
hasMore: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply ``transform`` to the thread array inside every cached
|
||||
* ``["threads", "search"]`` entry. The sidebar queries cache a plain
|
||||
* ``AgentThread[]`` while the paged chats list (`useThreadsPage`) caches a
|
||||
* ``ThreadsPageResult`` object under the same key prefix, so fuzzy
|
||||
* ``setQueriesData`` updaters must handle both shapes.
|
||||
*/
|
||||
function updateCachedThreadLists(
|
||||
queryClient: QueryClient,
|
||||
transform: (threads: AgentThread[]) => AgentThread[],
|
||||
) {
|
||||
queryClient.setQueriesData<AgentThread[] | ThreadsPageResult | undefined>(
|
||||
{ queryKey: ["threads", "search"], exact: false },
|
||||
(oldData) => {
|
||||
if (Array.isArray(oldData)) {
|
||||
return transform(oldData);
|
||||
}
|
||||
if (oldData && Array.isArray(oldData.threads)) {
|
||||
return { ...oldData, threads: transform(oldData.threads) };
|
||||
}
|
||||
return oldData;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side paginated conversation list for the chats page. Unlike
|
||||
* `useThreads` (which loads a bounded recent window for the sidebar), this
|
||||
@@ -935,17 +955,8 @@ export function useDeleteThread() {
|
||||
}
|
||||
},
|
||||
onSuccess(_, { threadId }) {
|
||||
queryClient.setQueriesData(
|
||||
{
|
||||
queryKey: ["threads", "search"],
|
||||
exact: false,
|
||||
},
|
||||
(oldData: Array<AgentThread> | undefined) => {
|
||||
if (oldData == null) {
|
||||
return oldData;
|
||||
}
|
||||
return oldData.filter((t) => t.thread_id !== threadId);
|
||||
},
|
||||
updateCachedThreadLists(queryClient, (threads) =>
|
||||
threads.filter((t) => t.thread_id !== threadId),
|
||||
);
|
||||
},
|
||||
onSettled() {
|
||||
@@ -1031,27 +1042,18 @@ export function useUpdateThreadMetadata() {
|
||||
}
|
||||
},
|
||||
onSuccess(_, { threadId, metadata }) {
|
||||
queryClient.setQueriesData(
|
||||
{
|
||||
queryKey: ["threads", "search"],
|
||||
exact: false,
|
||||
},
|
||||
(oldData: Array<AgentThread> | undefined) => {
|
||||
if (oldData == null) {
|
||||
return oldData;
|
||||
}
|
||||
return oldData.map((t) =>
|
||||
t.thread_id === threadId
|
||||
? {
|
||||
...t,
|
||||
metadata: {
|
||||
...(t.metadata as Record<string, unknown> | null),
|
||||
...metadata,
|
||||
},
|
||||
}
|
||||
: t,
|
||||
);
|
||||
},
|
||||
updateCachedThreadLists(queryClient, (threads) =>
|
||||
threads.map((t) =>
|
||||
t.thread_id === threadId
|
||||
? {
|
||||
...t,
|
||||
metadata: {
|
||||
...(t.metadata as Record<string, unknown> | null),
|
||||
...metadata,
|
||||
},
|
||||
}
|
||||
: t,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
@@ -1093,25 +1095,18 @@ export function useRenameThread() {
|
||||
});
|
||||
},
|
||||
onSuccess(_, { threadId, title }) {
|
||||
queryClient.setQueriesData(
|
||||
{
|
||||
queryKey: ["threads", "search"],
|
||||
exact: false,
|
||||
},
|
||||
(oldData: Array<AgentThread>) => {
|
||||
return oldData.map((t) => {
|
||||
if (t.thread_id === threadId) {
|
||||
return {
|
||||
updateCachedThreadLists(queryClient, (threads) =>
|
||||
threads.map((t) =>
|
||||
t.thread_id === threadId
|
||||
? {
|
||||
...t,
|
||||
values: {
|
||||
...t.values,
|
||||
title,
|
||||
},
|
||||
};
|
||||
}
|
||||
return t;
|
||||
});
|
||||
},
|
||||
}
|
||||
: t,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Bell, ChevronDown, FileText, Moon, Palette, Trophy, User } from "lucide-react";
|
||||
import { Bell, ChevronDown, FileText, Moon, Palette, User } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
|
||||
import { NotificationDialog } from "@/components/workspace/notification-dialog";
|
||||
@@ -54,26 +54,6 @@ const Header = ({ collapsed }: HeaderProps) => {
|
||||
CM助手
|
||||
</span>
|
||||
)}
|
||||
|
||||
{/* 用户排行榜入口:紧挨 CM助手,点击在新标签打开免登录的公开排行榜页。 */}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() =>
|
||||
window.open("#/public/leaderboard", "_blank", "noopener")
|
||||
}
|
||||
title="打开用户排行榜公开页"
|
||||
className={cn(
|
||||
"h-8 shrink-0 gap-1.5 rounded-sm px-2 text-xs font-medium",
|
||||
isDark
|
||||
? "text-slate-200 hover:bg-slate-800/50 hover:text-white"
|
||||
: "text-white hover:bg-[var(--brand-950)] active:bg-[#8a0618]",
|
||||
)}
|
||||
>
|
||||
<Trophy className="size-4" />
|
||||
{!collapsed && <span>排行榜</span>}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
|
||||
@@ -330,7 +330,9 @@ Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` /
|
||||
|
||||
### Schema Migrations / Manual Column-Add SQL (`scripts/sql/`)
|
||||
|
||||
Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup only `Base.metadata.create_all` runs, which creates **new tables** but never patches **existing** ones (no new columns, no new indexes). So a column added to an ORM model appears automatically on a brand-new deployment, but is **missing on any existing MySQL database** — and code reading/writing that column will error there until the column is added by hand.
|
||||
Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup `Base.metadata.create_all` runs (creates **new tables** only), followed by **`_ensure_orm_columns_sync`** — a best-effort sweep that compares every existing table against the ORM metadata and `ALTER TABLE … ADD COLUMN`s any missing column that is nullable or carries a `server_default` (plus the indexes covering those new columns). It works on all backends (sqlite/mysql/postgres; column DDL compiled through the active dialect), so a simple add-column lands automatically on existing databases at the next restart. NOT NULL columns without a server_default, type changes, widenings, and constraint changes are **not** auto-patched — those still need the manual SQL below. Tests: `tests/test_ensure_orm_columns.py`.
|
||||
|
||||
**MySQL index-key limit (required):** every index/unique key must fit MySQL's legacy 767-byte InnoDB prefix limit assuming utf8mb4 (4 bytes/char) — i.e. ≤191 chars total per key. Columns longer than that must use an explicit prefix index (`Index(..., mysql_length=191)` or `mysql_length={"col": N}` for composites) instead of `index=True`/`UniqueConstraint`, otherwise `create_all` fails with error 1071 on older MySQL servers. `scripts/audit_index_key_lengths.py` audits the whole metadata; run it after adding any indexed string column.
|
||||
|
||||
**Convention (required):** whenever code adds a field to an existing table (a new `mapped_column` on an ORM model), you must, in the same change:
|
||||
1. Keep the matching alembic version file under `persistence/migrations/versions/` (guarded with `_has_table`/`_has_column`), and
|
||||
|
||||
@@ -13,25 +13,35 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
|
||||
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
|
||||
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
|
||||
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
|
||||
# with error 1071 ("Specified key was too long").
|
||||
|
||||
|
||||
class KnowledgeNoteRow(Base):
|
||||
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
|
||||
|
||||
__tablename__ = "knowledge_notes"
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
|
||||
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024))
|
||||
source_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=0.0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
@@ -118,10 +128,15 @@ class KnowledgeEntityRow(Base):
|
||||
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
|
||||
|
||||
__tablename__ = "knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_entities_name", "name", mysql_length=191),
|
||||
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
|
||||
# realistically far shorter than 120 chars, so the prefix is lossless.
|
||||
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), index=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
entity_type: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
@@ -134,9 +149,10 @@ class KnowledgeGraphBlacklistRow(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_graph_blacklist"
|
||||
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
label: Mapped[str] = mapped_column(String(512), unique=True, index=True)
|
||||
label: Mapped[str] = mapped_column(String(512))
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
+6
-2
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -86,7 +86,11 @@ class AgentSkillRow(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "agent_skills"
|
||||
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
|
||||
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
|
||||
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
|
||||
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
|
||||
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
@@ -194,6 +194,79 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
|
||||
``_run_alembic_upgrade`` call) — so on an existing database a new
|
||||
``mapped_column`` on an ORM model would otherwise error at first query
|
||||
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Indexes covering a newly
|
||||
added column are created as well. Works on sqlite/mysql/postgres — the
|
||||
column DDL is compiled through the active dialect, so portable types
|
||||
(``PortableJSON``/``PortableLongText``/``BeijingDateTime``) resolve to the
|
||||
right backend type. Everything is per-column best-effort: a failure is
|
||||
logged and startup continues.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy.schema import CreateColumn
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
try:
|
||||
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
inspector = sa_inspect(sync_conn)
|
||||
existing_tables = set(inspector.get_table_names())
|
||||
preparer = sync_conn.dialect.identifier_preparer
|
||||
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if table.name not in existing_tables:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
added: set[str] = set()
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
table.name,
|
||||
col.name,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
added.add(col.name)
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
if not added:
|
||||
continue
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
@@ -344,6 +417,15 @@ async def init_engine(
|
||||
else:
|
||||
raise
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
+7
-2
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -14,7 +14,12 @@ from deerflow.persistence.types import BeijingDateTime
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
__table_args__ = (UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),)
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Audit ORM index/unique-key byte sizes against MySQL InnoDB prefix limits.
|
||||
|
||||
MySQL with the legacy 767-byte InnoDB index-prefix limit (5.6, or 5.7 with
|
||||
COMPACT row format) rejects any index whose key exceeds 767 bytes — and
|
||||
utf8mb4 costs 4 bytes per char, so an indexed VARCHAR key may total at most
|
||||
191 chars. Modern servers (DYNAMIC row format) allow 3072 bytes = 768 chars.
|
||||
|
||||
Run: PYTHONPATH=. .venv/Scripts/python scripts/audit_index_key_lengths.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
|
||||
import deerflow.persistence.models # noqa: F401 (registers all ORM models)
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
LIMIT_OLD = 767
|
||||
LIMIT_NEW = 3072
|
||||
BYTES_PER_CHAR = 4 # utf8mb4
|
||||
|
||||
|
||||
def _col_key_bytes(col) -> int | None:
|
||||
t = col.type
|
||||
# unwrap TypeDecorator
|
||||
impl = getattr(t, "impl", None)
|
||||
if impl is not None:
|
||||
t = impl
|
||||
if isinstance(t, String):
|
||||
return (t.length or 255) * BYTES_PER_CHAR
|
||||
return 8 # ints/datetimes/floats — small fixed size, close enough
|
||||
|
||||
|
||||
def _key_bytes(col, mysql_length) -> int:
|
||||
full = _col_key_bytes(col) or 0
|
||||
if mysql_length is None:
|
||||
return full
|
||||
if isinstance(mysql_length, dict):
|
||||
prefix = mysql_length.get(col.name)
|
||||
else:
|
||||
prefix = mysql_length
|
||||
if prefix is None:
|
||||
return full
|
||||
return min(full, int(prefix) * BYTES_PER_CHAR)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
bad = 0
|
||||
for table in Base.metadata.sorted_tables:
|
||||
keys: list[tuple[str, str, list, object]] = []
|
||||
for idx in table.indexes:
|
||||
keys.append(("INDEX", idx.name, list(idx.columns), idx.dialect_options["mysql"].get("length")))
|
||||
for cons in table.constraints:
|
||||
if isinstance(cons, UniqueConstraint):
|
||||
keys.append(("UNIQUE", cons.name or "?", list(cons.columns), None))
|
||||
for kind, name, cols, mysql_length in keys:
|
||||
total = sum(_key_bytes(c, mysql_length) for c in cols)
|
||||
if total > LIMIT_OLD:
|
||||
bad += 1
|
||||
tag = "EXCEEDS 3072 (fails on ALL MySQL)" if total > LIMIT_NEW else "exceeds 767 (fails on legacy MySQL)"
|
||||
detail = ", ".join(f"{c.name}({getattr(getattr(c.type, 'impl', c.type), 'length', '?')})" for c in cols)
|
||||
print(f"{table.name}: {kind} {name} = {total} bytes [{detail}] -- {tag}")
|
||||
if not bad:
|
||||
print("OK: every index key fits within 767 bytes (utf8mb4 worst case)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user