sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -0,0 +1,11 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "_shared"))
import render_xinhua_page as renderer
from fast_patch_page import run
if __name__ == "__main__":
sys.exit(run(renderer, output_stem="xinhua-page"))
@@ -0,0 +1,631 @@
#!/usr/bin/env python3
"""Validate Xinhua page data and render the bundled self-contained template."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
SKILL_DIR = Path(__file__).resolve().parents[1]
ASSETS_DIR = SKILL_DIR / "assets"
DEFAULT_DATA_PATH = ASSETS_DIR / "default-data.json"
TEMPLATE_PATH = ASSETS_DIR / "xinhua-template.html"
RUNTIME_PATH = ASSETS_DIR / "xinhua-runtime.js"
MODULES = (
"site",
"navigation",
"specials",
"headline",
"focus",
"features",
"channels",
"local",
"services",
"footer",
)
SPOT_VISUALS = {"harvest", "railway", "satellite", "culture", "wetland"}
FOCUS_VISUALS = {"city", "economy", "world", "green", "technology"}
CHANNEL_VISUALS = {"commentary", "interview", "video"}
LOCAL_VISUALS = {"capital", "coast", "mountain", "river", "bay"}
SERVICE_ICONS = {"app", "video", "platform", "radio"}
FORBIDDEN_TEXT = ("<script", "<iframe", "javascript:", "http://", "https://")
REMOTE_MEDIA_TYPES = {"image", "video", "webpage"}
MEDIA_URL_KEYS = {"mediaUrl", "posterUrl", "pageUrl"}
class DataValidationError(ValueError):
"""Raised when page JSON does not satisfy the fixed template contract."""
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise DataValidationError(f"input file does not exist: {path}") from exc
except json.JSONDecodeError as exc:
raise DataValidationError(
f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}"
) from exc
if not isinstance(value, dict):
raise DataValidationError("top-level JSON value must be an object")
return value
def _mapping(value: Any, path: str, errors: list[str]) -> dict[str, Any]:
if not isinstance(value, dict):
errors.append(f"{path} must be an object")
return {}
return value
def _array(
value: Any,
path: str,
errors: list[str],
*,
minimum: int = 0,
maximum: int | None = None,
) -> list[Any]:
if not isinstance(value, list):
errors.append(f"{path} must be an array")
return []
if len(value) < minimum:
errors.append(f"{path} must contain at least {minimum} item(s)")
if maximum is not None and len(value) > maximum:
errors.append(f"{path} must contain at most {maximum} item(s)")
return value
def _required_string(value: Any, path: str, errors: list[str]) -> None:
if not isinstance(value, str) or not value.strip():
errors.append(f"{path} must be a non-empty string")
def _string(value: Any, path: str, errors: list[str]) -> None:
if not isinstance(value, str):
errors.append(f"{path} must be a string")
def _href(value: Any, path: str, errors: list[str]) -> None:
_required_string(value, path, errors)
if isinstance(value, str) and not value.startswith("#"):
errors.append(f"{path} must be an in-page anchor beginning with #")
def _unique_id(item: dict[str, Any], path: str, seen: set[str], errors: list[str]) -> None:
item_id = item.get("id")
_required_string(item_id, f"{path}.id", errors)
if isinstance(item_id, str):
if item_id in seen:
errors.append(f"{path}.id duplicates {item_id!r}")
seen.add(item_id)
def _strings(
value: Any,
path: str,
errors: list[str],
*,
minimum: int = 0,
maximum: int | None = None,
) -> list[Any]:
values = _array(value, path, errors, minimum=minimum, maximum=maximum)
for index, item in enumerate(values):
_required_string(item, f"{path}[{index}]", errors)
return values
def _links(
value: Any,
path: str,
errors: list[str],
*,
minimum: int,
maximum: int,
) -> list[Any]:
items = _array(value, path, errors, minimum=minimum, maximum=maximum)
for index, raw in enumerate(items):
item = _mapping(raw, f"{path}[{index}]", errors)
_required_string(item.get("label"), f"{path}[{index}].label", errors)
_href(item.get("href"), f"{path}[{index}].href", errors)
return items
def _id_title_items(
value: Any,
path: str,
errors: list[str],
*,
minimum: int,
maximum: int,
) -> list[Any]:
items = _array(value, path, errors, minimum=minimum, maximum=maximum)
seen: set[str] = set()
for index, raw in enumerate(items):
item_path = f"{path}[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen, errors)
_required_string(item.get("title"), f"{item_path}.title", errors)
return items
def _validate_no_remote_or_markup(value: Any, path: str, errors: list[str]) -> None:
if isinstance(value, dict):
for key, item in value.items():
if key in MEDIA_URL_KEYS:
continue
_validate_no_remote_or_markup(item, f"{path}.{key}", errors)
elif isinstance(value, list):
for index, item in enumerate(value):
_validate_no_remote_or_markup(item, f"{path}[{index}]", errors)
elif isinstance(value, str):
lowered = value.lower()
if any(token in lowered for token in FORBIDDEN_TEXT):
errors.append(f"{path} must not contain HTML, script, iframe, or remote URLs")
def _remote_url(value: Any, path: str, errors: list[str], *, required: bool = False, data_prefix: str = "") -> None:
if value in (None, "") and not required:
return
if not isinstance(value, str) or not value.strip():
errors.append(f"{path} must be a non-empty URL")
return
candidate = value.strip()
if data_prefix and candidate.lower().startswith(data_prefix):
return
parsed = urlparse(candidate)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
errors.append(f"{path} must use an http(s) URL")
def _validate_remote_media_tree(value: Any, path: str, errors: list[str]) -> None:
if isinstance(value, list):
for index, item in enumerate(value):
_validate_remote_media_tree(item, f"{path}[{index}]", errors)
return
if not isinstance(value, dict):
return
media = value.get("media")
if isinstance(media, dict) and "type" in media:
media_path = f"{path}.media"
media_data = _mapping(media, media_path, errors)
media_type = media_data.get("type")
if media_type not in REMOTE_MEDIA_TYPES:
errors.append(f"{media_path}.type must be one of: {', '.join(sorted(REMOTE_MEDIA_TYPES))}")
elif media_type in {"image", "video"}:
_remote_url(media_data.get("mediaUrl"), f"{media_path}.mediaUrl", errors, required=True, data_prefix=f"data:{media_type}/")
if media_type == "video":
_remote_url(media_data.get("posterUrl"), f"{media_path}.posterUrl", errors, data_prefix="data:image/")
if media_type == "webpage":
_remote_url(media_data.get("pageUrl"), f"{media_path}.pageUrl", errors, required=True)
_remote_url(media_data.get("mediaUrl"), f"{media_path}.mediaUrl", errors, data_prefix="data:image/")
if "embed" in media_data and not isinstance(media_data.get("embed"), bool):
errors.append(f"{media_path}.embed must be a boolean")
elif media_type in {"image", "video"}:
_remote_url(media_data.get("pageUrl"), f"{media_path}.pageUrl", errors)
for key, item in value.items():
if key != "media":
_validate_remote_media_tree(item, f"{path}.{key}", errors)
def validate_document(data: dict[str, Any]) -> dict[str, int]:
errors: list[str] = []
missing = [name for name in MODULES if name not in data]
extra = [name for name in data if name not in MODULES]
if missing:
errors.append("missing top-level module(s): " + ", ".join(missing))
if extra:
errors.append("unknown top-level module(s): " + ", ".join(extra))
site = _mapping(data.get("site"), "site", errors)
for key in (
"brandName",
"brandEnglish",
"pageTitle",
"description",
"motto",
"clientLabel",
):
_required_string(site.get(key), f"site.{key}", errors)
if site.get("theme") not in {"light", "dark", "auto"}:
errors.append("site.theme must be light, dark, or auto")
navigation = _mapping(data.get("navigation"), "navigation", errors)
network_left = _links(
navigation.get("networkLeft"),
"navigation.networkLeft",
errors,
minimum=1,
maximum=8,
)
network_right = _links(
navigation.get("networkRight"),
"navigation.networkRight",
errors,
minimum=1,
maximum=6,
)
primary = _links(
navigation.get("primary"),
"navigation.primary",
errors,
minimum=1,
maximum=16,
)
drawer = _links(
navigation.get("drawer"),
"navigation.drawer",
errors,
minimum=1,
maximum=16,
)
for key in ("searchLabel", "searchPlaceholder", "searchButton"):
_required_string(navigation.get(key), f"navigation.{key}", errors)
specials = _mapping(data.get("specials"), "specials", errors)
_required_string(specials.get("seriesTitle"), "specials.seriesTitle", errors)
_required_string(specials.get("seriesEnglish"), "specials.seriesEnglish", errors)
series_items = _array(
specials.get("seriesItems"),
"specials.seriesItems",
errors,
minimum=1,
maximum=4,
)
seen_series: set[str] = set()
for index, raw in enumerate(series_items):
item_path = f"specials.seriesItems[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_series, errors)
_required_string(item.get("title"), f"{item_path}.title", errors)
_href(item.get("href"), f"{item_path}.href", errors)
_required_string(specials.get("spotlightLabel"), "specials.spotlightLabel", errors)
spotlight_items = _array(
specials.get("spotlightItems"),
"specials.spotlightItems",
errors,
minimum=1,
maximum=5,
)
seen_spotlight: set[str] = set()
for index, raw in enumerate(spotlight_items):
item_path = f"specials.spotlightItems[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_spotlight, errors)
_required_string(item.get("title"), f"{item_path}.title", errors)
_string(item.get("tag"), f"{item_path}.tag", errors)
if item.get("visual") not in SPOT_VISUALS:
errors.append(f"{item_path}.visual must be one of {sorted(SPOT_VISUALS)}")
latest = _mapping(specials.get("latest"), "specials.latest", errors)
for key in ("label", "title", "time"):
_required_string(latest.get(key), f"specials.latest.{key}", errors)
_href(latest.get("href"), "specials.latest.href", errors)
headline = _mapping(data.get("headline"), "headline", errors)
for key in ("kicker", "title", "summary"):
_required_string(headline.get(key), f"headline.{key}", errors)
headline_links = _links(
headline.get("links"),
"headline.links",
errors,
minimum=1,
maximum=6,
)
focus = _mapping(data.get("focus"), "focus", errors)
_required_string(focus.get("title"), "focus.title", errors)
_required_string(focus.get("moreLabel"), "focus.moreLabel", errors)
tabs = _array(focus.get("tabs"), "focus.tabs", errors, minimum=1, maximum=5)
seen_tabs: set[str] = set()
focus_columns = 0
for tab_index, raw_tab in enumerate(tabs):
tab_path = f"focus.tabs[{tab_index}]"
tab = _mapping(raw_tab, tab_path, errors)
_unique_id(tab, tab_path, seen_tabs, errors)
_required_string(tab.get("label"), f"{tab_path}.label", errors)
columns = _array(
tab.get("columns"),
f"{tab_path}.columns",
errors,
minimum=1,
maximum=4,
)
focus_columns += len(columns)
seen_columns: set[str] = set()
for column_index, raw_column in enumerate(columns):
column_path = f"{tab_path}.columns[{column_index}]"
column = _mapping(raw_column, column_path, errors)
_unique_id(column, column_path, seen_columns, errors)
_required_string(column.get("leadTitle"), f"{column_path}.leadTitle", errors)
_required_string(column.get("summary"), f"{column_path}.summary", errors)
if column.get("visual") not in FOCUS_VISUALS:
errors.append(
f"{column_path}.visual must be one of {sorted(FOCUS_VISUALS)}"
)
_strings(
column.get("links"),
f"{column_path}.links",
errors,
minimum=1,
maximum=5,
)
features = _mapping(data.get("features"), "features", errors)
for key in (
"columnTitle",
"columnSubtitle",
"topicsTitle",
"topicsMoreLabel",
"studiosTitle",
"studiosMoreLabel",
):
_required_string(features.get(key), f"features.{key}", errors)
tags = _strings(features.get("tags"), "features.tags", errors, minimum=1, maximum=10)
articles = _id_title_items(
features.get("articles"),
"features.articles",
errors,
minimum=1,
maximum=12,
)
for index, raw in enumerate(articles):
item = _mapping(raw, f"features.articles[{index}]", errors)
_required_string(item.get("meta"), f"features.articles[{index}].meta", errors)
topics = _id_title_items(
features.get("topics"),
"features.topics",
errors,
minimum=1,
maximum=12,
)
for index, raw in enumerate(topics):
item = _mapping(raw, f"features.topics[{index}]", errors)
_required_string(item.get("summary"), f"features.topics[{index}].summary", errors)
studios = _array(
features.get("studios"),
"features.studios",
errors,
minimum=1,
maximum=8,
)
seen_studios: set[str] = set()
for index, raw in enumerate(studios):
item_path = f"features.studios[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_studios, errors)
for key in ("initial", "name", "bio"):
_required_string(item.get(key), f"{item_path}.{key}", errors)
if isinstance(item.get("initial"), str) and len(item["initial"]) > 2:
errors.append(f"{item_path}.initial must contain at most 2 characters")
channels = _mapping(data.get("channels"), "channels", errors)
editorial = _array(
channels.get("editorialColumns"),
"channels.editorialColumns",
errors,
minimum=1,
maximum=5,
)
seen_editorial: set[str] = set()
live_count = 0
for index, raw in enumerate(editorial):
item_path = f"channels.editorialColumns[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_editorial, errors)
for key in ("accent", "title", "feature"):
_required_string(item.get(key), f"{item_path}.{key}", errors)
if item.get("visual") not in CHANNEL_VISUALS:
errors.append(f"{item_path}.visual must be one of {sorted(CHANNEL_VISUALS)}")
if not isinstance(item.get("live"), bool):
errors.append(f"{item_path}.live must be a boolean")
elif item["live"]:
live_count += 1
_strings(item.get("links"), f"{item_path}.links", errors, minimum=1, maximum=6)
_required_string(channels.get("industryTitle"), "channels.industryTitle", errors)
_required_string(channels.get("industryMoreLabel"), "channels.industryMoreLabel", errors)
industries = _array(
channels.get("industries"),
"channels.industries",
errors,
minimum=1,
maximum=12,
)
seen_industries: set[str] = set()
for index, raw in enumerate(industries):
item_path = f"channels.industries[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_industries, errors)
_required_string(item.get("title"), f"{item_path}.title", errors)
_strings(item.get("items"), f"{item_path}.items", errors, minimum=1, maximum=6)
local = _mapping(data.get("local"), "local", errors)
for key in ("title", "moreLabel", "selectorLabel"):
_required_string(local.get(key), f"local.{key}", errors)
provinces = _array(
local.get("provinces"),
"local.provinces",
errors,
minimum=1,
maximum=16,
)
seen_provinces: set[str] = set()
for index, raw in enumerate(provinces):
item_path = f"local.provinces[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_provinces, errors)
for key in ("name", "leadTitle", "summary"):
_required_string(item.get(key), f"{item_path}.{key}", errors)
if item.get("visual") not in LOCAL_VISUALS:
errors.append(f"{item_path}.visual must be one of {sorted(LOCAL_VISUALS)}")
_strings(item.get("links"), f"{item_path}.links", errors, minimum=1, maximum=8)
services = _mapping(data.get("services"), "services", errors)
_required_string(services.get("matrixTitle"), "services.matrixTitle", errors)
_required_string(services.get("matrixSubtitle"), "services.matrixSubtitle", errors)
matrix_items = _array(
services.get("matrixItems"),
"services.matrixItems",
errors,
minimum=1,
maximum=4,
)
seen_services: set[str] = set()
for index, raw in enumerate(matrix_items):
item_path = f"services.matrixItems[{index}]"
item = _mapping(raw, item_path, errors)
_unique_id(item, item_path, seen_services, errors)
for key in ("title", "caption"):
_required_string(item.get(key), f"{item_path}.{key}", errors)
if item.get("icon") not in SERVICE_ICONS:
errors.append(f"{item_path}.icon must be one of {sorted(SERVICE_ICONS)}")
live_modal = _mapping(services.get("liveModal"), "services.liveModal", errors)
for key in ("title", "streamTitle", "streamSubtitle", "currentProgram"):
_required_string(live_modal.get(key), f"services.liveModal.{key}", errors)
newsletter = _mapping(services.get("newsletter"), "services.newsletter", errors)
for key in ("title", "text", "placeholder", "button", "successPrefix"):
_required_string(newsletter.get(key), f"services.newsletter.{key}", errors)
footer = _mapping(data.get("footer"), "footer", errors)
footer_links = _links(
footer.get("links"), "footer.links", errors, minimum=1, maximum=10
)
_required_string(footer.get("copyright"), "footer.copyright", errors)
_required_string(footer.get("legal"), "footer.legal", errors)
_validate_remote_media_tree(data, "$", errors)
_validate_no_remote_or_markup(data, "$", errors)
if errors:
raise DataValidationError("\n".join(f"- {error}" for error in errors))
return {
"network_links": len(network_left) + len(network_right),
"primary_navigation": len(primary),
"drawer_navigation": len(drawer),
"series": len(series_items),
"spotlight": len(spotlight_items),
"headline_links": len(headline_links),
"focus_tabs": len(tabs),
"focus_columns": focus_columns,
"feature_tags": len(tags),
"feature_articles": len(articles),
"topics": len(topics),
"studios": len(studios),
"editorial_columns": len(editorial),
"live_columns": live_count,
"industries": len(industries),
"provinces": len(provinces),
"matrix_items": len(matrix_items),
"footer_links": len(footer_links),
}
def render_document(data: dict[str, Any], template: str, runtime: str) -> str:
validate_document(data)
data_text = json.dumps(data, ensure_ascii=False, separators=(",", ":"))
data_text = data_text.replace("</script", "<\\/script")
if template.count("__XINHUA_PAGE_DATA__") != 1:
raise DataValidationError("template must contain one __XINHUA_PAGE_DATA__ marker")
if template.count("__XINHUA_PAGE_RUNTIME__") != 1:
raise DataValidationError(
"template must contain one __XINHUA_PAGE_RUNTIME__ marker"
)
rendered = template.replace("__XINHUA_PAGE_DATA__", data_text).replace(
"__XINHUA_PAGE_RUNTIME__", runtime
)
if "__XINHUA_PAGE_" in rendered:
raise DataValidationError("unresolved template marker remains in rendered HTML")
return rendered
def _write_new(path: Path, content: str) -> None:
if path.exists():
raise DataValidationError(
f"refusing to overwrite existing file: {path}; choose a new path"
)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8", newline="\n")
def _emit(payload: dict[str, Any]) -> None:
print(json.dumps(payload, ensure_ascii=False))
def _command_init(output: Path) -> dict[str, Any]:
data = _read_json(DEFAULT_DATA_PATH)
counts = validate_document(data)
_write_new(output, json.dumps(data, ensure_ascii=False, indent=2) + "\n")
return {
"status": "ok",
"action": "init",
"output": str(output),
"modules": list(MODULES),
"counts": counts,
}
def _command_validate(input_path: Path) -> dict[str, Any]:
counts = validate_document(_read_json(input_path))
return {
"status": "ok",
"action": "validate",
"input": str(input_path),
"modules": list(MODULES),
"counts": counts,
}
def _command_render(input_path: Path, output: Path) -> dict[str, Any]:
data = _read_json(input_path)
counts = validate_document(data)
template = TEMPLATE_PATH.read_text(encoding="utf-8")
runtime = RUNTIME_PATH.read_text(encoding="utf-8")
_write_new(output, render_document(data, template, runtime))
return {
"status": "ok",
"action": "render",
"input": str(input_path),
"output": str(output),
"modules": list(MODULES),
"counts": counts,
}
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Initialize, validate, and render Xinhua-style news portal JSON."
)
subparsers = parser.add_subparsers(dest="command", required=True)
init_parser = subparsers.add_parser("init", help="write a fresh default JSON file")
init_parser.add_argument("--output", required=True, type=Path)
validate_parser = subparsers.add_parser("validate", help="validate a JSON file")
validate_parser.add_argument("--input", required=True, type=Path)
render_parser = subparsers.add_parser(
"render", help="render a validated JSON file into the fixed HTML template"
)
render_parser.add_argument("--input", required=True, type=Path)
render_parser.add_argument("--output", required=True, type=Path)
return parser
def main(argv: list[str] | None = None) -> int:
parser = _build_parser()
args = parser.parse_args(argv)
try:
if args.command == "init":
payload = _command_init(args.output)
elif args.command == "validate":
payload = _command_validate(args.input)
else:
payload = _command_render(args.input, args.output)
_emit(payload)
return 0
except (DataValidationError, OSError) as exc:
_emit({"status": "error", "action": args.command, "message": str(exc)})
return 1
if __name__ == "__main__":
sys.exit(main())