sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "_shared"))
|
||||
|
||||
import render_yomiuri_news_page as renderer
|
||||
from fast_patch_page import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run(renderer, output_stem="yomiuri-news-page"))
|
||||
+444
@@ -0,0 +1,444 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_DATA = SKILL_ROOT / "assets" / "default-data.json"
|
||||
TEMPLATE_PATH = SKILL_ROOT / "assets" / "yomiuri-news-template.html"
|
||||
RUNTIME_PATH = SKILL_ROOT / "assets" / "yomiuri-news-runtime.js"
|
||||
MODULES = (
|
||||
"site",
|
||||
"navigation",
|
||||
"breaking",
|
||||
"lead",
|
||||
"edition",
|
||||
"features",
|
||||
"news",
|
||||
"opinion",
|
||||
"media",
|
||||
"footer",
|
||||
)
|
||||
LEAD_VISUALS = {"transit-hub", "forest", "learning"}
|
||||
FEATURE_VISUALS = {"station"}
|
||||
NEWS_VISUALS = {"society", "politics", "economy", "world", "sports"}
|
||||
COLUMN_VISUALS = {"portrait-one", "portrait-two", "portrait-three"}
|
||||
LIFE_VISUALS = {"education", "health", "food"}
|
||||
VIDEO_VISUALS = {"transit-video", "forest-video", "sports-video"}
|
||||
TECHNICAL_KEYS = {
|
||||
"id",
|
||||
"href",
|
||||
"visual",
|
||||
"action",
|
||||
"language",
|
||||
"brandRoman",
|
||||
"type",
|
||||
"mediaUrl",
|
||||
"posterUrl",
|
||||
"pageUrl",
|
||||
}
|
||||
REMOTE_MEDIA_TYPES = {"image", "video", "webpage"}
|
||||
MEDIA_URL_KEYS = {"mediaUrl", "posterUrl", "pageUrl"}
|
||||
LATIN_RE = re.compile(r"[A-Za-z]")
|
||||
KANA_RE = re.compile(r"[\u3040-\u30ff]")
|
||||
CJK_RE = re.compile(r"[\u3400-\u9fff]")
|
||||
REMOTE_RE = re.compile(r"(?:https?://|javascript:|<\s*(?:script|iframe)\b)", re.I)
|
||||
|
||||
|
||||
class DataValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise DataValidationError(f"unable to read valid JSON from {path}: {exc}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise DataValidationError("top-level JSON must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _mapping(value: Any, path: str, errors: list[str]) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
errors.append(f"{path} must be an object")
|
||||
return {}
|
||||
return value
|
||||
|
||||
|
||||
def _array(value: Any, path: str, errors: list[str], minimum: int, maximum: int) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
errors.append(f"{path} must be an array")
|
||||
return []
|
||||
if not minimum <= len(value) <= maximum:
|
||||
errors.append(f"{path} must contain {minimum}-{maximum} items")
|
||||
return value
|
||||
|
||||
|
||||
def _strings(obj: dict[str, Any], path: str, keys: tuple[str, ...], errors: list[str]) -> None:
|
||||
for key in keys:
|
||||
if not isinstance(obj.get(key), str) or not obj[key].strip():
|
||||
errors.append(f"{path}.{key} must be a non-empty string")
|
||||
|
||||
|
||||
def _href(value: Any, path: str, errors: list[str]) -> None:
|
||||
if not isinstance(value, str) or not value.startswith("#"):
|
||||
errors.append(f"{path} must start with #")
|
||||
|
||||
|
||||
def _items(
|
||||
value: Any,
|
||||
path: str,
|
||||
errors: list[str],
|
||||
*,
|
||||
minimum: int,
|
||||
maximum: int,
|
||||
string_keys: tuple[str, ...],
|
||||
href: bool = False,
|
||||
visual_key: str | None = None,
|
||||
visuals: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
result: list[dict[str, Any]] = []
|
||||
ids: list[str] = []
|
||||
for index, item in enumerate(_array(value, path, errors, minimum, maximum)):
|
||||
current = _mapping(item, f"{path}[{index}]", errors)
|
||||
_strings(current, f"{path}[{index}]", ("id",) + string_keys, errors)
|
||||
if isinstance(current.get("id"), str):
|
||||
ids.append(current["id"])
|
||||
if href:
|
||||
_href(current.get("href"), f"{path}[{index}].href", errors)
|
||||
if visual_key and current.get(visual_key) not in (visuals or set()):
|
||||
errors.append(f"{path}[{index}].{visual_key} must be one of {sorted(visuals or set())}")
|
||||
result.append(current)
|
||||
duplicates = sorted({item_id for item_id in ids if ids.count(item_id) > 1})
|
||||
if duplicates:
|
||||
errors.append(f"{path} id duplicates: {', '.join(duplicates)}")
|
||||
return result
|
||||
|
||||
|
||||
def _links_or_actions(value: Any, path: str, errors: list[str], minimum: int, maximum: int) -> list[dict[str, Any]]:
|
||||
items = _items(value, path, errors, minimum=minimum, maximum=maximum, string_keys=("label",))
|
||||
for index, item in enumerate(items):
|
||||
has_href = isinstance(item.get("href"), str)
|
||||
has_action = isinstance(item.get("action"), str) and item.get("action") in {"paper", "trial", "login", "search"}
|
||||
if has_href:
|
||||
_href(item.get("href"), f"{path}[{index}].href", errors)
|
||||
if has_href == has_action:
|
||||
errors.append(f"{path}[{index}] must define exactly one valid href or action")
|
||||
return items
|
||||
|
||||
|
||||
def _walk_no_remote(value: Any, path: str, errors: list[str]) -> None:
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if key in MEDIA_URL_KEYS:
|
||||
continue
|
||||
_walk_no_remote(item, f"{path}.{key}", errors)
|
||||
elif isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
_walk_no_remote(item, f"{path}[{index}]", errors)
|
||||
elif isinstance(value, str) and REMOTE_RE.search(value):
|
||||
errors.append(f"{path} must not contain remote URLs, scripts, or iframes")
|
||||
|
||||
|
||||
def _remote_url(
|
||||
value: Any,
|
||||
path: str,
|
||||
errors: list[str],
|
||||
*,
|
||||
required: bool = False,
|
||||
data_prefix: str = "",
|
||||
) -> None:
|
||||
if value in (None, "") and not required:
|
||||
return
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"{path} must be a non-empty URL")
|
||||
return
|
||||
candidate = value.strip()
|
||||
if data_prefix and candidate.lower().startswith(data_prefix):
|
||||
return
|
||||
parsed = urlparse(candidate)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
errors.append(f"{path} must use an http(s) URL")
|
||||
|
||||
|
||||
def _validate_remote_media_tree(value: Any, path: str, errors: list[str]) -> None:
|
||||
if isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
_validate_remote_media_tree(item, f"{path}[{index}]", errors)
|
||||
return
|
||||
if not isinstance(value, dict):
|
||||
return
|
||||
media = value.get("media")
|
||||
if isinstance(media, dict) and "type" in media:
|
||||
media_path = f"{path}.media"
|
||||
media_type = media.get("type")
|
||||
if media_type not in REMOTE_MEDIA_TYPES:
|
||||
errors.append(
|
||||
f"{media_path}.type must be one of: {', '.join(sorted(REMOTE_MEDIA_TYPES))}"
|
||||
)
|
||||
elif media_type in {"image", "video"}:
|
||||
_remote_url(
|
||||
media.get("mediaUrl"),
|
||||
f"{media_path}.mediaUrl",
|
||||
errors,
|
||||
required=True,
|
||||
data_prefix=f"data:{media_type}/",
|
||||
)
|
||||
if media_type == "video":
|
||||
_remote_url(
|
||||
media.get("posterUrl"),
|
||||
f"{media_path}.posterUrl",
|
||||
errors,
|
||||
data_prefix="data:image/",
|
||||
)
|
||||
if media_type == "webpage":
|
||||
_remote_url(media.get("pageUrl"), f"{media_path}.pageUrl", errors, required=True)
|
||||
_remote_url(
|
||||
media.get("mediaUrl"),
|
||||
f"{media_path}.mediaUrl",
|
||||
errors,
|
||||
data_prefix="data:image/",
|
||||
)
|
||||
if "embed" in media and not isinstance(media.get("embed"), bool):
|
||||
errors.append(f"{media_path}.embed must be a boolean")
|
||||
elif media_type in {"image", "video"}:
|
||||
_remote_url(media.get("pageUrl"), f"{media_path}.pageUrl", errors)
|
||||
for key, item in value.items():
|
||||
if key != "media":
|
||||
_validate_remote_media_tree(item, f"{path}.{key}", errors)
|
||||
|
||||
|
||||
def _walk_chinese(value: Any, path: str, errors: list[str], key: str = "") -> None:
|
||||
if isinstance(value, dict):
|
||||
for child, item in value.items():
|
||||
_walk_chinese(item, f"{path}.{child}", errors, child)
|
||||
elif isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
_walk_chinese(item, f"{path}[{index}]", errors, key)
|
||||
elif isinstance(value, str) and key not in TECHNICAL_KEYS and not key.lower().endswith(("id", "href", "url")) and (KANA_RE.search(value) or (LATIN_RE.search(value) and not CJK_RE.search(value))):
|
||||
errors.append(f"{path} must use Simplified Chinese reader-facing copy")
|
||||
|
||||
|
||||
def validate_document(data: dict[str, Any]) -> dict[str, int]:
|
||||
errors: list[str] = []
|
||||
missing = [name for name in MODULES if name not in data]
|
||||
extra = [name for name in data if name not in MODULES]
|
||||
if missing:
|
||||
errors.append(f"missing top-level module(s): {', '.join(missing)}")
|
||||
if extra:
|
||||
errors.append(f"unknown top-level module(s): {', '.join(extra)}")
|
||||
|
||||
site = _mapping(data.get("site"), "site", errors)
|
||||
_strings(
|
||||
site,
|
||||
"site",
|
||||
(
|
||||
"brandPrimary",
|
||||
"brandRoman",
|
||||
"pageTitle",
|
||||
"description",
|
||||
"language",
|
||||
"skipLabel",
|
||||
"dateWeather",
|
||||
"subscribeLabel",
|
||||
"trialLabel",
|
||||
"trialMessage",
|
||||
"fontLabel",
|
||||
"fontNormal",
|
||||
"fontLarge",
|
||||
"fontNormalMessage",
|
||||
"fontLargeMessage",
|
||||
"memberLabel",
|
||||
),
|
||||
errors,
|
||||
)
|
||||
if site.get("language") != "zh-CN":
|
||||
errors.append("site.language must be zh-CN")
|
||||
|
||||
navigation = _mapping(data.get("navigation"), "navigation", errors)
|
||||
top_left = _links_or_actions(navigation.get("topLeft"), "navigation.topLeft", errors, 1, 8)
|
||||
top_right = _links_or_actions(navigation.get("topRight"), "navigation.topRight", errors, 1, 8)
|
||||
primary = _items(navigation.get("primary"), "navigation.primary", errors, minimum=5, maximum=14, string_keys=("label",), href=True)
|
||||
_strings(
|
||||
navigation,
|
||||
"navigation",
|
||||
(
|
||||
"menuLabel",
|
||||
"menuCloseLabel",
|
||||
"searchTitle",
|
||||
"searchPlaceholder",
|
||||
"searchHint",
|
||||
"searchEmpty",
|
||||
"searchPrefix",
|
||||
"searchCloseLabel",
|
||||
"loginTitle",
|
||||
"loginHeading",
|
||||
"loginNote",
|
||||
"emailLabel",
|
||||
"passwordLabel",
|
||||
"loginButton",
|
||||
"loginResult",
|
||||
"loginCloseLabel",
|
||||
),
|
||||
errors,
|
||||
)
|
||||
mega_columns = _items(navigation.get("megaColumns"), "navigation.megaColumns", errors, minimum=2, maximum=5, string_keys=("title",))
|
||||
mega_links = 0
|
||||
for index, column in enumerate(mega_columns):
|
||||
mega_links += len(_links_or_actions(column.get("links"), f"navigation.megaColumns[{index}].links", errors, 1, 8))
|
||||
|
||||
breaking = _mapping(data.get("breaking"), "breaking", errors)
|
||||
_strings(breaking, "breaking", ("label", "featured", "openLabel", "closeLabel"), errors)
|
||||
breaking_items = _items(breaking.get("items"), "breaking.items", errors, minimum=1, maximum=6, string_keys=("time", "title"), href=True)
|
||||
|
||||
lead = _mapping(data.get("lead"), "lead", errors)
|
||||
main = _mapping(lead.get("main"), "lead.main", errors)
|
||||
_strings(main, "lead.main", ("id", "kicker", "title", "summary", "time", "visual", "saveLabel", "savedLabel", "savedMessage", "removedMessage"), errors)
|
||||
if main.get("visual") not in LEAD_VISUALS:
|
||||
errors.append(f"lead.main.visual must be one of {sorted(LEAD_VISUALS)}")
|
||||
side_stories = _items(lead.get("sideStories"), "lead.sideStories", errors, minimum=1, maximum=4, string_keys=("kicker", "title", "summary", "visual"), visual_key="visual", visuals=LEAD_VISUALS)
|
||||
_strings(lead, "lead", ("timelineTitle", "timelineMoreLabel"), errors)
|
||||
_href(lead.get("timelineMoreHref"), "lead.timelineMoreHref", errors)
|
||||
timeline = _items(lead.get("timeline"), "lead.timeline", errors, minimum=3, maximum=8, string_keys=("time", "title"), href=True)
|
||||
|
||||
edition = _mapping(data.get("edition"), "edition", errors)
|
||||
_strings(edition, "edition", ("brand", "coverTitle", "title", "summary", "paperLabel", "trialLabel", "modalTitle", "date", "closeLabel", "previousLabel", "nextLabel"), errors)
|
||||
pages = _items(edition.get("pages"), "edition.pages", errors, minimum=2, maximum=8, string_keys=("section", "headline", "summary"))
|
||||
|
||||
features = _mapping(data.get("features"), "features", errors)
|
||||
_strings(features, "features", ("title", "moreLabel", "seriesTitle"), errors)
|
||||
_href(features.get("moreHref"), "features.moreHref", errors)
|
||||
feature_main = _mapping(features.get("main"), "features.main", errors)
|
||||
_strings(feature_main, "features.main", ("id", "kicker", "title", "summary", "visual"), errors)
|
||||
if feature_main.get("visual") not in FEATURE_VISUALS:
|
||||
errors.append(f"features.main.visual must be one of {sorted(FEATURE_VISUALS)}")
|
||||
feature_cards = _items(features.get("cards"), "features.cards", errors, minimum=2, maximum=6, string_keys=("kicker", "title", "summary"))
|
||||
series = _items(features.get("series"), "features.series", errors, minimum=2, maximum=8, string_keys=("title", "meta"), href=True)
|
||||
|
||||
news = _mapping(data.get("news"), "news", errors)
|
||||
_strings(news, "news", ("title", "moreLabel"), errors)
|
||||
_href(news.get("moreHref"), "news.moreHref", errors)
|
||||
news_tabs = _items(news.get("tabs"), "news.tabs", errors, minimum=2, maximum=6, string_keys=("label",))
|
||||
news_items = 0
|
||||
for index, tab in enumerate(news_tabs):
|
||||
feature = _mapping(tab.get("feature"), f"news.tabs[{index}].feature", errors)
|
||||
_strings(feature, f"news.tabs[{index}].feature", ("kicker", "title", "summary", "visual"), errors)
|
||||
if feature.get("visual") not in NEWS_VISUALS:
|
||||
errors.append(f"news.tabs[{index}].feature.visual must be one of {sorted(NEWS_VISUALS)}")
|
||||
news_items += len(_items(tab.get("items"), f"news.tabs[{index}].items", errors, minimum=1, maximum=6, string_keys=("kicker", "title", "time")))
|
||||
|
||||
opinion = _mapping(data.get("opinion"), "opinion", errors)
|
||||
_strings(opinion, "opinion", ("title", "lifeTitle", "cultureTitle"), errors)
|
||||
editorial = _mapping(opinion.get("editorial"), "opinion.editorial", errors)
|
||||
_strings(editorial, "opinion.editorial", ("label", "title", "summary", "actionLabel"), errors)
|
||||
_href(editorial.get("href"), "opinion.editorial.href", errors)
|
||||
columns = _items(opinion.get("columns"), "opinion.columns", errors, minimum=2, maximum=5, string_keys=("title", "author", "summary", "visual"), visual_key="visual", visuals=COLUMN_VISUALS)
|
||||
life = _items(opinion.get("life"), "opinion.life", errors, minimum=2, maximum=6, string_keys=("kicker", "title", "summary", "visual"), visual_key="visual", visuals=LIFE_VISUALS)
|
||||
culture = _items(opinion.get("culture"), "opinion.culture", errors, minimum=2, maximum=6, string_keys=("kicker", "title", "summary", "actionLabel"), href=True)
|
||||
|
||||
media = _mapping(data.get("media"), "media", errors)
|
||||
_strings(media, "media", ("title", "moreLabel", "modalBrand", "modalNote", "closeLabel", "captionsOn", "captionsOff", "newsletterTitle", "newsletterText", "emailLabel", "emailPlaceholder", "buttonLabel", "successPrefix"), errors)
|
||||
_href(media.get("moreHref"), "media.moreHref", errors)
|
||||
videos = _items(media.get("videos"), "media.videos", errors, minimum=1, maximum=6, string_keys=("kicker", "title", "visual"), visual_key="visual", visuals=VIDEO_VISUALS)
|
||||
|
||||
footer = _mapping(data.get("footer"), "footer", errors)
|
||||
_strings(footer, "footer", ("brand", "description", "copyright", "legal"), errors)
|
||||
footer_columns = _items(footer.get("columns"), "footer.columns", errors, minimum=2, maximum=6, string_keys=("title",))
|
||||
footer_links = 0
|
||||
for index, column in enumerate(footer_columns):
|
||||
footer_links += len(_items(column.get("links"), f"footer.columns[{index}].links", errors, minimum=1, maximum=8, string_keys=("label",), href=True))
|
||||
|
||||
_validate_remote_media_tree(data, "$", errors)
|
||||
_walk_no_remote(data, "$", errors)
|
||||
_walk_chinese(data, "$", errors)
|
||||
if errors:
|
||||
raise DataValidationError("\n".join(f"- {error}" for error in errors))
|
||||
return {
|
||||
"top_links": len(top_left) + len(top_right),
|
||||
"primary_links": len(primary),
|
||||
"mega_links": mega_links,
|
||||
"breaking_items": len(breaking_items),
|
||||
"lead_stories": 1 + len(side_stories),
|
||||
"timeline_items": len(timeline),
|
||||
"edition_pages": len(pages),
|
||||
"feature_cards": len(feature_cards),
|
||||
"series_items": len(series),
|
||||
"news_tabs": len(news_tabs),
|
||||
"news_items": news_items,
|
||||
"opinion_columns": len(columns),
|
||||
"life_items": len(life),
|
||||
"culture_items": len(culture),
|
||||
"videos": len(videos),
|
||||
"footer_columns": len(footer_columns),
|
||||
"footer_links": footer_links,
|
||||
}
|
||||
|
||||
|
||||
def render_document(data: dict[str, Any], template: str, runtime: str) -> str:
|
||||
validate_document(data)
|
||||
rendered = template.replace("__YOMIURI_NEWS_PAGE_DATA__", json.dumps(data, ensure_ascii=False, separators=(",", ":"))).replace("__YOMIURI_NEWS_PAGE_RUNTIME__", runtime)
|
||||
if "__YOMIURI_NEWS_PAGE_" in rendered:
|
||||
raise DataValidationError("template placeholders remain after rendering")
|
||||
return rendered
|
||||
|
||||
|
||||
def _write_new(path: Path, text: str) -> None:
|
||||
if path.exists():
|
||||
raise DataValidationError(f"refusing to overwrite existing path: {path}")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _init(output: Path) -> dict[str, Any]:
|
||||
data = _read_json(DEFAULT_DATA)
|
||||
counts = validate_document(data)
|
||||
_write_new(output, json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
return {"status": "ok", "action": "init", "output": str(output), "modules": list(MODULES), "counts": counts}
|
||||
|
||||
|
||||
def _validate(input_path: Path) -> dict[str, Any]:
|
||||
counts = validate_document(_read_json(input_path))
|
||||
return {"status": "ok", "action": "validate", "input": str(input_path), "modules": list(MODULES), "counts": counts}
|
||||
|
||||
|
||||
def _render(input_path: Path, output: Path) -> dict[str, Any]:
|
||||
data = _read_json(input_path)
|
||||
counts = validate_document(data)
|
||||
html = render_document(data, TEMPLATE_PATH.read_text(encoding="utf-8"), RUNTIME_PATH.read_text(encoding="utf-8"))
|
||||
_write_new(output, html)
|
||||
return {"status": "ok", "action": "render", "input": str(input_path), "output": str(output), "modules": list(MODULES), "counts": counts}
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Render the bundled Chinese Yomiuri-style news portal")
|
||||
subs = parser.add_subparsers(dest="action", required=True)
|
||||
init_parser = subs.add_parser("init")
|
||||
init_parser.add_argument("--output", type=Path, required=True)
|
||||
validate_parser = subs.add_parser("validate")
|
||||
validate_parser.add_argument("--input", type=Path, required=True)
|
||||
render_parser = subs.add_parser("render")
|
||||
render_parser.add_argument("--input", type=Path, required=True)
|
||||
render_parser.add_argument("--output", type=Path, required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
try:
|
||||
result = _init(args.output) if args.action == "init" else _validate(args.input) if args.action == "validate" else _render(args.input, args.output)
|
||||
except DataValidationError as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user