sync update from LAN repo (source master @ ca48c4ec)
This commit is contained in:
1 parent
8bc8b0e082
commit
ea9dcff9c0
873 files changed
+107866
-14011
No files matched your search
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "_shared"))
|
||||
|
||||
import render_yahoo_japan_page as renderer
|
||||
from fast_patch_page import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run(renderer, output_stem="yahoo-japan-page"))
|
||||
+422
@@ -0,0 +1,422 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_DATA = SKILL_ROOT / "assets" / "default-data.json"
|
||||
TEMPLATE_PATH = SKILL_ROOT / "assets" / "yahoo-japan-template.html"
|
||||
RUNTIME_PATH = SKILL_ROOT / "assets" / "yahoo-japan-runtime.js"
|
||||
MODULES = (
|
||||
"site",
|
||||
"search",
|
||||
"services",
|
||||
"news",
|
||||
"account",
|
||||
"weather",
|
||||
"utilities",
|
||||
"shopping",
|
||||
"pulse",
|
||||
"footer",
|
||||
)
|
||||
SERVICE_VISUALS = {
|
||||
"shopping", "auction", "weather", "news", "sports", "finance", "transit",
|
||||
"property", "car", "travel", "calendar", "community", "tax", "jobs", "books",
|
||||
}
|
||||
RECOMMEND_VISUALS = {"trail", "recipe", "plaza"}
|
||||
PRODUCT_VISUALS = {"backpack", "clock", "umbrella", "mill", "purifier"}
|
||||
FEATURE_VISUALS = {"transit-map"}
|
||||
REMOTE_MEDIA_TYPES = {"image", "video", "webpage"}
|
||||
MEDIA_URL_KEYS = {"mediaUrl", "posterUrl", "pageUrl"}
|
||||
TECHNICAL_KEYS = {
|
||||
"id", "href", "visual", "theme", "language", "brandPrimary", "brandAccent",
|
||||
"value", "price", "score", "change", "previous", "rank", "type",
|
||||
"mediaUrl", "posterUrl", "pageUrl",
|
||||
}
|
||||
LATIN_RE = re.compile(r"[A-Za-z]")
|
||||
KANA_RE = re.compile(r"[\u3040-\u30ff]")
|
||||
CJK_RE = re.compile(r"[\u3400-\u9fff]")
|
||||
REMOTE_RE = re.compile(r"(?:https?://|javascript:|<\s*(?:script|iframe)\b)", re.I)
|
||||
|
||||
|
||||
class DataValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise DataValidationError(f"unable to read valid JSON from {path}: {exc}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise DataValidationError("top-level JSON must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _mapping(value: Any, path: str, errors: list[str]) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
errors.append(f"{path} must be an object")
|
||||
return {}
|
||||
return value
|
||||
|
||||
|
||||
def _list(value: Any, path: str, errors: list[str], minimum: int = 0, maximum: int = 99) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
errors.append(f"{path} must be an array")
|
||||
return []
|
||||
if not minimum <= len(value) <= maximum:
|
||||
errors.append(f"{path} must contain {minimum}-{maximum} items")
|
||||
return value
|
||||
|
||||
|
||||
def _strings(obj: dict[str, Any], path: str, keys: tuple[str, ...], errors: list[str]) -> None:
|
||||
for key in keys:
|
||||
if not isinstance(obj.get(key), str) or not obj[key].strip():
|
||||
errors.append(f"{path}.{key} must be a non-empty string")
|
||||
|
||||
|
||||
def _href(value: Any, path: str, errors: list[str]) -> None:
|
||||
if not isinstance(value, str) or not value.startswith("#"):
|
||||
errors.append(f"{path} must start with #")
|
||||
|
||||
|
||||
def _items(
|
||||
value: Any,
|
||||
path: str,
|
||||
errors: list[str],
|
||||
*,
|
||||
minimum: int,
|
||||
maximum: int,
|
||||
string_keys: tuple[str, ...],
|
||||
href_key: str | None = None,
|
||||
visual_key: str | None = None,
|
||||
visuals: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
raw = _list(value, path, errors, minimum, maximum)
|
||||
result: list[dict[str, Any]] = []
|
||||
ids: list[str] = []
|
||||
for index, item in enumerate(raw):
|
||||
current = _mapping(item, f"{path}[{index}]", errors)
|
||||
_strings(current, f"{path}[{index}]", ("id",) + string_keys, errors)
|
||||
item_id = current.get("id")
|
||||
if isinstance(item_id, str):
|
||||
ids.append(item_id)
|
||||
if href_key:
|
||||
_href(current.get(href_key), f"{path}[{index}].{href_key}", errors)
|
||||
if visual_key and current.get(visual_key) not in (visuals or set()):
|
||||
errors.append(f"{path}[{index}].{visual_key} must be one of {sorted(visuals or set())}")
|
||||
result.append(current)
|
||||
duplicates = sorted({item_id for item_id in ids if ids.count(item_id) > 1})
|
||||
if duplicates:
|
||||
errors.append(f"{path} id duplicates: {', '.join(duplicates)}")
|
||||
return result
|
||||
|
||||
|
||||
def _walk_no_remote(value: Any, path: str, errors: list[str]) -> None:
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if key in MEDIA_URL_KEYS:
|
||||
continue
|
||||
_walk_no_remote(item, f"{path}.{key}", errors)
|
||||
elif isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
_walk_no_remote(item, f"{path}[{index}]", errors)
|
||||
elif isinstance(value, str) and REMOTE_RE.search(value):
|
||||
errors.append(f"{path} must not contain remote URLs, scripts, or iframes")
|
||||
|
||||
|
||||
def _remote_url(value: Any, path: str, errors: list[str], *, required: bool = False, data_prefix: str = "") -> None:
|
||||
if value in (None, "") and not required:
|
||||
return
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"{path} must be a non-empty URL")
|
||||
return
|
||||
candidate = value.strip()
|
||||
if data_prefix and candidate.lower().startswith(data_prefix):
|
||||
return
|
||||
parsed = urlparse(candidate)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
errors.append(f"{path} must use an http(s) URL")
|
||||
|
||||
|
||||
def _validate_remote_media_tree(value: Any, path: str, errors: list[str]) -> None:
|
||||
if isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
_validate_remote_media_tree(item, f"{path}[{index}]", errors)
|
||||
return
|
||||
if not isinstance(value, dict):
|
||||
return
|
||||
media = value.get("media")
|
||||
if isinstance(media, dict) and "type" in media:
|
||||
media_path, media_type = f"{path}.media", media.get("type")
|
||||
if media_type not in REMOTE_MEDIA_TYPES:
|
||||
errors.append(
|
||||
f"{media_path}.type must be one of: "
|
||||
f"{', '.join(sorted(REMOTE_MEDIA_TYPES))}"
|
||||
)
|
||||
elif media_type in {"image", "video"}:
|
||||
_remote_url(
|
||||
media.get("mediaUrl"),
|
||||
f"{media_path}.mediaUrl",
|
||||
errors,
|
||||
required=True,
|
||||
data_prefix=f"data:{media_type}/",
|
||||
)
|
||||
if media_type == "video":
|
||||
_remote_url(
|
||||
media.get("posterUrl"),
|
||||
f"{media_path}.posterUrl",
|
||||
errors,
|
||||
data_prefix="data:image/",
|
||||
)
|
||||
if media_type == "webpage":
|
||||
_remote_url(
|
||||
media.get("pageUrl"),
|
||||
f"{media_path}.pageUrl",
|
||||
errors,
|
||||
required=True,
|
||||
)
|
||||
_remote_url(
|
||||
media.get("mediaUrl"),
|
||||
f"{media_path}.mediaUrl",
|
||||
errors,
|
||||
data_prefix="data:image/",
|
||||
)
|
||||
if "embed" in media and not isinstance(media.get("embed"), bool):
|
||||
errors.append(f"{media_path}.embed must be a boolean")
|
||||
elif media_type in {"image", "video"}:
|
||||
_remote_url(media.get("pageUrl"), f"{media_path}.pageUrl", errors)
|
||||
for key, item in value.items():
|
||||
if key != "media":
|
||||
_validate_remote_media_tree(item, f"{path}.{key}", errors)
|
||||
|
||||
|
||||
def _walk_chinese(value: Any, path: str, errors: list[str], key: str = "") -> None:
|
||||
if isinstance(value, dict):
|
||||
for child, item in value.items():
|
||||
_walk_chinese(item, f"{path}.{child}", errors, child)
|
||||
elif isinstance(value, list):
|
||||
for index, item in enumerate(value):
|
||||
_walk_chinese(item, f"{path}[{index}]", errors, key)
|
||||
elif (
|
||||
isinstance(value, str)
|
||||
and key not in TECHNICAL_KEYS
|
||||
and not key.lower().endswith(("id", "href", "url"))
|
||||
and (KANA_RE.search(value) or (LATIN_RE.search(value) and not CJK_RE.search(value)))
|
||||
):
|
||||
errors.append(f"{path} must use Simplified Chinese reader-facing copy")
|
||||
|
||||
|
||||
def validate_document(data: dict[str, Any]) -> dict[str, int]:
|
||||
errors: list[str] = []
|
||||
missing = [name for name in MODULES if name not in data]
|
||||
extra = [name for name in data if name not in MODULES]
|
||||
if missing:
|
||||
errors.append(f"missing top-level module(s): {', '.join(missing)}")
|
||||
if extra:
|
||||
errors.append(f"unknown top-level module(s): {', '.join(extra)}")
|
||||
|
||||
site = _mapping(data.get("site"), "site", errors)
|
||||
_strings(
|
||||
site,
|
||||
"site",
|
||||
(
|
||||
"brandPrimary", "brandAccent", "tagline", "pageTitle", "description",
|
||||
"language", "theme", "skipLabel", "dateLabel", "loginLabel",
|
||||
"mailLabel", "mailCount", "menuLabel", "themeLabel",
|
||||
"themeDarkMessage", "themeLightMessage",
|
||||
),
|
||||
errors,
|
||||
)
|
||||
if site.get("language") != "zh-CN":
|
||||
errors.append("site.language must be zh-CN")
|
||||
utility_left = _items(site.get("utilityLeft"), "site.utilityLeft", errors, minimum=1, maximum=8, string_keys=("label",), href_key="href")
|
||||
utility_right = _items(site.get("utilityRight"), "site.utilityRight", errors, minimum=1, maximum=8, string_keys=("label",), href_key="href")
|
||||
notice = _mapping(site.get("notice"), "site.notice", errors)
|
||||
_strings(notice, "site.notice", ("label", "text"), errors)
|
||||
_href(notice.get("href"), "site.notice.href", errors)
|
||||
|
||||
search = _mapping(data.get("search"), "search", errors)
|
||||
_strings(search, "search", ("title", "placeholderSuffix", "button", "hotLabel", "emptyMessage", "resultPrefix", "resultKeyword"), errors)
|
||||
search_tabs = _items(search.get("tabs"), "search.tabs", errors, minimum=2, maximum=10, string_keys=("label",))
|
||||
suggestions = _list(search.get("suggestions"), "search.suggestions", errors, 1, 8)
|
||||
if any(not isinstance(item, str) or not item.strip() for item in suggestions):
|
||||
errors.append("search.suggestions must contain non-empty strings")
|
||||
|
||||
services = _mapping(data.get("services"), "services", errors)
|
||||
_strings(services, "services", ("title", "allLabel", "moreLabel", "lessLabel"), errors)
|
||||
_href(services.get("allHref"), "services.allHref", errors)
|
||||
service_primary = _items(services.get("primary"), "services.primary", errors, minimum=5, maximum=12, string_keys=("label", "visual"), href_key="href", visual_key="visual", visuals=SERVICE_VISUALS)
|
||||
service_extra = _items(services.get("extra"), "services.extra", errors, minimum=1, maximum=8, string_keys=("label", "visual"), href_key="href", visual_key="visual", visuals=SERVICE_VISUALS)
|
||||
|
||||
news = _mapping(data.get("news"), "news", errors)
|
||||
_strings(news, "news", ("title", "updated", "moreLabel", "topicsLabel", "recommendTitle", "refreshLabel", "refreshMessage"), errors)
|
||||
_href(news.get("moreHref"), "news.moreHref", errors)
|
||||
_href(news.get("topicsHref"), "news.topicsHref", errors)
|
||||
news_tabs = _items(news.get("tabs"), "news.tabs", errors, minimum=2, maximum=8, string_keys=("label",))
|
||||
news_items = 0
|
||||
for index, tab in enumerate(news_tabs):
|
||||
items = _items(tab.get("items"), f"news.tabs[{index}].items", errors, minimum=1, maximum=8, string_keys=("title", "time"), href_key="href")
|
||||
news_items += len(items)
|
||||
for item_index, item in enumerate(items):
|
||||
if not isinstance(item.get("tag"), str):
|
||||
errors.append(f"news.tabs[{index}].items[{item_index}].tag must be a string")
|
||||
if not isinstance(item.get("isNew"), bool):
|
||||
errors.append(f"news.tabs[{index}].items[{item_index}].isNew must be boolean")
|
||||
feature = _mapping(news.get("feature"), "news.feature", errors)
|
||||
_strings(feature, "news.feature", ("kicker", "title", "summary", "visual"), errors)
|
||||
_href(feature.get("href"), "news.feature.href", errors)
|
||||
if feature.get("visual") not in FEATURE_VISUALS:
|
||||
errors.append(f"news.feature.visual must be one of {sorted(FEATURE_VISUALS)}")
|
||||
recommendations = _items(news.get("recommendations"), "news.recommendations", errors, minimum=1, maximum=6, string_keys=("title", "category", "visual"), visual_key="visual", visuals=RECOMMEND_VISUALS)
|
||||
|
||||
account = _mapping(data.get("account"), "account", errors)
|
||||
_strings(account, "account", ("title", "text", "loginLabel", "newIdLabel", "profileLabel", "modalTitle", "welcome", "demoNote", "emailLabel", "passwordLabel", "submitLabel", "resultMessage", "closeLabel"), errors)
|
||||
_href(account.get("newIdHref"), "account.newIdHref", errors)
|
||||
_href(account.get("profileHref"), "account.profileHref", errors)
|
||||
statuses = _items(account.get("statuses"), "account.statuses", errors, minimum=1, maximum=4, string_keys=("label", "visual"), href_key="href")
|
||||
|
||||
weather = _mapping(data.get("weather"), "weather", errors)
|
||||
_strings(weather, "weather", ("title", "regionLabel", "updatedMessage"), errors)
|
||||
locations = _items(weather.get("locations"), "weather.locations", errors, minimum=2, maximum=6, string_keys=("city",))
|
||||
for index, location in enumerate(locations):
|
||||
for day_name in ("today", "tomorrow"):
|
||||
day = _mapping(location.get(day_name), f"weather.locations[{index}].{day_name}", errors)
|
||||
_strings(day, f"weather.locations[{index}].{day_name}", ("label", "high", "low", "summary"), errors)
|
||||
weather_links = _items(weather.get("links"), "weather.links", errors, minimum=1, maximum=6, string_keys=("label", "message"))
|
||||
|
||||
utilities = _mapping(data.get("utilities"), "utilities", errors)
|
||||
transit = _mapping(utilities.get("transit"), "utilities.transit", errors)
|
||||
_strings(transit, "utilities.transit", ("title", "detailLabel", "status", "fromLabel", "toLabel", "buttonLabel", "emptyMessage", "resultPrefix", "resultDetail"), errors)
|
||||
_href(transit.get("detailHref"), "utilities.transit.detailHref", errors)
|
||||
finance = _mapping(utilities.get("finance"), "utilities.finance", errors)
|
||||
_strings(finance, "utilities.finance", ("title", "moreLabel", "updated", "previousPrefix"), errors)
|
||||
_href(finance.get("moreHref"), "utilities.finance.moreHref", errors)
|
||||
markets = _items(finance.get("markets"), "utilities.finance.markets", errors, minimum=2, maximum=6, string_keys=("label", "value", "change", "previous"))
|
||||
|
||||
shopping = _mapping(data.get("shopping"), "shopping", errors)
|
||||
_strings(shopping, "shopping", ("title", "moreLabel", "saveLabel", "removeLabel", "savedMessage", "removedMessage"), errors)
|
||||
_href(shopping.get("moreHref"), "shopping.moreHref", errors)
|
||||
products = _items(shopping.get("products"), "shopping.products", errors, minimum=2, maximum=8, string_keys=("title", "price", "visual"), visual_key="visual", visuals=PRODUCT_VISUALS)
|
||||
|
||||
pulse = _mapping(data.get("pulse"), "pulse", errors)
|
||||
_strings(pulse, "pulse", ("sportsTitle", "sportsMoreLabel", "trendsTitle", "refreshLabel", "refreshMessage"), errors)
|
||||
_href(pulse.get("sportsMoreHref"), "pulse.sportsMoreHref", errors)
|
||||
sports_tabs = _items(pulse.get("sportsTabs"), "pulse.sportsTabs", errors, minimum=2, maximum=5, string_keys=("label",))
|
||||
sports_games = 0
|
||||
for index, tab in enumerate(sports_tabs):
|
||||
games = _items(tab.get("games"), f"pulse.sportsTabs[{index}].games", errors, minimum=1, maximum=5, string_keys=("home", "homeNote", "score", "away", "awayNote"))
|
||||
sports_games += len(games)
|
||||
trend_tabs = _items(pulse.get("trendTabs"), "pulse.trendTabs", errors, minimum=2, maximum=5, string_keys=("label",))
|
||||
trend_items = 0
|
||||
for index, tab in enumerate(trend_tabs):
|
||||
items = _items(tab.get("items"), f"pulse.trendTabs[{index}].items", errors, minimum=1, maximum=8, string_keys=("rank", "title", "meta"), href_key="href")
|
||||
trend_items += len(items)
|
||||
|
||||
footer = _mapping(data.get("footer"), "footer", errors)
|
||||
_strings(footer, "footer", ("drawerTitle", "drawerText", "copyright", "legal", "backTopLabel"), errors)
|
||||
drawer_links = _items(footer.get("drawerLinks"), "footer.drawerLinks", errors, minimum=2, maximum=12, string_keys=("label",), href_key="href")
|
||||
footer_links = _items(footer.get("links"), "footer.links", errors, minimum=2, maximum=12, string_keys=("label",), href_key="href")
|
||||
|
||||
_validate_remote_media_tree(data, "$", errors)
|
||||
_walk_no_remote(data, "$", errors)
|
||||
_walk_chinese(data, "$", errors)
|
||||
if errors:
|
||||
raise DataValidationError("\n".join(f"- {error}" for error in errors))
|
||||
return {
|
||||
"utility_links": len(utility_left) + len(utility_right),
|
||||
"search_tabs": len(search_tabs),
|
||||
"suggestions": len(suggestions),
|
||||
"services": len(service_primary) + len(service_extra),
|
||||
"news_tabs": len(news_tabs),
|
||||
"news_items": news_items,
|
||||
"recommendations": len(recommendations),
|
||||
"account_statuses": len(statuses),
|
||||
"weather_locations": len(locations),
|
||||
"weather_links": len(weather_links),
|
||||
"markets": len(markets),
|
||||
"products": len(products),
|
||||
"sports_tabs": len(sports_tabs),
|
||||
"sports_games": sports_games,
|
||||
"trend_tabs": len(trend_tabs),
|
||||
"trend_items": trend_items,
|
||||
"drawer_links": len(drawer_links),
|
||||
"footer_links": len(footer_links),
|
||||
}
|
||||
|
||||
|
||||
def render_document(data: dict[str, Any], template: str, runtime: str) -> str:
|
||||
validate_document(data)
|
||||
rendered = template.replace(
|
||||
"__YAHOO_JAPAN_PAGE_DATA__",
|
||||
json.dumps(data, ensure_ascii=False, separators=(",", ":")),
|
||||
).replace("__YAHOO_JAPAN_PAGE_RUNTIME__", runtime)
|
||||
if "__YAHOO_JAPAN_PAGE_" in rendered:
|
||||
raise DataValidationError("template placeholders remain after rendering")
|
||||
return rendered
|
||||
|
||||
|
||||
def _write_new(path: Path, text: str) -> None:
|
||||
if path.exists():
|
||||
raise DataValidationError(f"refusing to overwrite existing path: {path}")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _init(output: Path) -> dict[str, Any]:
|
||||
data = _read_json(DEFAULT_DATA)
|
||||
counts = validate_document(data)
|
||||
_write_new(output, json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
return {"status":"ok","action":"init","output":str(output),"modules":list(MODULES),"counts":counts}
|
||||
|
||||
|
||||
def _validate(input_path: Path) -> dict[str, Any]:
|
||||
counts = validate_document(_read_json(input_path))
|
||||
return {"status":"ok","action":"validate","input":str(input_path),"modules":list(MODULES),"counts":counts}
|
||||
|
||||
|
||||
def _render(input_path: Path, output: Path) -> dict[str, Any]:
|
||||
data = _read_json(input_path)
|
||||
counts = validate_document(data)
|
||||
html = render_document(data, TEMPLATE_PATH.read_text(encoding="utf-8"), RUNTIME_PATH.read_text(encoding="utf-8"))
|
||||
_write_new(output, html)
|
||||
return {"status":"ok","action":"render","input":str(input_path),"output":str(output),"modules":list(MODULES),"counts":counts}
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Render the bundled Chinese Yahoo! JAPAN-style portal")
|
||||
subparsers = parser.add_subparsers(dest="action", required=True)
|
||||
init_parser = subparsers.add_parser("init")
|
||||
init_parser.add_argument("--output", type=Path, required=True)
|
||||
validate_parser = subparsers.add_parser("validate")
|
||||
validate_parser.add_argument("--input", type=Path, required=True)
|
||||
render_parser = subparsers.add_parser("render")
|
||||
render_parser.add_argument("--input", type=Path, required=True)
|
||||
render_parser.add_argument("--output", type=Path, required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
try:
|
||||
if args.action == "init":
|
||||
result = _init(args.output)
|
||||
elif args.action == "validate":
|
||||
result = _validate(args.input)
|
||||
else:
|
||||
result = _render(args.input, args.output)
|
||||
except DataValidationError as exc:
|
||||
print(json.dumps({"status":"error","message":str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user