sync update from LAN repo (source master @ ca48c4ec)
This commit is contained in:
1 parent
8bc8b0e082
commit
ea9dcff9c0
873 files changed
+107866
-14011
No files matched your search
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "_shared"))
|
||||
|
||||
import render_youtube_page as renderer
|
||||
from fast_patch_page import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run(renderer, output_stem="youtube-page"))
|
||||
+512
@@ -0,0 +1,512 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate YouTube page data and inject it into the bundled HTML template."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html as html_lib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
SKILL_ROOT = Path(__file__).resolve().parent.parent
|
||||
ASSETS_DIR = SKILL_ROOT / "assets"
|
||||
DEFAULT_DATA_PATH = ASSETS_DIR / "default-data.json"
|
||||
TEMPLATE_PATH = ASSETS_DIR / "youtube-template.html"
|
||||
RUNTIME_PATH = ASSETS_DIR / "youtube-runtime.js"
|
||||
DATA_MARKER = "__YOUTUBE_PAGE_DATA__"
|
||||
RUNTIME_MARKER = "__YOUTUBE_PAGE_RUNTIME__"
|
||||
TITLE_RE = re.compile(r"<title>[^<]*</title>", re.IGNORECASE)
|
||||
MODULES = (
|
||||
"site",
|
||||
"navigation",
|
||||
"categories",
|
||||
"videos",
|
||||
"shorts",
|
||||
"subscriptions",
|
||||
"comments",
|
||||
)
|
||||
VISUAL_TYPES = {
|
||||
"city",
|
||||
"studio",
|
||||
"food",
|
||||
"mountain",
|
||||
"code",
|
||||
"science",
|
||||
"music",
|
||||
"travel",
|
||||
}
|
||||
REMOTE_MEDIA_TYPES = {"image", "video", "webpage"}
|
||||
|
||||
|
||||
class DataValidationError(ValueError):
|
||||
"""Raised when the YouTube page JSON cannot satisfy the template contract."""
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise DataValidationError(f"JSON file does not exist: {path}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DataValidationError(
|
||||
f"Invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}"
|
||||
) from exc
|
||||
if not isinstance(data, dict):
|
||||
raise DataValidationError("Top-level JSON value must be an object")
|
||||
return data
|
||||
|
||||
|
||||
def _mapping(value: Any, path: str, errors: list[str]) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
errors.append(f"{path} must be an object")
|
||||
return {}
|
||||
return value
|
||||
|
||||
|
||||
def _array(
|
||||
value: Any,
|
||||
path: str,
|
||||
errors: list[str],
|
||||
*,
|
||||
limit: int | None = None,
|
||||
) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
errors.append(f"{path} must be an array")
|
||||
return []
|
||||
if limit is not None and len(value) > limit:
|
||||
errors.append(f"{path} may contain at most {limit} items")
|
||||
return value
|
||||
|
||||
|
||||
def _non_empty_string(value: Any, path: str, errors: list[str]) -> None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"{path} must be a non-empty string")
|
||||
|
||||
|
||||
def _remote_url(value: Any, path: str, errors: list[str], *, required: bool = False, data_prefix: str = "") -> None:
|
||||
if value in (None, "") and not required:
|
||||
return
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"{path} must be a non-empty URL")
|
||||
return
|
||||
candidate = value.strip()
|
||||
if data_prefix and candidate.lower().startswith(data_prefix):
|
||||
return
|
||||
parsed = urlparse(candidate)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
errors.append(f"{path} must use an http(s) URL")
|
||||
|
||||
|
||||
def _validate_remote_media(value: Any, path: str, errors: list[str]) -> None:
|
||||
if value is None:
|
||||
return
|
||||
media = _mapping(value, path, errors)
|
||||
media_type = media.get("type")
|
||||
if media_type not in REMOTE_MEDIA_TYPES:
|
||||
errors.append(f"{path}.type must be one of: {', '.join(sorted(REMOTE_MEDIA_TYPES))}")
|
||||
return
|
||||
if media_type in {"image", "video"}:
|
||||
_remote_url(media.get("mediaUrl"), f"{path}.mediaUrl", errors, required=True, data_prefix=f"data:{media_type}/")
|
||||
if media_type == "video":
|
||||
_remote_url(media.get("posterUrl"), f"{path}.posterUrl", errors, data_prefix="data:image/")
|
||||
if media_type == "webpage":
|
||||
_remote_url(media.get("pageUrl"), f"{path}.pageUrl", errors, required=True)
|
||||
_remote_url(media.get("mediaUrl"), f"{path}.mediaUrl", errors, data_prefix="data:image/")
|
||||
if "embed" in media and not isinstance(media.get("embed"), bool):
|
||||
errors.append(f"{path}.embed must be a boolean")
|
||||
else:
|
||||
_remote_url(media.get("pageUrl"), f"{path}.pageUrl", errors)
|
||||
|
||||
|
||||
def _non_negative_number(value: Any, path: str, errors: list[str]) -> None:
|
||||
if (
|
||||
not isinstance(value, (int, float))
|
||||
or isinstance(value, bool)
|
||||
or value < 0
|
||||
):
|
||||
errors.append(f"{path} must be a non-negative number")
|
||||
|
||||
|
||||
def _colors(value: Any, path: str, errors: list[str]) -> None:
|
||||
if (
|
||||
not isinstance(value, list)
|
||||
or len(value) != 2
|
||||
or not all(isinstance(color, str) for color in value)
|
||||
):
|
||||
errors.append(f"{path} must contain exactly two color strings")
|
||||
|
||||
|
||||
def _validate_people(items: list[Any], path: str, errors: list[str]) -> None:
|
||||
for index, item in enumerate(items):
|
||||
person = _mapping(item, f"{path}[{index}]", errors)
|
||||
for key in ("name", "handle", "initial"):
|
||||
_non_empty_string(
|
||||
person.get(key),
|
||||
f"{path}[{index}].{key}",
|
||||
errors,
|
||||
)
|
||||
_colors(person.get("colors"), f"{path}[{index}].colors", errors)
|
||||
|
||||
|
||||
def validate_document(data: dict[str, Any]) -> dict[str, Any]:
|
||||
errors: list[str] = []
|
||||
for module in MODULES:
|
||||
if module not in data:
|
||||
errors.append(f"Missing top-level module: {module}")
|
||||
|
||||
site = _mapping(data.get("site"), "site", errors)
|
||||
for key in (
|
||||
"brand",
|
||||
"pageTitle",
|
||||
"description",
|
||||
"searchPlaceholder",
|
||||
"createLabel",
|
||||
):
|
||||
_non_empty_string(site.get(key), f"site.{key}", errors)
|
||||
current_user = _mapping(site.get("currentUser"), "site.currentUser", errors)
|
||||
for key in ("name", "handle", "initial"):
|
||||
_non_empty_string(
|
||||
current_user.get(key),
|
||||
f"site.currentUser.{key}",
|
||||
errors,
|
||||
)
|
||||
_colors(current_user.get("colors"), "site.currentUser.colors", errors)
|
||||
_non_negative_number(
|
||||
current_user.get("notifications"),
|
||||
"site.currentUser.notifications",
|
||||
errors,
|
||||
)
|
||||
|
||||
navigation = _mapping(data.get("navigation"), "navigation", errors)
|
||||
for key, limit in (("primary", 12), ("library", 12), ("mobile", 6)):
|
||||
items = _array(
|
||||
navigation.get(key),
|
||||
f"navigation.{key}",
|
||||
errors,
|
||||
limit=limit,
|
||||
)
|
||||
for index, item in enumerate(items):
|
||||
nav_item = _mapping(
|
||||
item,
|
||||
f"navigation.{key}[{index}]",
|
||||
errors,
|
||||
)
|
||||
for field in ("label", "icon"):
|
||||
_non_empty_string(
|
||||
nav_item.get(field),
|
||||
f"navigation.{key}[{index}].{field}",
|
||||
errors,
|
||||
)
|
||||
_non_empty_string(navigation.get("footer"), "navigation.footer", errors)
|
||||
|
||||
categories = _mapping(data.get("categories"), "categories", errors)
|
||||
category_items = _array(
|
||||
categories.get("items"),
|
||||
"categories.items",
|
||||
errors,
|
||||
limit=20,
|
||||
)
|
||||
if not category_items:
|
||||
errors.append("categories.items must contain at least one category")
|
||||
if not all(
|
||||
isinstance(item, str) and item.strip() for item in category_items
|
||||
):
|
||||
errors.append("categories.items must contain non-empty strings")
|
||||
|
||||
videos = _mapping(data.get("videos"), "videos", errors)
|
||||
_non_empty_string(videos.get("sectionTitle"), "videos.sectionTitle", errors)
|
||||
video_items = _array(
|
||||
videos.get("items"),
|
||||
"videos.items",
|
||||
errors,
|
||||
limit=30,
|
||||
)
|
||||
if not video_items:
|
||||
errors.append("videos.items must contain at least one video")
|
||||
video_ids: set[str] = set()
|
||||
for index, item in enumerate(video_items):
|
||||
video = _mapping(item, f"videos.items[{index}]", errors)
|
||||
video_id = video.get("id")
|
||||
_non_empty_string(video_id, f"videos.items[{index}].id", errors)
|
||||
if isinstance(video_id, str):
|
||||
if video_id in video_ids:
|
||||
errors.append(f"videos.items[{index}].id must be unique")
|
||||
video_ids.add(video_id)
|
||||
for key in (
|
||||
"title",
|
||||
"channel",
|
||||
"channelInitial",
|
||||
"category",
|
||||
"views",
|
||||
"time",
|
||||
"description",
|
||||
):
|
||||
_non_empty_string(
|
||||
video.get(key),
|
||||
f"videos.items[{index}].{key}",
|
||||
errors,
|
||||
)
|
||||
_colors(
|
||||
video.get("channelColors"),
|
||||
f"videos.items[{index}].channelColors",
|
||||
errors,
|
||||
)
|
||||
if video.get("visual") not in VISUAL_TYPES:
|
||||
errors.append(
|
||||
f"videos.items[{index}].visual must be one of: "
|
||||
f"{', '.join(sorted(VISUAL_TYPES))}"
|
||||
)
|
||||
_validate_remote_media(video.get("media"), f"videos.items[{index}].media", errors)
|
||||
metrics = _mapping(
|
||||
video.get("metrics"),
|
||||
f"videos.items[{index}].metrics",
|
||||
errors,
|
||||
)
|
||||
for key in ("likes", "comments"):
|
||||
_non_negative_number(
|
||||
metrics.get(key),
|
||||
f"videos.items[{index}].metrics.{key}",
|
||||
errors,
|
||||
)
|
||||
|
||||
shorts = _mapping(data.get("shorts"), "shorts", errors)
|
||||
_non_empty_string(shorts.get("sectionTitle"), "shorts.sectionTitle", errors)
|
||||
short_items = _array(
|
||||
shorts.get("items"),
|
||||
"shorts.items",
|
||||
errors,
|
||||
limit=20,
|
||||
)
|
||||
short_ids: set[str] = set()
|
||||
for index, item in enumerate(short_items):
|
||||
short = _mapping(item, f"shorts.items[{index}]", errors)
|
||||
short_id = short.get("id")
|
||||
_non_empty_string(short_id, f"shorts.items[{index}].id", errors)
|
||||
if isinstance(short_id, str):
|
||||
if short_id in short_ids:
|
||||
errors.append(f"shorts.items[{index}].id must be unique")
|
||||
short_ids.add(short_id)
|
||||
for key in ("title", "views"):
|
||||
_non_empty_string(
|
||||
short.get(key),
|
||||
f"shorts.items[{index}].{key}",
|
||||
errors,
|
||||
)
|
||||
if short.get("visual") not in VISUAL_TYPES:
|
||||
errors.append(
|
||||
f"shorts.items[{index}].visual must be one of: "
|
||||
f"{', '.join(sorted(VISUAL_TYPES))}"
|
||||
)
|
||||
_validate_remote_media(short.get("media"), f"shorts.items[{index}].media", errors)
|
||||
|
||||
subscriptions = _mapping(
|
||||
data.get("subscriptions"),
|
||||
"subscriptions",
|
||||
errors,
|
||||
)
|
||||
_non_empty_string(
|
||||
subscriptions.get("title"),
|
||||
"subscriptions.title",
|
||||
errors,
|
||||
)
|
||||
subscription_items = _array(
|
||||
subscriptions.get("items"),
|
||||
"subscriptions.items",
|
||||
errors,
|
||||
limit=20,
|
||||
)
|
||||
_validate_people(subscription_items, "subscriptions.items", errors)
|
||||
|
||||
comments = _mapping(data.get("comments"), "comments", errors)
|
||||
for key in ("titleSuffix", "inputPlaceholder"):
|
||||
_non_empty_string(comments.get(key), f"comments.{key}", errors)
|
||||
comment_items = _array(
|
||||
comments.get("items"),
|
||||
"comments.items",
|
||||
errors,
|
||||
limit=40,
|
||||
)
|
||||
comment_ids: set[str] = set()
|
||||
for index, item in enumerate(comment_items):
|
||||
comment = _mapping(item, f"comments.items[{index}]", errors)
|
||||
comment_id = comment.get("id")
|
||||
_non_empty_string(
|
||||
comment_id,
|
||||
f"comments.items[{index}].id",
|
||||
errors,
|
||||
)
|
||||
if isinstance(comment_id, str):
|
||||
if comment_id in comment_ids:
|
||||
errors.append(
|
||||
f"comments.items[{index}].id must be unique"
|
||||
)
|
||||
comment_ids.add(comment_id)
|
||||
for key in ("name", "initial", "text", "time"):
|
||||
_non_empty_string(
|
||||
comment.get(key),
|
||||
f"comments.items[{index}].{key}",
|
||||
errors,
|
||||
)
|
||||
_colors(
|
||||
comment.get("colors"),
|
||||
f"comments.items[{index}].colors",
|
||||
errors,
|
||||
)
|
||||
_non_negative_number(
|
||||
comment.get("likes"),
|
||||
f"comments.items[{index}].likes",
|
||||
errors,
|
||||
)
|
||||
|
||||
if errors:
|
||||
raise DataValidationError("; ".join(errors))
|
||||
|
||||
return {
|
||||
"modules": list(MODULES),
|
||||
"counts": {
|
||||
"categories": len(category_items),
|
||||
"videos": len(video_items),
|
||||
"shorts": len(short_items),
|
||||
"subscriptions": len(subscription_items),
|
||||
"comments": len(comment_items),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_document(data: dict[str, Any], template: str, runtime: str) -> str:
|
||||
validate_document(data)
|
||||
if template.count(DATA_MARKER) != 1:
|
||||
raise DataValidationError(
|
||||
"Bundled template must contain exactly one data marker"
|
||||
)
|
||||
if template.count(RUNTIME_MARKER) != 1:
|
||||
raise DataValidationError(
|
||||
"Bundled template must contain exactly one runtime marker"
|
||||
)
|
||||
serialized = json.dumps(data, ensure_ascii=False, indent=2).replace(
|
||||
"</",
|
||||
"<\\/",
|
||||
)
|
||||
safe_runtime = runtime.replace("</script", "<\\/script")
|
||||
rendered = template.replace(DATA_MARKER, serialized)
|
||||
rendered = rendered.replace(RUNTIME_MARKER, safe_runtime)
|
||||
page_title = str((data.get("site") or {}).get("pageTitle") or "").strip()
|
||||
if page_title:
|
||||
rendered = TITLE_RE.sub(
|
||||
f"<title>{html_lib.escape(page_title)}</title>",
|
||||
rendered,
|
||||
count=1,
|
||||
)
|
||||
return rendered
|
||||
|
||||
|
||||
def _write_new(path: Path, content: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with path.open("x", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(content)
|
||||
except FileExistsError as exc:
|
||||
raise DataValidationError(
|
||||
f"Refusing to overwrite existing file: {path}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _emit(payload: dict[str, Any]) -> None:
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
|
||||
def _command_init(output: Path) -> dict[str, Any]:
|
||||
data = _read_json(DEFAULT_DATA_PATH)
|
||||
summary = validate_document(data)
|
||||
_write_new(
|
||||
output,
|
||||
json.dumps(data, ensure_ascii=False, indent=2) + "\n",
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"action": "init",
|
||||
"output": str(output),
|
||||
**summary,
|
||||
}
|
||||
|
||||
|
||||
def _command_validate(input_path: Path) -> dict[str, Any]:
|
||||
data = _read_json(input_path)
|
||||
summary = validate_document(data)
|
||||
return {
|
||||
"status": "ok",
|
||||
"action": "validate",
|
||||
"input": str(input_path),
|
||||
**summary,
|
||||
}
|
||||
|
||||
|
||||
def _command_render(input_path: Path, output: Path) -> dict[str, Any]:
|
||||
data = _read_json(input_path)
|
||||
template = TEMPLATE_PATH.read_text(encoding="utf-8")
|
||||
runtime = RUNTIME_PATH.read_text(encoding="utf-8")
|
||||
rendered = render_document(data, template, runtime)
|
||||
_write_new(output, rendered)
|
||||
summary = validate_document(data)
|
||||
return {
|
||||
"status": "ok",
|
||||
"action": "render",
|
||||
"input": str(input_path),
|
||||
"output": str(output),
|
||||
**summary,
|
||||
}
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
init_parser = subparsers.add_parser(
|
||||
"init",
|
||||
help="Create a new editable JSON document",
|
||||
)
|
||||
init_parser.add_argument("--output", type=Path, required=True)
|
||||
|
||||
validate_parser = subparsers.add_parser(
|
||||
"validate",
|
||||
help="Validate an existing JSON document",
|
||||
)
|
||||
validate_parser.add_argument("--input", type=Path, required=True)
|
||||
|
||||
render_parser = subparsers.add_parser(
|
||||
"render",
|
||||
help="Render validated JSON into the bundled HTML template",
|
||||
)
|
||||
render_parser.add_argument("--input", type=Path, required=True)
|
||||
render_parser.add_argument("--output", type=Path, required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.command == "init":
|
||||
result = _command_init(args.output)
|
||||
elif args.command == "validate":
|
||||
result = _command_validate(args.input)
|
||||
else:
|
||||
result = _command_render(args.input, args.output)
|
||||
except (DataValidationError, OSError) as exc:
|
||||
_emit({
|
||||
"status": "error",
|
||||
"action": args.command,
|
||||
"error": str(exc),
|
||||
})
|
||||
return 2
|
||||
_emit(result)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
Reference in new issue
Block a user