From d16c025753b7d8e8ea9042b414a3665208d46a17 Mon Sep 17 00:00:00 2001 From: James Price Date: Tue, 16 Jun 2026 16:12:17 -0400 Subject: [PATCH] Fix YouTube downloads: bump yt-dlp, require Node 22, drop YT cookies Two YouTube regressions stacked up: - yt-dlp 2026.3.17 hit "HTTP Error 403: Forbidden" on the actual media download. Bumping to 2026.6.9 fixes the data-download 403, but its EJS n-sig challenge solver now requires Node >= 22, and Debian 13 only ships Node 20 ("node-20.x (unsupported)" -> only image formats resolve). bot.py now points --js-runtimes at a standalone /opt/node22 via the new JS_RUNTIME constant. - With a cookies.txt present, yt-dlp skips the cookieless android_vr/android clients (whose URLs work) and falls back to the `tv` client, whose media URLs 403. New _cookie_args(url) withholds cookies for YouTube hosts only, while still sending them to Instagram/TikTok/X. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 6 +++++- bot.py | 37 ++++++++++++++++++++++++++++++------- requirements.txt | 7 +++++-- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 46ad0ed..c615b29 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,11 @@ Signal ──► signal-cli-rest-api (json-rpc, :8080) ──► bot.py (sig python3 -m venv venv venv/bin/pip install -r requirements.txt ``` - YouTube also needs a system `node` on PATH (for `yt-dlp-ejs`). + YouTube also needs **Node >= 22** on PATH (for `yt-dlp-ejs`'s challenge + solver). If your distro only ships an older Node, install a standalone Node 22+ + and point `bot.py`'s `_NODE22` at it. Note: YouTube cookies are deliberately + withheld (`_cookie_args`) — sending account cookies forces yt-dlp onto a player + client whose media URLs 403; cookieless YouTube requests work. 3. **Cookies:** copy `cookies.txt.example` to `cookies.txt` and fill in real exported cookies for the sites you want auth'd. (gitignored) diff --git a/bot.py b/bot.py index 9d517a2..a149f39 100644 --- a/bot.py +++ b/bot.py @@ -12,7 +12,7 @@ import subprocess import tempfile import time from datetime import datetime -from urllib.parse import quote +from urllib.parse import quote, urlsplit from signalbot import Command, Context, SignalBot from signalbot.command import regex_triggered, triggered @@ -42,6 +42,29 @@ COOKIES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cookies.txt" STATE_DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bot-state.db") ADMIN_NUMBERS = {n.strip() for n in os.environ.get("BOT_ADMINS", "").split(",") if n.strip()} +# YouTube's JS "n-sig" challenge solver (yt-dlp-ejs) requires Node >= 22, but +# Debian 13 only ships Node 20. We install a standalone Node 22 at /opt/node22 and +# point yt-dlp at it explicitly via `--js-runtimes node:`; fall back to PATH +# `node` if that binary is ever missing. +_NODE22 = "/opt/node22/bin/node" +JS_RUNTIME = f"node:{_NODE22}" if os.path.exists(_NODE22) else "node" + +# YouTube is the one host where we must NOT send cookies: with account cookies +# present, yt-dlp skips the cookieless `android_vr`/`android` clients (whose media +# URLs work) and falls back to the `tv` client, whose URLs now 403 on the data +# download. A cookieless YouTube request just works. Cookies are still needed for +# Instagram/TikTok/X, so we withhold them per-URL rather than dropping the file. +_YOUTUBE_HOST_RE = re.compile(r"(?:^|\.)(?:youtube\.com|youtu\.be|youtube-nocookie\.com)$", re.I) + +def _cookie_args(url: str) -> list[str]: + """yt-dlp --cookies args for this URL, withheld for YouTube (see note above).""" + if not os.path.exists(COOKIES): + return [] + host = (urlsplit(url).hostname or "").lower() + if _YOUTUBE_HOST_RE.search(host): + return [] + return ["--cookies", COOKIES] + # --- /force: host an oversized video as a temporary download link --- FORCE_DURATION_THRESHOLD = 3600 # offer /force for videos longer than this (1 hr) FORCE_OFFER_TTL = 600 # a /force offer stays valid for 10 min @@ -484,7 +507,7 @@ def _run_ytdlp(url: str, outpath: str, tmpdir: str, "--no-playlist", # YouTube wraps URLs in a JS "n-sig" challenge; node solves it # via yt-dlp-ejs. Without this, only image/thumb formats resolve. - "--js-runtimes", "node", + "--js-runtimes", JS_RUNTIME, ] if clip is not None: start, end = clip @@ -507,7 +530,7 @@ def _run_ytdlp(url: str, outpath: str, tmpdir: str, ] cmd += [ "--merge-output-format", "mp4", - *(["--cookies", COOKIES] if os.path.exists(COOKIES) else []), + *_cookie_args(url), "-o", outpath, "--", url, ] @@ -587,9 +610,9 @@ def _probe_metadata(url: str) -> tuple[int | None, str | None]: """Fetch (duration_seconds, title) without downloading. Blocking — use to_thread.""" cmd = [ YTDLP, "--no-playlist", "--quiet", "--no-warnings", - "--js-runtimes", "node", + "--js-runtimes", JS_RUNTIME, "--print", "%(duration)s\n%(title)s", - *(["--cookies", COOKIES] if os.path.exists(COOKIES) else []), + *_cookie_args(url), "--", url, ] try: @@ -662,10 +685,10 @@ def _force_download_and_publish(url: str, token: str) -> tuple[str | None, int | os.makedirs(tokdir) cmd = [ *_NICE, - YTDLP, "--no-playlist", "--no-warnings", "--js-runtimes", "node", + YTDLP, "--no-playlist", "--no-warnings", "--js-runtimes", JS_RUNTIME, "-f", "best[ext=mp4]/best", "--merge-output-format", "mp4", "--max-filesize", FORCE_MAX_FILESIZE, - *(["--cookies", COOKIES] if os.path.exists(COOKIES) else []), + *_cookie_args(url), "-o", os.path.join(tokdir, "video.%(ext)s"), "--", url, ] diff --git a/requirements.txt b/requirements.txt index a3c15e3..6947e99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ signalbot==1.1.0 # yt-dlp is kept current for site/extractor changes; bump the floor deliberately. -yt-dlp>=2026.3.17 +# 2026.6.9 fixes a YouTube 403-on-data-download regression; it also requires the +# JS challenge to run on Node >= 22 (see below). +yt-dlp>=2026.6.9 # Needed for YouTube: yt-dlp wraps URLs in a JS "n-sig" challenge that a JS -# runtime must solve. Requires a system `node` on PATH plus this package. +# runtime must solve. Requires Node >= 22 on PATH (bot.py points at a standalone +# /opt/node22 since Debian 13 only ships Node 20) plus this package. yt-dlp-ejs>=0.8.0