Run /force downloads at low CPU + idle IO priority (nice/ionice)
Wrap the forced yt-dlp download (and its node/ffmpeg children) and the rsync upload with 'nice -n 19 ionice -c 3' so the known-big, slow /force job yields the CPU and disk to everything else under load while still finishing fast when idle. Normal short downloads stay at full priority. Best-effort: prefix is empty if the tools are absent. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -61,6 +61,14 @@ _SSH_OPTS = ["-p", FORCE_SSH_PORT, "-o", "BatchMode=yes", "-o", "StrictHostKeyCh
|
|||||||
*(["-i", FORCE_SSH_KEY] if FORCE_SSH_KEY else [])]
|
*(["-i", FORCE_SSH_KEY] if FORCE_SSH_KEY else [])]
|
||||||
_RSYNC_RSH = "ssh " + " ".join(shlex.quote(o) for o in _SSH_OPTS)
|
_RSYNC_RSH = "ssh " + " ".join(shlex.quote(o) for o in _SSH_OPTS)
|
||||||
|
|
||||||
|
# /force is known to be big and slow, so run its subprocesses (yt-dlp + the node
|
||||||
|
# n-sig solver and ffmpeg merge it spawns, plus rsync) at lowest CPU and idle IO
|
||||||
|
# priority. They yield to everything else under load but still run full-speed when
|
||||||
|
# the box is idle; child processes inherit the niceness. Normal short downloads
|
||||||
|
# are left at full priority for responsiveness.
|
||||||
|
_NICE = (["nice", "-n", "19"] if shutil.which("nice") else []) + \
|
||||||
|
(["ionice", "-c", "3"] if shutil.which("ionice") else [])
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||||
@@ -649,6 +657,7 @@ def _force_download_and_publish(url: str, token: str) -> tuple[str | None, int |
|
|||||||
tokdir = os.path.join(stage, token)
|
tokdir = os.path.join(stage, token)
|
||||||
os.makedirs(tokdir)
|
os.makedirs(tokdir)
|
||||||
cmd = [
|
cmd = [
|
||||||
|
*_NICE,
|
||||||
YTDLP, "--no-playlist", "--no-warnings", "--js-runtimes", "node",
|
YTDLP, "--no-playlist", "--no-warnings", "--js-runtimes", "node",
|
||||||
"-f", "best[ext=mp4]/best", "--merge-output-format", "mp4",
|
"-f", "best[ext=mp4]/best", "--merge-output-format", "mp4",
|
||||||
"--max-filesize", FORCE_MAX_FILESIZE,
|
"--max-filesize", FORCE_MAX_FILESIZE,
|
||||||
@@ -674,7 +683,7 @@ def _force_download_and_publish(url: str, token: str) -> tuple[str | None, int |
|
|||||||
size_mb = os.path.getsize(os.path.join(tokdir, fname)) // (1024 * 1024)
|
size_mb = os.path.getsize(os.path.join(tokdir, fname)) // (1024 * 1024)
|
||||||
|
|
||||||
up = subprocess.run(
|
up = subprocess.run(
|
||||||
["rsync", "-a", "--chmod=F644,D755", "-e", _RSYNC_RSH,
|
[*_NICE, "rsync", "-a", "--chmod=F644,D755", "-e", _RSYNC_RSH,
|
||||||
tokdir, f"{FORCE_REMOTE}:{FORCE_REMOTE_DIR}/"],
|
tokdir, f"{FORCE_REMOTE}:{FORCE_REMOTE_DIR}/"],
|
||||||
capture_output=True, text=True, timeout=RSYNC_TIMEOUT,
|
capture_output=True, text=True, timeout=RSYNC_TIMEOUT,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user