Drop a link: .url/.txt in _incoming/ is downloaded, verified as a PDF, stored as the book's source PDF; workflow also triggers on link files
This commit is contained in:
Audiobook Producer
@@ -0,0 +1,162 @@
|
||||
"""Drop a link instead of a file: _incoming/<anything>.url or .txt holding one http(s) address of a PDF.
|
||||
|
||||
* The file may be plain text (just the address) or a Windows internet shortcut ([InternetShortcut] / URL=...).
|
||||
* An arXiv page address (arxiv.org/abs/<id>) is rewritten to the PDF address (arxiv.org/pdf/<id>).
|
||||
* The download uses curl: follows redirects, connection and total time limits, a size cap (settings fetch.max_mb;
|
||||
enforced by curl and again by a hard file-size limit on the process and a check afterwards).
|
||||
* The result must really be a PDF: the bytes must start with %PDF- and pdfinfo must read it. Otherwise the run fails
|
||||
and says why (bad address, HTTP error, not a PDF, too large, timeout...). Nothing is committed on failure.
|
||||
* The file name comes from the server (Content-Disposition) or the last part of the address, with .pdf added if missing.
|
||||
The download stays in the scratch folder; the pipeline then stores it as the book's source PDF and records the address in
|
||||
book.json (source_url). Nothing is published anywhere else."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import shutil
|
||||
import subprocess
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
UA = "Mozilla/5.0 (compatible; audiobook-pipeline/1.0; +drop-folder)"
|
||||
|
||||
|
||||
class FetchError(Exception):
|
||||
"""The message is what the operator reads in the run log."""
|
||||
|
||||
|
||||
def read_url_file(path):
|
||||
"""The one http(s) address in a .url/.txt file."""
|
||||
with open(path, encoding="utf-8-sig", errors="replace") as f:
|
||||
lines = [ln.strip() for ln in f.read().splitlines()]
|
||||
found = []
|
||||
for ln in lines:
|
||||
if not ln or ln.startswith(("#", ";", "[")):
|
||||
continue
|
||||
if re.match(r"^url\s*=", ln, re.I):
|
||||
ln = ln.split("=", 1)[1].strip()
|
||||
elif "=" in ln.split()[0] and not ln.lower().startswith(("http://", "https://")):
|
||||
continue # other shortcut keys (IconIndex=0, ...)
|
||||
found.append(ln)
|
||||
if not found:
|
||||
raise FetchError("%s is empty: put one web address (https://...) in it" % os.path.basename(path))
|
||||
if len(set(found)) > 1:
|
||||
raise FetchError("%s has %d different lines; it must hold exactly one web address (https://...)" % (os.path.basename(path), len(set(found))))
|
||||
url = found[0]
|
||||
if not re.match(r"^https?://[^\s/]+", url, re.I) or re.search(r"\s", url):
|
||||
raise FetchError("%s: %r is not a web address. It must start with http:// or https:// and contain no spaces" % (os.path.basename(path), url[:120]))
|
||||
return url
|
||||
|
||||
|
||||
def normalize_url(url):
|
||||
"""arxiv.org/abs/<id> -> arxiv.org/pdf/<id>. Everything else is used as it is."""
|
||||
m = re.match(r"^https?://(?:www\.|export\.)?arxiv\.org/abs/(.+?)/?(?:[?#].*)?$", url, re.I)
|
||||
if m:
|
||||
return "https://arxiv.org/pdf/" + m.group(1)
|
||||
return url
|
||||
|
||||
|
||||
def safe_name(name):
|
||||
name = unquote(name).strip().strip(".")
|
||||
name = re.sub(r"[^A-Za-z0-9._() \-]+", "_", os.path.basename(name)).strip(" _")
|
||||
if not name:
|
||||
name = "download"
|
||||
if not name.lower().endswith(".pdf"):
|
||||
name += ".pdf"
|
||||
return name
|
||||
|
||||
|
||||
def name_from_response(headers_text, final_url):
|
||||
"""File name from the last Content-Disposition header, else the last part of the final address."""
|
||||
found = re.findall(r"^content-disposition:.*$", headers_text, re.I | re.M)
|
||||
if found:
|
||||
m = re.search(r"filename\*\s*=\s*[^']*'[^']*'([^;\r\n]+)", found[-1], re.I) or re.search(r'filename\s*=\s*"?([^";\r\n]+)"?', found[-1], re.I)
|
||||
if m:
|
||||
return safe_name(m.group(1))
|
||||
path = urlparse(final_url).path.rstrip("/")
|
||||
return safe_name(path.rsplit("/", 1)[-1] if path else "")
|
||||
|
||||
|
||||
def check_pdf(path):
|
||||
with open(path, "rb") as f:
|
||||
head = f.read(2048)
|
||||
if b"%PDF-" not in head[:1024]:
|
||||
kind = "an HTML page" if re.search(rb"<\s*(!doctype|html|head|body)", head[:512], re.I) else "not a PDF file"
|
||||
title = re.search(rb"<title[^>]*>(.*?)</title>", head, re.I | re.S)
|
||||
raise FetchError("the link did not return a PDF (the download is %s%s). Use the direct link to the PDF file" %
|
||||
(kind, ": %r" % title.group(1).decode("utf-8", "replace").strip()[:80] if title else ""))
|
||||
r = subprocess.run(["pdfinfo", path], capture_output=True, text=True)
|
||||
pages = re.search(r"^Pages:\s+(\d+)", r.stdout, re.M)
|
||||
if r.returncode != 0 or not pages or int(pages.group(1)) < 1:
|
||||
raise FetchError("the download starts like a PDF but pdfinfo cannot read it (damaged or cut short): %s" % (r.stderr.strip()[:200] or "no pages"))
|
||||
return int(pages.group(1))
|
||||
|
||||
|
||||
CURL_ERRORS = {
|
||||
3: "the web address is malformed", 5: "cannot resolve the proxy", 6: "the host name does not exist (check the address)",
|
||||
7: "the server refused the connection", 28: "the download timed out", 35: "TLS handshake failed", 47: "too many redirects",
|
||||
51: "the server certificate does not match", 52: "the server sent an empty reply", 56: "the connection broke during the download",
|
||||
60: "the server certificate is not trusted", 63: "the file is larger than the allowed maximum",
|
||||
}
|
||||
|
||||
|
||||
def download(cfg, url, out_dir):
|
||||
"""Download url into out_dir. Returns {'pdf', 'bytes', 'pages', 'final_url'}. Raises FetchError."""
|
||||
fc = cfg.get("fetch", {})
|
||||
max_bytes = int(float(fc.get("max_mb", 300)) * 1024 * 1024)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
part = os.path.join(out_dir, "download.partial")
|
||||
hdr = os.path.join(out_dir, "headers.txt")
|
||||
for p in (part, hdr):
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
cmd = ["curl", "-sS", "-L", "--fail", "--max-redirs", "10", "--connect-timeout", str(fc.get("connect_timeout_s", 30)),
|
||||
"--max-time", str(fc.get("max_time_s", 900)), "--max-filesize", str(max_bytes), "-A", UA, "-D", hdr, "-o", part,
|
||||
"-w", "%{url_effective}", "--proto", "=http,https", "--proto-redir", "=http,https", url]
|
||||
limit = lambda: resource.setrlimit(resource.RLIMIT_FSIZE, (max_bytes + (1 << 20), max_bytes + (1 << 20)))
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit)
|
||||
except FileNotFoundError:
|
||||
raise FetchError("curl is not installed on this host")
|
||||
if r.returncode != 0:
|
||||
if os.path.exists(part):
|
||||
os.remove(part)
|
||||
if r.returncode in (-25, 25) or r.returncode == 63:
|
||||
raise FetchError("the file is larger than the allowed maximum of %s MB (settings fetch.max_mb)" % fc.get("max_mb", 300))
|
||||
detail = r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "curl exit %d" % r.returncode
|
||||
m = re.search(r"returned error: (\d+)", detail)
|
||||
why = ("the server answered HTTP %s (%s)" % (m.group(1), {"404": "not found", "403": "forbidden", "401": "login required", "410": "gone"}.get(m.group(1), "error"))
|
||||
if m else CURL_ERRORS.get(r.returncode, detail))
|
||||
raise FetchError("could not download %s: %s" % (url, why))
|
||||
size = os.path.getsize(part)
|
||||
if size == 0:
|
||||
raise FetchError("the download from %s was empty" % url)
|
||||
if size > max_bytes:
|
||||
os.remove(part)
|
||||
raise FetchError("the file is %.0f MB, larger than the allowed maximum of %s MB (settings fetch.max_mb)" % (size / 1048576, fc.get("max_mb", 300)))
|
||||
final_url = r.stdout.strip() or url
|
||||
headers = ""
|
||||
if os.path.exists(hdr):
|
||||
with open(hdr, encoding="utf-8", errors="replace") as f:
|
||||
headers = f.read()
|
||||
try:
|
||||
pages = check_pdf(part)
|
||||
except FetchError:
|
||||
os.remove(part)
|
||||
raise
|
||||
pdf = os.path.join(out_dir, name_from_response(headers, final_url))
|
||||
shutil.move(part, pdf)
|
||||
return {"pdf": pdf, "bytes": size, "pages": pages, "final_url": final_url}
|
||||
|
||||
|
||||
def fetch_from_file(cfg, urlfile, out_dir):
|
||||
"""Everything for one .url/.txt file. Returns a JSON-able dict: pdf, source_url, original_url, name_hints, bytes, pages."""
|
||||
original = read_url_file(urlfile)
|
||||
url = normalize_url(original)
|
||||
print("link file %s -> %s%s" % (os.path.basename(urlfile), url, " (rewritten from %s)" % original if url != original else ""))
|
||||
got = download(cfg, url, out_dir)
|
||||
side = os.path.splitext(urlfile)[0] + ".json"
|
||||
if os.path.exists(side):
|
||||
shutil.copyfile(side, os.path.splitext(got["pdf"])[0] + ".json")
|
||||
print("downloaded %s: %.1f MB, %d pages (valid PDF)" % (os.path.basename(got["pdf"]), got["bytes"] / 1048576, got["pages"]))
|
||||
return {"pdf": got["pdf"], "source_url": url,
|
||||
"original_url": original, "name_hints": [os.path.basename(urlfile)], "bytes": got["bytes"], "pages": got["pages"]}
|
||||
Reference in New Issue
Block a user