diff --git a/.gitea/workflows/audiobook.yml b/.gitea/workflows/audiobook.yml index 2c4eca5..0b953fa 100644 --- a/.gitea/workflows/audiobook.yml +++ b/.gitea/workflows/audiobook.yml @@ -1,15 +1,15 @@ -# Drop a PDF in _incoming/ and commit it. This job turns it into a finished audiobook folder and commits that back. +# Drop a PDF (or a .url/.txt file with a link to a PDF) in _incoming/ and commit it. This job turns it into a finished audiobook folder and commits that back. # No bot is involved: every step is a deterministic script in _pipeline/. name: audiobook on: push: branches: [main, 'test/**'] # test/** lets the pipeline be proven on a branch without touching main - paths: ['_incoming/*.pdf'] # only a PDF landing in _incoming/ starts a run + paths: ['_incoming/*.pdf', '_incoming/*.PDF', '_incoming/*.url', '_incoming/*.URL', '_incoming/*.txt'] # a PDF or a link file landing in _incoming/ starts a run workflow_dispatch: inputs: pdf: - description: 'PDF path inside the repo (empty = every PDF waiting in _incoming/; a dry run with nothing waiting uses the Clark book as a regression sample)' + description: 'PDF or .url/.txt link-file path inside the repo (empty = everything waiting in _incoming/; a dry run with nothing waiting uses the Clark book as a regression sample)' required: false default: '' voice_endpoint: diff --git a/_pipeline/ci/install.sh b/_pipeline/ci/install.sh index 66362e6..3cedeb2 100755 --- a/_pipeline/ci/install.sh +++ b/_pipeline/ci/install.sh @@ -3,7 +3,7 @@ set -euo pipefail export DEBIAN_FRONTEND=noninteractive apt-get update -qq -apt-get install -y -qq --no-install-recommends ffmpeg poppler-utils tesseract-ocr tesseract-ocr-eng ocrmypdf ghostscript fonts-urw-base35 \ +apt-get install -y -qq --no-install-recommends ffmpeg poppler-utils tesseract-ocr tesseract-ocr-eng ocrmypdf qpdf ghostscript fonts-urw-base35 \ python3 python3-venv python3-pip git curl ca-certificates python3 -m venv .venv .venv/bin/pip install -q --disable-pip-version-check -r _pipeline/requirements.txt diff --git a/_pipeline/ci/run.sh b/_pipeline/ci/run.sh index 24aa434..57b9b8e 100755 --- a/_pipeline/ci/run.sh +++ b/_pipeline/ci/run.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # The whole job, in one place. Inputs (environment): # DRY_RUN true|false (true: text stages only, voice reachability, push-permission test, commits nothing to the branch) -# PDF_INPUT optional path relative to the repo root (manual runs). Empty = every PDF waiting in _incoming/ +# PDF_INPUT optional path relative to the repo root (manual runs): a PDF or a .url/.txt link file. +# Empty = every PDF, .url and .txt file waiting in _incoming/ # BRANCH branch to commit back to # AUDIOBOOK_* setting overrides (see _pipeline/settings.py) set -uo pipefail @@ -22,7 +23,7 @@ if [ -n "${PDF_INPUT:-}" ]; then [ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; } pdfs=("$PDF_INPUT") else - while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f -iname '*.pdf' | sort) + while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f \( -iname '*.pdf' -o -iname '*.url' -o -iname '*.txt' \) | sort) fi CLARK_PDF="books/clark-glenn/the-man-who-tapped-the-secrets-of-the-universe/The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf" if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then @@ -31,10 +32,10 @@ if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then pdfs=("$CLARK_PDF") fi if [ "${#pdfs[@]}" -eq 0 ]; then - echo "Nothing to do: no PDF in _incoming/." + echo "Nothing to do: no PDF or link file (.url/.txt) in _incoming/." exit 0 fi -echo "PDFs to process: ${#pdfs[@]}"; printf ' %s\n' "${pdfs[@]}" +echo "Items to process: ${#pdfs[@]}"; printf ' %s\n' "${pdfs[@]}" push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and try again local i @@ -47,10 +48,25 @@ push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and } failed=0 -for pdf in "${pdfs[@]}"; do - echo; echo "################ $pdf" +n=0 +for item in "${pdfs[@]}"; do + n=$((n+1)) + pdf="$item"; extra=(); hints=() + echo; echo "################ $item" result="$(mktemp)" - rel="$(python3 _pipeline/cli.py bookpath "$pdf")" || { echo "FAILED: $rel"; failed=1; continue; } + case "${item,,}" in + *.url|*.txt) # a link to a PDF: download it into the scratch folder, check it is a real PDF, then treat it like a dropped PDF + fdir="/tmp/audiobook-fetch/$n"; rm -rf "$fdir"; mkdir -p "$fdir" + if ! python3 _pipeline/cli.py fetch "$item" --out "$fdir"; then + echo "FAILED: could not get the PDF for $item (reason above). Nothing was committed. The link file stays where it is." + failed=1; continue + fi + pdf="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['pdf'])" "$fdir/fetch.json")" + surl="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['source_url'])" "$fdir/fetch.json")" + hints=(--name-hint "$(basename "$item")"); extra=(--source-url "$surl" "${hints[@]}") + ;; + esac + rel="$(python3 _pipeline/cli.py bookpath "$pdf" "${hints[@]}")" || { echo "FAILED: $rel"; failed=1; continue; } echo "book folder: $rel" e="$(esc "$rel")" if git cat-file -e "HEAD:$rel" 2>/dev/null; then @@ -71,10 +87,10 @@ for pdf in "${pdfs[@]}"; do unset AUDIOBOOK_BOOKS_DIR args=() fi - python3 _pipeline/cli.py process "$pdf" "${args[@]}" --result-file "$result" + python3 _pipeline/cli.py process "$pdf" "${args[@]}" "${extra[@]}" --result-file "$result" rc=$? if [ $rc -ne 0 ]; then - echo "FAILED: process exited $rc for $pdf. Nothing was committed. The PDF stays where it is." + echo "FAILED: process exited $rc for $item. Nothing was committed. The file stays where it is in _incoming/." git checkout -q -- . 2>/dev/null; git clean -qfd -- books 2>/dev/null failed=1; continue fi @@ -87,14 +103,14 @@ for pdf in "${pdfs[@]}"; do if ! git ls-files --error-unmatch -- "$rel/book.json" >/dev/null 2>&1; then echo "FAILED: $rel/book.json is not tracked after git add. Nothing was committed."; failed=1; continue fi - case "$pdf" in - _incoming/*) - git rm -q -- "$pdf" - side="${pdf%.[pP][dD][fF]}.json"; [ -f "$side" ] && git rm -q -- "$side" + case "$item" in + _incoming/*) # the dropped PDF or link file (and its optional .json) leaves _incoming/ in the same commit + git rm -q -- "$item" + side="${item%.*}.json"; [ -f "$side" ] && git rm -q -- "$side" ;; esac - if git diff --cached --quiet; then echo "No changes to commit for $pdf"; continue; fi - git commit -q -m "Audiobook: $rel (from $pdf, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]" + if git diff --cached --quiet; then echo "No changes to commit for $item"; continue; fi + git commit -q -m "Audiobook: $rel (from $item, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]" git --no-pager show --stat --format='commit %H' HEAD | head -40 if push_with_retry; then echo "PUSHED to $BRANCH: $rel"; else echo "FAILED: could not push to $BRANCH"; failed=1; fi done diff --git a/_pipeline/cli.py b/_pipeline/cli.py index 6299da7..b4d6be4 100644 --- a/_pipeline/cli.py +++ b/_pipeline/cli.py @@ -9,6 +9,7 @@ python3 _pipeline/cli.py check ... ACX checker, pass/fail table """ import argparse +import json import os import sys @@ -71,8 +72,14 @@ def main(argv=None): p.add_argument("--compare", help="a books folder (like the repo books/); the text-stage output is compared with the same / folder in it") p.add_argument("--compare-strict", action="store_true", help="with --compare: fail when the output differs (default: report only)") p.add_argument("--result-file", help="write {ok, book_dir, pdf} JSON here") + p.add_argument("--source-url", help="where the PDF was downloaded from (recorded in book.json)") + p.add_argument("--name-hint", action="append", default=[], help="another file name to read a title from (repeatable)") p = sub.add_parser("bookpath", help="print the book folder (relative to the repo root) this PDF will go to") p.add_argument("pdf") + p.add_argument("--name-hint", action="append", default=[]) + p = sub.add_parser("fetch", help="download the PDF named by a .url/.txt file into a scratch folder; writes /fetch.json") + p.add_argument("urlfile") + p.add_argument("--out", required=True) p = sub.add_parser("ingest") p.add_argument("pdf") p.add_argument("--title") @@ -97,14 +104,24 @@ def main(argv=None): if a.cmd == "bookpath": import process try: - title, author, _ = process.resolve_meta(os.path.abspath(a.pdf)) + m = process.resolve_meta(cfg, os.path.abspath(a.pdf), name_hints=a.name_hint) except process.Failed as e: sys.exit(str(e)) - print(os.path.relpath(ingest.book_folder(cfg, author, title), settings.REPO_ROOT)) + print("\n".join(m["log"]), file=sys.stderr) + print(os.path.relpath(ingest.book_folder(cfg, m["author"], m["title"]), settings.REPO_ROOT)) + return + if a.cmd == "fetch": + import fetch + try: + res = fetch.fetch_from_file(cfg, a.urlfile, a.out) + with open(os.path.join(a.out, "fetch.json"), "w", encoding="utf-8") as f: + json.dump(res, f) + except fetch.FetchError as e: + sys.exit("FETCH FAILED: %s" % e) return if a.cmd == "process": import process - sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict)) + sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict, a.source_url, a.name_hint)) if a.cmd in ("run", "ingest"): book_dir = ingest.run(cfg, os.path.abspath(a.pdf), a.title, a.author) print("ingest done ->", book_dir) diff --git a/_pipeline/fetch.py b/_pipeline/fetch.py new file mode 100644 index 0000000..a9a290b --- /dev/null +++ b/_pipeline/fetch.py @@ -0,0 +1,162 @@ +"""Drop a link instead of a file: _incoming/.url or .txt holding one http(s) address of a PDF. + + * The file may be plain text (just the address) or a Windows internet shortcut ([InternetShortcut] / URL=...). + * An arXiv page address (arxiv.org/abs/) is rewritten to the PDF address (arxiv.org/pdf/). + * The download uses curl: follows redirects, connection and total time limits, a size cap (settings fetch.max_mb; + enforced by curl and again by a hard file-size limit on the process and a check afterwards). + * The result must really be a PDF: the bytes must start with %PDF- and pdfinfo must read it. Otherwise the run fails + and says why (bad address, HTTP error, not a PDF, too large, timeout...). Nothing is committed on failure. + * The file name comes from the server (Content-Disposition) or the last part of the address, with .pdf added if missing. +The download stays in the scratch folder; the pipeline then stores it as the book's source PDF and records the address in +book.json (source_url). Nothing is published anywhere else.""" +import json +import os +import re +import resource +import shutil +import subprocess +from urllib.parse import unquote, urlparse + +UA = "Mozilla/5.0 (compatible; audiobook-pipeline/1.0; +drop-folder)" + + +class FetchError(Exception): + """The message is what the operator reads in the run log.""" + + +def read_url_file(path): + """The one http(s) address in a .url/.txt file.""" + with open(path, encoding="utf-8-sig", errors="replace") as f: + lines = [ln.strip() for ln in f.read().splitlines()] + found = [] + for ln in lines: + if not ln or ln.startswith(("#", ";", "[")): + continue + if re.match(r"^url\s*=", ln, re.I): + ln = ln.split("=", 1)[1].strip() + elif "=" in ln.split()[0] and not ln.lower().startswith(("http://", "https://")): + continue # other shortcut keys (IconIndex=0, ...) + found.append(ln) + if not found: + raise FetchError("%s is empty: put one web address (https://...) in it" % os.path.basename(path)) + if len(set(found)) > 1: + raise FetchError("%s has %d different lines; it must hold exactly one web address (https://...)" % (os.path.basename(path), len(set(found)))) + url = found[0] + if not re.match(r"^https?://[^\s/]+", url, re.I) or re.search(r"\s", url): + raise FetchError("%s: %r is not a web address. It must start with http:// or https:// and contain no spaces" % (os.path.basename(path), url[:120])) + return url + + +def normalize_url(url): + """arxiv.org/abs/ -> arxiv.org/pdf/. Everything else is used as it is.""" + m = re.match(r"^https?://(?:www\.|export\.)?arxiv\.org/abs/(.+?)/?(?:[?#].*)?$", url, re.I) + if m: + return "https://arxiv.org/pdf/" + m.group(1) + return url + + +def safe_name(name): + name = unquote(name).strip().strip(".") + name = re.sub(r"[^A-Za-z0-9._() \-]+", "_", os.path.basename(name)).strip(" _") + if not name: + name = "download" + if not name.lower().endswith(".pdf"): + name += ".pdf" + return name + + +def name_from_response(headers_text, final_url): + """File name from the last Content-Disposition header, else the last part of the final address.""" + found = re.findall(r"^content-disposition:.*$", headers_text, re.I | re.M) + if found: + m = re.search(r"filename\*\s*=\s*[^']*'[^']*'([^;\r\n]+)", found[-1], re.I) or re.search(r'filename\s*=\s*"?([^";\r\n]+)"?', found[-1], re.I) + if m: + return safe_name(m.group(1)) + path = urlparse(final_url).path.rstrip("/") + return safe_name(path.rsplit("/", 1)[-1] if path else "") + + +def check_pdf(path): + with open(path, "rb") as f: + head = f.read(2048) + if b"%PDF-" not in head[:1024]: + kind = "an HTML page" if re.search(rb"<\s*(!doctype|html|head|body)", head[:512], re.I) else "not a PDF file" + title = re.search(rb"]*>(.*?)", head, re.I | re.S) + raise FetchError("the link did not return a PDF (the download is %s%s). Use the direct link to the PDF file" % + (kind, ": %r" % title.group(1).decode("utf-8", "replace").strip()[:80] if title else "")) + r = subprocess.run(["pdfinfo", path], capture_output=True, text=True) + pages = re.search(r"^Pages:\s+(\d+)", r.stdout, re.M) + if r.returncode != 0 or not pages or int(pages.group(1)) < 1: + raise FetchError("the download starts like a PDF but pdfinfo cannot read it (damaged or cut short): %s" % (r.stderr.strip()[:200] or "no pages")) + return int(pages.group(1)) + + +CURL_ERRORS = { + 3: "the web address is malformed", 5: "cannot resolve the proxy", 6: "the host name does not exist (check the address)", + 7: "the server refused the connection", 28: "the download timed out", 35: "TLS handshake failed", 47: "too many redirects", + 51: "the server certificate does not match", 52: "the server sent an empty reply", 56: "the connection broke during the download", + 60: "the server certificate is not trusted", 63: "the file is larger than the allowed maximum", +} + + +def download(cfg, url, out_dir): + """Download url into out_dir. Returns {'pdf', 'bytes', 'pages', 'final_url'}. Raises FetchError.""" + fc = cfg.get("fetch", {}) + max_bytes = int(float(fc.get("max_mb", 300)) * 1024 * 1024) + os.makedirs(out_dir, exist_ok=True) + part = os.path.join(out_dir, "download.partial") + hdr = os.path.join(out_dir, "headers.txt") + for p in (part, hdr): + if os.path.exists(p): + os.remove(p) + cmd = ["curl", "-sS", "-L", "--fail", "--max-redirs", "10", "--connect-timeout", str(fc.get("connect_timeout_s", 30)), + "--max-time", str(fc.get("max_time_s", 900)), "--max-filesize", str(max_bytes), "-A", UA, "-D", hdr, "-o", part, + "-w", "%{url_effective}", "--proto", "=http,https", "--proto-redir", "=http,https", url] + limit = lambda: resource.setrlimit(resource.RLIMIT_FSIZE, (max_bytes + (1 << 20), max_bytes + (1 << 20))) + try: + r = subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit) + except FileNotFoundError: + raise FetchError("curl is not installed on this host") + if r.returncode != 0: + if os.path.exists(part): + os.remove(part) + if r.returncode in (-25, 25) or r.returncode == 63: + raise FetchError("the file is larger than the allowed maximum of %s MB (settings fetch.max_mb)" % fc.get("max_mb", 300)) + detail = r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "curl exit %d" % r.returncode + m = re.search(r"returned error: (\d+)", detail) + why = ("the server answered HTTP %s (%s)" % (m.group(1), {"404": "not found", "403": "forbidden", "401": "login required", "410": "gone"}.get(m.group(1), "error")) + if m else CURL_ERRORS.get(r.returncode, detail)) + raise FetchError("could not download %s: %s" % (url, why)) + size = os.path.getsize(part) + if size == 0: + raise FetchError("the download from %s was empty" % url) + if size > max_bytes: + os.remove(part) + raise FetchError("the file is %.0f MB, larger than the allowed maximum of %s MB (settings fetch.max_mb)" % (size / 1048576, fc.get("max_mb", 300))) + final_url = r.stdout.strip() or url + headers = "" + if os.path.exists(hdr): + with open(hdr, encoding="utf-8", errors="replace") as f: + headers = f.read() + try: + pages = check_pdf(part) + except FetchError: + os.remove(part) + raise + pdf = os.path.join(out_dir, name_from_response(headers, final_url)) + shutil.move(part, pdf) + return {"pdf": pdf, "bytes": size, "pages": pages, "final_url": final_url} + + +def fetch_from_file(cfg, urlfile, out_dir): + """Everything for one .url/.txt file. Returns a JSON-able dict: pdf, source_url, original_url, name_hints, bytes, pages.""" + original = read_url_file(urlfile) + url = normalize_url(original) + print("link file %s -> %s%s" % (os.path.basename(urlfile), url, " (rewritten from %s)" % original if url != original else "")) + got = download(cfg, url, out_dir) + side = os.path.splitext(urlfile)[0] + ".json" + if os.path.exists(side): + shutil.copyfile(side, os.path.splitext(got["pdf"])[0] + ".json") + print("downloaded %s: %.1f MB, %d pages (valid PDF)" % (os.path.basename(got["pdf"]), got["bytes"] / 1048576, got["pages"])) + return {"pdf": got["pdf"], "source_url": url, + "original_url": original, "name_hints": [os.path.basename(urlfile)], "bytes": got["bytes"], "pages": got["pages"]} diff --git a/_pipeline/tests/test_fetch.py b/_pipeline/tests/test_fetch.py new file mode 100644 index 0000000..f4431bc --- /dev/null +++ b/_pipeline/tests/test_fetch.py @@ -0,0 +1,189 @@ +"""Link files in _incoming/: .url / .txt reading, arXiv rewrite, and the download against a local test server.""" +import http.server +import json +import os +import sys +import tempfile +import threading +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +sys.path.insert(0, HERE) + +import bookio +import fetch +import ingest +import settings + +try: + from helpers import make_pdf +except ImportError: + make_pdf = None + +PDF_BYTES = b"" + + +class Handler(http.server.BaseHTTPRequestHandler): + hits = [] + + def log_message(self, *a): + pass + + def send(self, code, body=b"", ctype="application/pdf", extra=None): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + for k, v in (extra or {}).items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + Handler.hits.append(self.path) + if self.path == "/files/Deep-Work.pdf": + self.send(200, PDF_BYTES) + elif self.path == "/redirect": + self.send(302, extra={"Location": "/files/Deep-Work.pdf"}) + elif self.path == "/pdf/2401.01234": # no extension, like arxiv.org/pdf/ + self.send(200, PDF_BYTES) + elif self.path == "/dl?id=7": + self.send(200, PDF_BYTES, extra={"Content-Disposition": 'attachment; filename="Named By Server.pdf"'}) + elif self.path == "/page": + self.send(200, b"Sign inlogin", "text/html") + elif self.path == "/cut": + self.send(200, PDF_BYTES[:200]) + elif self.path == "/empty": + self.send(200, b"") + else: + self.send(404, b"not found", "text/plain") + + +@unittest.skipIf(make_pdf is None, "reportlab not installed") +class FetchTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + global PDF_BYTES + d = tempfile.mkdtemp() + make_pdf(os.path.join(d, "a.pdf"), [["Deep Work", "By Cal Newport"], ["text"]], title="Deep Work") + PDF_BYTES = open(os.path.join(d, "a.pdf"), "rb").read() + cls.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=cls.server.serve_forever, daemon=True).start() + cls.base = "http://127.0.0.1:%d" % cls.server.server_port + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.d = self.tmp.name + self.cfg = settings.load() + self.out = os.path.join(self.d, "out") + + def tearDown(self): + self.tmp.cleanup() + + def urlfile(self, content, name="Some Link.url"): + p = os.path.join(self.d, name) + with open(p, "w", encoding="utf-8") as f: + f.write(content) + return p + + def test_download_follows_redirects_and_verifies_pdf(self): + r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/redirect\n"), self.out) + self.assertEqual(os.path.basename(r["pdf"]), "Deep-Work.pdf") + self.assertEqual(r["pages"], 2) + self.assertEqual(open(r["pdf"], "rb").read(), PDF_BYTES) + self.assertEqual(r["source_url"], self.base + "/redirect") + self.assertEqual(r["name_hints"], ["Some Link.url"]) + + def test_windows_shortcut_format_and_txt(self): + r = fetch.fetch_from_file(self.cfg, self.urlfile("[InternetShortcut]\r\nURL=%s/files/Deep-Work.pdf\r\nIconIndex=0\r\n" % self.base), self.out) + self.assertTrue(r["pdf"].endswith("Deep-Work.pdf")) + r = fetch.fetch_from_file(self.cfg, self.urlfile("# my link\n%s/files/Deep-Work.pdf\n" % self.base, "note.txt"), os.path.join(self.d, "o2")) + self.assertTrue(os.path.exists(r["pdf"])) + + def test_name_from_url_without_extension_and_from_content_disposition(self): + r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/pdf/2401.01234"), self.out) + self.assertEqual(os.path.basename(r["pdf"]), "2401.01234.pdf") + r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/dl?id=7"), os.path.join(self.d, "o2")) + self.assertEqual(os.path.basename(r["pdf"]), "Named By Server.pdf") + + def test_sidecar_json_next_to_the_url_file_is_carried_over(self): + self.urlfile('{"title": "T", "author": "A"}', "Some Link.json") + r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/files/Deep-Work.pdf"), self.out) + self.assertEqual(json.load(open(os.path.splitext(r["pdf"])[0] + ".json"))["author"], "A") + + def test_html_page_is_refused_with_a_clear_reason(self): + with self.assertRaises(fetch.FetchError) as cm: + fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/page"), self.out) + self.assertIn("did not return a PDF", str(cm.exception)) + self.assertIn("HTML page", str(cm.exception)) + self.assertEqual(os.listdir(self.out) and [n for n in os.listdir(self.out) if n.endswith(".pdf")], []) + + def test_http_404_cut_off_pdf_and_empty_download(self): + for path, text in (("/nothing", "HTTP 404 (not found)"), ("/cut", "cannot read it"), ("/empty", "empty")): + with self.assertRaises(fetch.FetchError) as cm: + fetch.fetch_from_file(self.cfg, self.urlfile(self.base + path), self.out) + self.assertIn(text, str(cm.exception), path) + + def test_too_large_is_refused(self): + self.cfg["fetch"]["max_mb"] = 0.0005 # about 500 bytes; the test PDF is a few KB + with self.assertRaises(fetch.FetchError) as cm: + fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/files/Deep-Work.pdf"), self.out) + self.assertIn("larger than the allowed maximum", str(cm.exception)) + + def test_unreachable_host(self): + with self.assertRaises(fetch.FetchError) as cm: + fetch.fetch_from_file(self.cfg, self.urlfile("http://127.0.0.1:9/x.pdf"), self.out) + self.assertIn("refused the connection", str(cm.exception)) + + def test_ingest_stores_the_downloaded_pdf_and_records_the_source_url(self): + r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/files/Deep-Work.pdf"), self.out) + self.cfg["books_dir"] = os.path.join(self.d, "books") + self.cfg["work_dir"] = os.path.join(self.d, "work") + book_dir = ingest.run(self.cfg, r["pdf"], source_url=r["source_url"]) + book = bookio.read_book(book_dir) + self.assertEqual(book["source_url"], self.base + "/files/Deep-Work.pdf") + self.assertEqual((book["author"], book["title"]), ("Cal Newport", "Deep Work")) + self.assertTrue(os.path.exists(os.path.join(book_dir, "Deep-Work.pdf"))) + + +class PureTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + + def tearDown(self): + self.tmp.cleanup() + + def read(self, content): + p = os.path.join(self.tmp.name, "x.url") + with open(p, "w", encoding="utf-8-sig") as f: + f.write(content) + return fetch.read_url_file(p) + + def test_arxiv_abs_is_rewritten(self): + self.assertEqual(fetch.normalize_url("https://arxiv.org/abs/2401.01234"), "https://arxiv.org/pdf/2401.01234") + self.assertEqual(fetch.normalize_url("http://www.arxiv.org/abs/math/0301001v2"), "https://arxiv.org/pdf/math/0301001v2") + self.assertEqual(fetch.normalize_url("https://arxiv.org/pdf/2401.01234"), "https://arxiv.org/pdf/2401.01234") + self.assertEqual(fetch.normalize_url("https://example.org/a.pdf"), "https://example.org/a.pdf") + + def test_reading_link_files(self): + self.assertEqual(self.read("https://example.org/a.pdf \n\n"), "https://example.org/a.pdf") + self.assertEqual(self.read("[InternetShortcut]\nURL=https://example.org/b.pdf\nIDList=\n"), "https://example.org/b.pdf") + for bad, why in (("", "empty"), ("ftp://x/y.pdf", "not a web address"), ("see https://a.org/x.pdf", "not a web address"), + ("https://a.org/1.pdf\nhttps://a.org/2.pdf", "exactly one")): + with self.assertRaises(fetch.FetchError) as cm: + self.read(bad) + self.assertIn(why, str(cm.exception), bad) + + def test_safe_names(self): + self.assertEqual(fetch.safe_name("a%20b.pdf"), "a b.pdf") + self.assertEqual(fetch.safe_name(""), "download.pdf") + self.assertEqual(fetch.safe_name("2401.01234"), "2401.01234.pdf") + self.assertEqual(fetch.name_from_response("HTTP/1.1 200 OK\r\n", "https://x.org/files/My%20Book.pdf?x=1"), "My Book.pdf") + + +if __name__ == "__main__": + unittest.main()