Drop a link: .url/.txt in _incoming/ is downloaded, verified as a PDF, stored as the book's source PDF; workflow also triggers on link files

.gitea/workflows/audiobook.yml

@@ -1,15 +1,15 @@
-# Drop a PDF in _incoming/ and commit it. This job turns it into a finished audiobook folder and commits that back.
+# Drop a PDF (or a .url/.txt file with a link to a PDF) in _incoming/ and commit it. This job turns it into a finished audiobook folder and commits that back.
 # No bot is involved: every step is a deterministic script in _pipeline/.
 name: audiobook
 
 on:
   push:
     branches: [main, 'test/**']       # test/** lets the pipeline be proven on a branch without touching main
-    paths: ['_incoming/*.pdf']        # only a PDF landing in _incoming/ starts a run
+    paths: ['_incoming/*.pdf', '_incoming/*.PDF', '_incoming/*.url', '_incoming/*.URL', '_incoming/*.txt']   # a PDF or a link file landing in _incoming/ starts a run
   workflow_dispatch:
     inputs:
       pdf:
-        description: 'PDF path inside the repo (empty = every PDF waiting in _incoming/; a dry run with nothing waiting uses the Clark book as a regression sample)'
+        description: 'PDF or .url/.txt link-file path inside the repo (empty = everything waiting in _incoming/; a dry run with nothing waiting uses the Clark book as a regression sample)'
         required: false
         default: ''
       voice_endpoint:

_pipeline/ci/install.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
-apt-get install -y -qq --no-install-recommends ffmpeg poppler-utils tesseract-ocr tesseract-ocr-eng ocrmypdf ghostscript fonts-urw-base35 \
+apt-get install -y -qq --no-install-recommends ffmpeg poppler-utils tesseract-ocr tesseract-ocr-eng ocrmypdf qpdf ghostscript fonts-urw-base35 \
   python3 python3-venv python3-pip git curl ca-certificates
 python3 -m venv .venv
 .venv/bin/pip install -q --disable-pip-version-check -r _pipeline/requirements.txt

_pipeline/ci/run.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 # The whole job, in one place. Inputs (environment):
 #   DRY_RUN      true|false   (true: text stages only, voice reachability, push-permission test, commits nothing to the branch)
-#   PDF_INPUT    optional path relative to the repo root (manual runs). Empty = every PDF waiting in _incoming/
+#   PDF_INPUT    optional path relative to the repo root (manual runs): a PDF or a .url/.txt link file.
+#                Empty = every PDF, .url and .txt file waiting in _incoming/
 #   BRANCH       branch to commit back to
 #   AUDIOBOOK_*  setting overrides (see _pipeline/settings.py)
 set -uo pipefail
@@ -22,7 +23,7 @@ if [ -n "${PDF_INPUT:-}" ]; then
   [ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; }
   pdfs=("$PDF_INPUT")
 else
-  while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f -iname '*.pdf' | sort)
+  while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f \( -iname '*.pdf' -o -iname '*.url' -o -iname '*.txt' \) | sort)
 fi
 CLARK_PDF="books/clark-glenn/the-man-who-tapped-the-secrets-of-the-universe/The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf"
 if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then
@@ -31,10 +32,10 @@ if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then
   pdfs=("$CLARK_PDF")
 fi
 if [ "${#pdfs[@]}" -eq 0 ]; then
-  echo "Nothing to do: no PDF in _incoming/."
+  echo "Nothing to do: no PDF or link file (.url/.txt) in _incoming/."
   exit 0
 fi
-echo "PDFs to process: ${#pdfs[@]}"; printf '  %s\n' "${pdfs[@]}"
+echo "Items to process: ${#pdfs[@]}"; printf '  %s\n' "${pdfs[@]}"
 
 push_with_retry() {   # push HEAD to the branch; if the branch moved, rebase and try again
   local i
@@ -47,10 +48,25 @@ push_with_retry() {   # push HEAD to the branch; if the branch moved, rebase and
 }
 
 failed=0
-for pdf in "${pdfs[@]}"; do
+n=0
-  echo; echo "################ $pdf"
+for item in "${pdfs[@]}"; do
+  n=$((n+1))
+  pdf="$item"; extra=(); hints=()
+  echo; echo "################ $item"
   result="$(mktemp)"
-  rel="$(python3 _pipeline/cli.py bookpath "$pdf")" || { echo "FAILED: $rel"; failed=1; continue; }
+  case "${item,,}" in
+    *.url|*.txt)       # a link to a PDF: download it into the scratch folder, check it is a real PDF, then treat it like a dropped PDF
+      fdir="/tmp/audiobook-fetch/$n"; rm -rf "$fdir"; mkdir -p "$fdir"
+      if ! python3 _pipeline/cli.py fetch "$item" --out "$fdir"; then
+        echo "FAILED: could not get the PDF for $item (reason above). Nothing was committed. The link file stays where it is."
+        failed=1; continue
+      fi
+      pdf="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['pdf'])" "$fdir/fetch.json")"
+      surl="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['source_url'])" "$fdir/fetch.json")"
+      hints=(--name-hint "$(basename "$item")"); extra=(--source-url "$surl" "${hints[@]}")
+      ;;
+  esac
+  rel="$(python3 _pipeline/cli.py bookpath "$pdf" "${hints[@]}")" || { echo "FAILED: $rel"; failed=1; continue; }
   echo "book folder: $rel"
   e="$(esc "$rel")"
   if git cat-file -e "HEAD:$rel" 2>/dev/null; then
@@ -71,10 +87,10 @@ for pdf in "${pdfs[@]}"; do
     unset AUDIOBOOK_BOOKS_DIR
     args=()
   fi
-  python3 _pipeline/cli.py process "$pdf" "${args[@]}" --result-file "$result"
+  python3 _pipeline/cli.py process "$pdf" "${args[@]}" "${extra[@]}" --result-file "$result"
   rc=$?
   if [ $rc -ne 0 ]; then
-    echo "FAILED: process exited $rc for $pdf. Nothing was committed. The PDF stays where it is."
+    echo "FAILED: process exited $rc for $item. Nothing was committed. The file stays where it is in _incoming/."
     git checkout -q -- . 2>/dev/null; git clean -qfd -- books 2>/dev/null
     failed=1; continue
   fi
@@ -87,14 +103,14 @@ for pdf in "${pdfs[@]}"; do
   if ! git ls-files --error-unmatch -- "$rel/book.json" >/dev/null 2>&1; then
     echo "FAILED: $rel/book.json is not tracked after git add. Nothing was committed."; failed=1; continue
   fi
-  case "$pdf" in
+  case "$item" in
-    _incoming/*)
+    _incoming/*)       # the dropped PDF or link file (and its optional .json) leaves _incoming/ in the same commit
-      git rm -q -- "$pdf"
+      git rm -q -- "$item"
-      side="${pdf%.[pP][dD][fF]}.json"; [ -f "$side" ] && git rm -q -- "$side"
+      side="${item%.*}.json"; [ -f "$side" ] && git rm -q -- "$side"
       ;;
   esac
-  if git diff --cached --quiet; then echo "No changes to commit for $pdf"; continue; fi
+  if git diff --cached --quiet; then echo "No changes to commit for $item"; continue; fi
-  git commit -q -m "Audiobook: $rel (from $pdf, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]"
+  git commit -q -m "Audiobook: $rel (from $item, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]"
   git --no-pager show --stat --format='commit %H' HEAD | head -40
   if push_with_retry; then echo "PUSHED to $BRANCH: $rel"; else echo "FAILED: could not push to $BRANCH"; failed=1; fi
 done

_pipeline/cli.py

@@ -9,6 +9,7 @@
   python3 _pipeline/cli.py check <mp3 or folder> ...        ACX checker, pass/fail table
 """
 import argparse
+import json
 import os
 import sys
 
@@ -71,8 +72,14 @@ def main(argv=None):
     p.add_argument("--compare", help="a books folder (like the repo books/); the text-stage output is compared with the same <author>/<book> folder in it")
     p.add_argument("--compare-strict", action="store_true", help="with --compare: fail when the output differs (default: report only)")
     p.add_argument("--result-file", help="write {ok, book_dir, pdf} JSON here")
+    p.add_argument("--source-url", help="where the PDF was downloaded from (recorded in book.json)")
+    p.add_argument("--name-hint", action="append", default=[], help="another file name to read a title from (repeatable)")
     p = sub.add_parser("bookpath", help="print the book folder (relative to the repo root) this PDF will go to")
     p.add_argument("pdf")
+    p.add_argument("--name-hint", action="append", default=[])
+    p = sub.add_parser("fetch", help="download the PDF named by a .url/.txt file into a scratch folder; writes <out>/fetch.json")
+    p.add_argument("urlfile")
+    p.add_argument("--out", required=True)
     p = sub.add_parser("ingest")
     p.add_argument("pdf")
     p.add_argument("--title")
@@ -97,14 +104,24 @@ def main(argv=None):
     if a.cmd == "bookpath":
         import process
         try:
-            title, author, _ = process.resolve_meta(os.path.abspath(a.pdf))
+            m = process.resolve_meta(cfg, os.path.abspath(a.pdf), name_hints=a.name_hint)
         except process.Failed as e:
             sys.exit(str(e))
-        print(os.path.relpath(ingest.book_folder(cfg, author, title), settings.REPO_ROOT))
+        print("\n".join(m["log"]), file=sys.stderr)
+        print(os.path.relpath(ingest.book_folder(cfg, m["author"], m["title"]), settings.REPO_ROOT))
+        return
+    if a.cmd == "fetch":
+        import fetch
+        try:
+            res = fetch.fetch_from_file(cfg, a.urlfile, a.out)
+            with open(os.path.join(a.out, "fetch.json"), "w", encoding="utf-8") as f:
+                json.dump(res, f)
+        except fetch.FetchError as e:
+            sys.exit("FETCH FAILED: %s" % e)
         return
     if a.cmd == "process":
         import process
-        sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict))
+        sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict, a.source_url, a.name_hint))
     if a.cmd in ("run", "ingest"):
         book_dir = ingest.run(cfg, os.path.abspath(a.pdf), a.title, a.author)
         print("ingest     done ->", book_dir)

_pipeline/fetch.py

@@ -0,0 +1,162 @@
+"""Drop a link instead of a file: _incoming/<anything>.url or .txt holding one http(s) address of a PDF.
+
+  * The file may be plain text (just the address) or a Windows internet shortcut ([InternetShortcut] / URL=...).
+  * An arXiv page address (arxiv.org/abs/<id>) is rewritten to the PDF address (arxiv.org/pdf/<id>).
+  * The download uses curl: follows redirects, connection and total time limits, a size cap (settings fetch.max_mb;
+    enforced by curl and again by a hard file-size limit on the process and a check afterwards).
+  * The result must really be a PDF: the bytes must start with %PDF- and pdfinfo must read it. Otherwise the run fails
+    and says why (bad address, HTTP error, not a PDF, too large, timeout...). Nothing is committed on failure.
+  * The file name comes from the server (Content-Disposition) or the last part of the address, with .pdf added if missing.
+The download stays in the scratch folder; the pipeline then stores it as the book's source PDF and records the address in
+book.json (source_url). Nothing is published anywhere else."""
+import json
+import os
+import re
+import resource
+import shutil
+import subprocess
+from urllib.parse import unquote, urlparse
+
+UA = "Mozilla/5.0 (compatible; audiobook-pipeline/1.0; +drop-folder)"
+
+
+class FetchError(Exception):
+    """The message is what the operator reads in the run log."""
+
+
+def read_url_file(path):
+    """The one http(s) address in a .url/.txt file."""
+    with open(path, encoding="utf-8-sig", errors="replace") as f:
+        lines = [ln.strip() for ln in f.read().splitlines()]
+    found = []
+    for ln in lines:
+        if not ln or ln.startswith(("#", ";", "[")):
+            continue
+        if re.match(r"^url\s*=", ln, re.I):
+            ln = ln.split("=", 1)[1].strip()
+        elif "=" in ln.split()[0] and not ln.lower().startswith(("http://", "https://")):
+            continue                      # other shortcut keys (IconIndex=0, ...)
+        found.append(ln)
+    if not found:
+        raise FetchError("%s is empty: put one web address (https://...) in it" % os.path.basename(path))
+    if len(set(found)) > 1:
+        raise FetchError("%s has %d different lines; it must hold exactly one web address (https://...)" % (os.path.basename(path), len(set(found))))
+    url = found[0]
+    if not re.match(r"^https?://[^\s/]+", url, re.I) or re.search(r"\s", url):
+        raise FetchError("%s: %r is not a web address. It must start with http:// or https:// and contain no spaces" % (os.path.basename(path), url[:120]))
+    return url
+
+
+def normalize_url(url):
+    """arxiv.org/abs/<id> -> arxiv.org/pdf/<id>. Everything else is used as it is."""
+    m = re.match(r"^https?://(?:www\.|export\.)?arxiv\.org/abs/(.+?)/?(?:[?#].*)?$", url, re.I)
+    if m:
+        return "https://arxiv.org/pdf/" + m.group(1)
+    return url
+
+
+def safe_name(name):
+    name = unquote(name).strip().strip(".")
+    name = re.sub(r"[^A-Za-z0-9._() \-]+", "_", os.path.basename(name)).strip(" _")
+    if not name:
+        name = "download"
+    if not name.lower().endswith(".pdf"):
+        name += ".pdf"
+    return name
+
+
+def name_from_response(headers_text, final_url):
+    """File name from the last Content-Disposition header, else the last part of the final address."""
+    found = re.findall(r"^content-disposition:.*$", headers_text, re.I | re.M)
+    if found:
+        m = re.search(r"filename\*\s*=\s*[^']*'[^']*'([^;\r\n]+)", found[-1], re.I) or re.search(r'filename\s*=\s*"?([^";\r\n]+)"?', found[-1], re.I)
+        if m:
+            return safe_name(m.group(1))
+    path = urlparse(final_url).path.rstrip("/")
+    return safe_name(path.rsplit("/", 1)[-1] if path else "")
+
+
+def check_pdf(path):
+    with open(path, "rb") as f:
+        head = f.read(2048)
+    if b"%PDF-" not in head[:1024]:
+        kind = "an HTML page" if re.search(rb"<\s*(!doctype|html|head|body)", head[:512], re.I) else "not a PDF file"
+        title = re.search(rb"<title[^>]*>(.*?)</title>", head, re.I | re.S)
+        raise FetchError("the link did not return a PDF (the download is %s%s). Use the direct link to the PDF file" %
+                         (kind, ": %r" % title.group(1).decode("utf-8", "replace").strip()[:80] if title else ""))
+    r = subprocess.run(["pdfinfo", path], capture_output=True, text=True)
+    pages = re.search(r"^Pages:\s+(\d+)", r.stdout, re.M)
+    if r.returncode != 0 or not pages or int(pages.group(1)) < 1:
+        raise FetchError("the download starts like a PDF but pdfinfo cannot read it (damaged or cut short): %s" % (r.stderr.strip()[:200] or "no pages"))
+    return int(pages.group(1))
+
+
+CURL_ERRORS = {
+    3: "the web address is malformed", 5: "cannot resolve the proxy", 6: "the host name does not exist (check the address)",
+    7: "the server refused the connection", 28: "the download timed out", 35: "TLS handshake failed", 47: "too many redirects",
+    51: "the server certificate does not match", 52: "the server sent an empty reply", 56: "the connection broke during the download",
+    60: "the server certificate is not trusted", 63: "the file is larger than the allowed maximum",
+}
+
+
+def download(cfg, url, out_dir):
+    """Download url into out_dir. Returns {'pdf', 'bytes', 'pages', 'final_url'}. Raises FetchError."""
+    fc = cfg.get("fetch", {})
+    max_bytes = int(float(fc.get("max_mb", 300)) * 1024 * 1024)
+    os.makedirs(out_dir, exist_ok=True)
+    part = os.path.join(out_dir, "download.partial")
+    hdr = os.path.join(out_dir, "headers.txt")
+    for p in (part, hdr):
+        if os.path.exists(p):
+            os.remove(p)
+    cmd = ["curl", "-sS", "-L", "--fail", "--max-redirs", "10", "--connect-timeout", str(fc.get("connect_timeout_s", 30)),
+           "--max-time", str(fc.get("max_time_s", 900)), "--max-filesize", str(max_bytes), "-A", UA, "-D", hdr, "-o", part,
+           "-w", "%{url_effective}", "--proto", "=http,https", "--proto-redir", "=http,https", url]
+    limit = lambda: resource.setrlimit(resource.RLIMIT_FSIZE, (max_bytes + (1 << 20), max_bytes + (1 << 20)))
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit)
+    except FileNotFoundError:
+        raise FetchError("curl is not installed on this host")
+    if r.returncode != 0:
+        if os.path.exists(part):
+            os.remove(part)
+        if r.returncode in (-25, 25) or r.returncode == 63:
+            raise FetchError("the file is larger than the allowed maximum of %s MB (settings fetch.max_mb)" % fc.get("max_mb", 300))
+        detail = r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "curl exit %d" % r.returncode
+        m = re.search(r"returned error: (\d+)", detail)
+        why = ("the server answered HTTP %s (%s)" % (m.group(1), {"404": "not found", "403": "forbidden", "401": "login required", "410": "gone"}.get(m.group(1), "error"))
+               if m else CURL_ERRORS.get(r.returncode, detail))
+        raise FetchError("could not download %s: %s" % (url, why))
+    size = os.path.getsize(part)
+    if size == 0:
+        raise FetchError("the download from %s was empty" % url)
+    if size > max_bytes:
+        os.remove(part)
+        raise FetchError("the file is %.0f MB, larger than the allowed maximum of %s MB (settings fetch.max_mb)" % (size / 1048576, fc.get("max_mb", 300)))
+    final_url = r.stdout.strip() or url
+    headers = ""
+    if os.path.exists(hdr):
+        with open(hdr, encoding="utf-8", errors="replace") as f:
+            headers = f.read()
+    try:
+        pages = check_pdf(part)
+    except FetchError:
+        os.remove(part)
+        raise
+    pdf = os.path.join(out_dir, name_from_response(headers, final_url))
+    shutil.move(part, pdf)
+    return {"pdf": pdf, "bytes": size, "pages": pages, "final_url": final_url}
+
+
+def fetch_from_file(cfg, urlfile, out_dir):
+    """Everything for one .url/.txt file. Returns a JSON-able dict: pdf, source_url, original_url, name_hints, bytes, pages."""
+    original = read_url_file(urlfile)
+    url = normalize_url(original)
+    print("link file %s -> %s%s" % (os.path.basename(urlfile), url, "  (rewritten from %s)" % original if url != original else ""))
+    got = download(cfg, url, out_dir)
+    side = os.path.splitext(urlfile)[0] + ".json"
+    if os.path.exists(side):
+        shutil.copyfile(side, os.path.splitext(got["pdf"])[0] + ".json")
+    print("downloaded %s: %.1f MB, %d pages (valid PDF)" % (os.path.basename(got["pdf"]), got["bytes"] / 1048576, got["pages"]))
+    return {"pdf": got["pdf"], "source_url": url,
+            "original_url": original, "name_hints": [os.path.basename(urlfile)], "bytes": got["bytes"], "pages": got["pages"]}

_pipeline/tests/test_fetch.py

@@ -0,0 +1,189 @@
+"""Link files in _incoming/: .url / .txt reading, arXiv rewrite, and the download against a local test server."""
+import http.server
+import json
+import os
+import sys
+import tempfile
+import threading
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(HERE, ".."))
+sys.path.insert(0, HERE)
+
+import bookio
+import fetch
+import ingest
+import settings
+
+try:
+    from helpers import make_pdf
+except ImportError:
+    make_pdf = None
+
+PDF_BYTES = b""
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    hits = []
+
+    def log_message(self, *a):
+        pass
+
+    def send(self, code, body=b"", ctype="application/pdf", extra=None):
+        self.send_response(code)
+        self.send_header("Content-Type", ctype)
+        self.send_header("Content-Length", str(len(body)))
+        for k, v in (extra or {}).items():
+            self.send_header(k, v)
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self):
+        Handler.hits.append(self.path)
+        if self.path == "/files/Deep-Work.pdf":
+            self.send(200, PDF_BYTES)
+        elif self.path == "/redirect":
+            self.send(302, extra={"Location": "/files/Deep-Work.pdf"})
+        elif self.path == "/pdf/2401.01234":                  # no extension, like arxiv.org/pdf/<id>
+            self.send(200, PDF_BYTES)
+        elif self.path == "/dl?id=7":
+            self.send(200, PDF_BYTES, extra={"Content-Disposition": 'attachment; filename="Named By Server.pdf"'})
+        elif self.path == "/page":
+            self.send(200, b"<!doctype html><html><head><title>Sign in</title></head><body>login</body></html>", "text/html")
+        elif self.path == "/cut":
+            self.send(200, PDF_BYTES[:200])
+        elif self.path == "/empty":
+            self.send(200, b"")
+        else:
+            self.send(404, b"not found", "text/plain")
+
+
+@unittest.skipIf(make_pdf is None, "reportlab not installed")
+class FetchTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        global PDF_BYTES
+        d = tempfile.mkdtemp()
+        make_pdf(os.path.join(d, "a.pdf"), [["Deep Work", "By Cal Newport"], ["text"]], title="Deep Work")
+        PDF_BYTES = open(os.path.join(d, "a.pdf"), "rb").read()
+        cls.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        threading.Thread(target=cls.server.serve_forever, daemon=True).start()
+        cls.base = "http://127.0.0.1:%d" % cls.server.server_port
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.server.shutdown()
+
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.d = self.tmp.name
+        self.cfg = settings.load()
+        self.out = os.path.join(self.d, "out")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def urlfile(self, content, name="Some Link.url"):
+        p = os.path.join(self.d, name)
+        with open(p, "w", encoding="utf-8") as f:
+            f.write(content)
+        return p
+
+    def test_download_follows_redirects_and_verifies_pdf(self):
+        r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/redirect\n"), self.out)
+        self.assertEqual(os.path.basename(r["pdf"]), "Deep-Work.pdf")
+        self.assertEqual(r["pages"], 2)
+        self.assertEqual(open(r["pdf"], "rb").read(), PDF_BYTES)
+        self.assertEqual(r["source_url"], self.base + "/redirect")
+        self.assertEqual(r["name_hints"], ["Some Link.url"])
+
+    def test_windows_shortcut_format_and_txt(self):
+        r = fetch.fetch_from_file(self.cfg, self.urlfile("[InternetShortcut]\r\nURL=%s/files/Deep-Work.pdf\r\nIconIndex=0\r\n" % self.base), self.out)
+        self.assertTrue(r["pdf"].endswith("Deep-Work.pdf"))
+        r = fetch.fetch_from_file(self.cfg, self.urlfile("# my link\n%s/files/Deep-Work.pdf\n" % self.base, "note.txt"), os.path.join(self.d, "o2"))
+        self.assertTrue(os.path.exists(r["pdf"]))
+
+    def test_name_from_url_without_extension_and_from_content_disposition(self):
+        r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/pdf/2401.01234"), self.out)
+        self.assertEqual(os.path.basename(r["pdf"]), "2401.01234.pdf")
+        r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/dl?id=7"), os.path.join(self.d, "o2"))
+        self.assertEqual(os.path.basename(r["pdf"]), "Named By Server.pdf")
+
+    def test_sidecar_json_next_to_the_url_file_is_carried_over(self):
+        self.urlfile('{"title": "T", "author": "A"}', "Some Link.json")
+        r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/files/Deep-Work.pdf"), self.out)
+        self.assertEqual(json.load(open(os.path.splitext(r["pdf"])[0] + ".json"))["author"], "A")
+
+    def test_html_page_is_refused_with_a_clear_reason(self):
+        with self.assertRaises(fetch.FetchError) as cm:
+            fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/page"), self.out)
+        self.assertIn("did not return a PDF", str(cm.exception))
+        self.assertIn("HTML page", str(cm.exception))
+        self.assertEqual(os.listdir(self.out) and [n for n in os.listdir(self.out) if n.endswith(".pdf")], [])
+
+    def test_http_404_cut_off_pdf_and_empty_download(self):
+        for path, text in (("/nothing", "HTTP 404 (not found)"), ("/cut", "cannot read it"), ("/empty", "empty")):
+            with self.assertRaises(fetch.FetchError) as cm:
+                fetch.fetch_from_file(self.cfg, self.urlfile(self.base + path), self.out)
+            self.assertIn(text, str(cm.exception), path)
+
+    def test_too_large_is_refused(self):
+        self.cfg["fetch"]["max_mb"] = 0.0005          # about 500 bytes; the test PDF is a few KB
+        with self.assertRaises(fetch.FetchError) as cm:
+            fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/files/Deep-Work.pdf"), self.out)
+        self.assertIn("larger than the allowed maximum", str(cm.exception))
+
+    def test_unreachable_host(self):
+        with self.assertRaises(fetch.FetchError) as cm:
+            fetch.fetch_from_file(self.cfg, self.urlfile("http://127.0.0.1:9/x.pdf"), self.out)
+        self.assertIn("refused the connection", str(cm.exception))
+
+    def test_ingest_stores_the_downloaded_pdf_and_records_the_source_url(self):
+        r = fetch.fetch_from_file(self.cfg, self.urlfile(self.base + "/files/Deep-Work.pdf"), self.out)
+        self.cfg["books_dir"] = os.path.join(self.d, "books")
+        self.cfg["work_dir"] = os.path.join(self.d, "work")
+        book_dir = ingest.run(self.cfg, r["pdf"], source_url=r["source_url"])
+        book = bookio.read_book(book_dir)
+        self.assertEqual(book["source_url"], self.base + "/files/Deep-Work.pdf")
+        self.assertEqual((book["author"], book["title"]), ("Cal Newport", "Deep Work"))
+        self.assertTrue(os.path.exists(os.path.join(book_dir, "Deep-Work.pdf")))
+
+
+class PureTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def read(self, content):
+        p = os.path.join(self.tmp.name, "x.url")
+        with open(p, "w", encoding="utf-8-sig") as f:
+            f.write(content)
+        return fetch.read_url_file(p)
+
+    def test_arxiv_abs_is_rewritten(self):
+        self.assertEqual(fetch.normalize_url("https://arxiv.org/abs/2401.01234"), "https://arxiv.org/pdf/2401.01234")
+        self.assertEqual(fetch.normalize_url("http://www.arxiv.org/abs/math/0301001v2"), "https://arxiv.org/pdf/math/0301001v2")
+        self.assertEqual(fetch.normalize_url("https://arxiv.org/pdf/2401.01234"), "https://arxiv.org/pdf/2401.01234")
+        self.assertEqual(fetch.normalize_url("https://example.org/a.pdf"), "https://example.org/a.pdf")
+
+    def test_reading_link_files(self):
+        self.assertEqual(self.read("https://example.org/a.pdf  \n\n"), "https://example.org/a.pdf")
+        self.assertEqual(self.read("[InternetShortcut]\nURL=https://example.org/b.pdf\nIDList=\n"), "https://example.org/b.pdf")
+        for bad, why in (("", "empty"), ("ftp://x/y.pdf", "not a web address"), ("see https://a.org/x.pdf", "not a web address"),
+                         ("https://a.org/1.pdf\nhttps://a.org/2.pdf", "exactly one")):
+            with self.assertRaises(fetch.FetchError) as cm:
+                self.read(bad)
+            self.assertIn(why, str(cm.exception), bad)
+
+    def test_safe_names(self):
+        self.assertEqual(fetch.safe_name("a%20b.pdf"), "a b.pdf")
+        self.assertEqual(fetch.safe_name(""), "download.pdf")
+        self.assertEqual(fetch.safe_name("2401.01234"), "2401.01234.pdf")
+        self.assertEqual(fetch.name_from_response("HTTP/1.1 200 OK\r\n", "https://x.org/files/My%20Book.pdf?x=1"), "My Book.pdf")
+
+
+if __name__ == "__main__":
+    unittest.main()