Audiobook Producer
108 lines
5.3 KiB
Python
108 lines
5.3 KiB
Python
"""Stage 1, ingest: copy the original PDF into its book folder (never modify it), record its sha256,
|
|
find pages without a text layer and OCR only if there are any."""
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
|
|
import bookio
|
|
import pdftext
|
|
import settings
|
|
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for block in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def author_slug(author):
|
|
"""'Glenn Clark' -> 'clark-glenn' (surname first, last word taken as surname)."""
|
|
parts = author.split()
|
|
return bookio.slug(" ".join([parts[-1]] + parts[:-1])) if parts else "unknown"
|
|
|
|
|
|
def book_folder(cfg, author, title):
|
|
return os.path.join(settings.resolve(cfg, "books_dir"), author_slug(author), bookio.slug(title))
|
|
|
|
|
|
def run(cfg, pdf, title=None, author=None, meta=None, source_url=None):
|
|
"""Returns the book folder path. Title and author come from `meta` (see meta.resolve) or, if none is given, from
|
|
meta.resolve here. Missing metadata never stops ingest: it falls back and logs a warning."""
|
|
import meta as meta_mod
|
|
if meta is None:
|
|
meta = meta_mod.resolve(cfg, pdf, title, author)
|
|
title, author = meta["title"], meta["author"]
|
|
book_dir = book_folder(cfg, author, title)
|
|
os.makedirs(book_dir, exist_ok=True)
|
|
copy = os.path.join(book_dir, os.path.basename(pdf))
|
|
if os.path.exists(copy) and sha256(copy) != sha256(pdf):
|
|
raise SystemExit("a different file with this name is already in the book folder: " + copy)
|
|
if not os.path.exists(copy):
|
|
shutil.copyfile(pdf, copy)
|
|
book = {}
|
|
if os.path.exists(os.path.join(book_dir, "book.json")):
|
|
book = bookio.read_book(book_dir)
|
|
counts = pdftext.chars_per_page(copy)
|
|
book.update({
|
|
"title": title,
|
|
"author": author,
|
|
"metadata_sources": {"title": meta["title_source"], "author": meta["author_source"]},
|
|
"source_pdf": os.path.basename(pdf),
|
|
"sha256": sha256(copy),
|
|
"pdf_pages": len(counts),
|
|
"pages_without_text_layer": [i + 1 for i, c in enumerate(counts) if c < cfg["ocr"]["min_chars_per_page"]],
|
|
})
|
|
if source_url:
|
|
book["source_url"] = source_url
|
|
book.setdefault("coauthors", [])
|
|
book.setdefault("publisher", "")
|
|
book.setdefault("edition", "")
|
|
book.setdefault("series", "")
|
|
book.setdefault("summaries", {})
|
|
bookio.write_book(book_dir, book)
|
|
bookio.write_warnings(book_dir, "ingest-metadata", meta["warnings"])
|
|
warnings = ocr_if_needed(cfg, book_dir, book)
|
|
bookio.write_warnings(book_dir, "ingest", warnings)
|
|
return book_dir
|
|
|
|
|
|
def work_pdf(cfg, book_dir, book):
|
|
"""The PDF text is read from: the OCR copy if one exists, else the untouched original."""
|
|
ocr_pdf = os.path.join(settings.resolve(cfg, "work_dir"), os.path.basename(book_dir), "ocr.pdf")
|
|
return ocr_pdf if os.path.exists(ocr_pdf) else os.path.join(book_dir, book["source_pdf"])
|
|
|
|
|
|
def ocr_if_needed(cfg, book_dir, book):
|
|
"""OCR a scratch copy (never the original) when pages lack a text layer. Returns warnings."""
|
|
missing = book["pages_without_text_layer"]
|
|
if not missing:
|
|
return []
|
|
where = "pdf pages " + ",".join(map(str, missing))
|
|
if not cfg["ocr"]["enabled"]:
|
|
return [("no-text-layer", where, "no text layer and OCR is disabled in settings")]
|
|
if shutil.which("ocrmypdf") is None:
|
|
return [("no-text-layer", where, "no text layer and ocrmypdf is not installed on this host; those pages have no text")]
|
|
out_dir = os.path.join(settings.resolve(cfg, "work_dir"), os.path.basename(book_dir))
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
out = os.path.join(out_dir, "ocr.pdf")
|
|
# OCR a copy kept in the scratch folder: Ghostscript 10 (-dSAFER, used inside ocrmypdf) cannot open a PDF that sits
|
|
# behind a symlink outside its permitted paths, e.g. in a CI workspace. Measured in a job container (2026-10-18).
|
|
src = os.path.join(out_dir, "in.pdf")
|
|
shutil.copyfile(os.path.join(book_dir, book["source_pdf"]), src)
|
|
if "yes" in (re.search(r"^Encrypted:\s+(\w+)", subprocess.run(["pdfinfo", src], capture_output=True, text=True).stdout, re.M) or [0, ""])[1]:
|
|
# owner-password-only PDFs (copy/print restrictions) cannot be OCRed as they are: ocrmypdf refuses them. Decrypt the scratch copy only.
|
|
dec = os.path.join(out_dir, "in-decrypted.pdf")
|
|
q = subprocess.run(["qpdf", "--decrypt", src, dec], capture_output=True, text=True) if shutil.which("qpdf") else None
|
|
if q is None or q.returncode not in (0, 3) or not os.path.exists(dec):
|
|
return [("ocr-failed", where, "the PDF is encrypted and could not be decrypted for OCR: %s" % (q.stderr.strip()[:300] if q else "qpdf is not installed"))]
|
|
src = dec
|
|
r = subprocess.run(["ocrmypdf", "--skip-text", "--output-type", "pdf", "-l", cfg["ocr"]["language"], src, out], capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
print("ocrmypdf failed, exit %d. stdout: %r stderr: %r" % (r.returncode, r.stdout[-1500:], r.stderr[-3000:]))
|
|
return [("ocr-failed", where, "ocrmypdf exit %d: %s" % (r.returncode, (r.stderr.strip()[:700] + " ... " + r.stderr.strip()[-500:]) if len(r.stderr) > 1250 else r.stderr.strip()))]
|
|
return [("ocr", where, "OCR applied to a scratch copy; original untouched")]
|