Ty_Tech/old-books
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Metadata never fails a drop: deterministic title/author detection (metadata, first-page text, OCR of scanned first pages, file name), sources recorded in book.json; decrypt scratch copy before OCR

Browse Source
This commit is contained in:
Audiobook Producer
2026-10-18 13:25:01 -05:00
parent be069d3c22
commit b72c0464ef
6 changed files with 583 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_pipeline/ingest.py
+19 -28
View File
@@ -19,21 +19,6 @@ def sha256(path):
return h.hexdigest()
def find_title(pdf):
out = subprocess.run(["pdfinfo", pdf], capture_output=True, text=True).stdout
m = re.search(r"^Title:\s+(.+)$", out, re.M)
return bookio.title_case(m.group(1).strip()) if m else ""
def find_author(unit_texts):
"""The author is the line 'By <Name>' on the first pages."""
for text in unit_texts:
m = re.search(r"^\s*By\s+([A-Z][A-Za-z.\- ]+?)\s*$", text, re.M)
if m:
return m.group(1).strip()
return ""
def author_slug(author):
"""'Glenn Clark' -> 'clark-glenn' (surname first, last word taken as surname)."""
parts = author.split()
@@ -44,18 +29,13 @@ def book_folder(cfg, author, title):
return os.path.join(settings.resolve(cfg, "books_dir"), author_slug(author), bookio.slug(title))
def detect(pdf, title=None, author=None):
"""Title and author from the PDF itself (title: PDF metadata; author: a 'By <Name>' line on the first pages)."""
author = author or find_author([u["text"] for u in pdftext.units(pdf, first_pages=6)])
title = title or find_title(pdf)
return title, author
def run(cfg, pdf, title=None, author=None):
"""Returns the book folder path."""
title, author = detect(pdf, title, author)
if not author or not title:
raise SystemExit("could not find title/author in the PDF; pass --title and --author")
def run(cfg, pdf, title=None, author=None, meta=None, source_url=None):
"""Returns the book folder path. Title and author come from `meta` (see meta.resolve) or, if none is given, from
meta.resolve here. Missing metadata never stops ingest: it falls back and logs a warning."""
import meta as meta_mod
if meta is None:
meta = meta_mod.resolve(cfg, pdf, title, author)
title, author = meta["title"], meta["author"]
book_dir = book_folder(cfg, author, title)
os.makedirs(book_dir, exist_ok=True)
copy = os.path.join(book_dir, os.path.basename(pdf))
@@ -70,17 +50,21 @@ def run(cfg, pdf, title=None, author=None):
book.update({
"title": title,
"author": author,
"metadata_sources": {"title": meta["title_source"], "author": meta["author_source"]},
"source_pdf": os.path.basename(pdf),
"sha256": sha256(copy),
"pdf_pages": len(counts),
"pages_without_text_layer": [i + 1 for i, c in enumerate(counts) if c < cfg["ocr"]["min_chars_per_page"]],
})
if source_url:
book["source_url"] = source_url
book.setdefault("coauthors", [])
book.setdefault("publisher", "")
book.setdefault("edition", "")
book.setdefault("series", "")
book.setdefault("summaries", {})
bookio.write_book(book_dir, book)
bookio.write_warnings(book_dir, "ingest-metadata", meta["warnings"])
warnings = ocr_if_needed(cfg, book_dir, book)
bookio.write_warnings(book_dir, "ingest", warnings)
return book_dir
@@ -109,7 +93,14 @@ def ocr_if_needed(cfg, book_dir, book):
# behind a symlink outside its permitted paths, e.g. in a CI workspace. Measured in a job container (2026-10-18).
src = os.path.join(out_dir, "in.pdf")
shutil.copyfile(os.path.join(book_dir, book["source_pdf"]), src)
r = subprocess.run(["ocrmypdf", "--skip-text", "-l", cfg["ocr"]["language"], src, out], capture_output=True, text=True)
if "yes" in (re.search(r"^Encrypted:\s+(\w+)", subprocess.run(["pdfinfo", src], capture_output=True, text=True).stdout, re.M) or [0, ""])[1]:
# owner-password-only PDFs (copy/print restrictions) cannot be OCRed as they are: ocrmypdf refuses them. Decrypt the scratch copy only.
dec = os.path.join(out_dir, "in-decrypted.pdf")
q = subprocess.run(["qpdf", "--decrypt", src, dec], capture_output=True, text=True) if shutil.which("qpdf") else None
if q is None or q.returncode not in (0, 3) or not os.path.exists(dec):
return [("ocr-failed", where, "the PDF is encrypted and could not be decrypted for OCR: %s" % (q.stderr.strip()[:300] if q else "qpdf is not installed"))]
src = dec
r = subprocess.run(["ocrmypdf", "--skip-text", "--output-type", "pdf", "-l", cfg["ocr"]["language"], src, out], capture_output=True, text=True)
if r.returncode != 0:
print("ocrmypdf failed, exit %d. stdout: %r stderr: %r" % (r.returncode, r.stdout[-1500:], r.stderr[-3000:]))
return [("ocr-failed", where, "ocrmypdf exit %d: %s" % (r.returncode, (r.stderr.strip()[:700] + " ... " + r.stderr.strip()[-500:]) if len(r.stderr) > 1250 else r.stderr.strip()))]