Structure: fuzzy CHAPTER keyword for OCR, cap run-on titles, prefer contents-page title when it has fewer unknown words; metadata: multi-resolution OCR of title pages, OCR name correction from repeated words; ingest: qpdf optional

_pipeline/meta.py

@@ -92,7 +92,8 @@ def looks_like_name(tokens):
 
 def first_author(text):
     """'Glenn Clark and Walter Russell' / 'A. B, C. D' -> first person; None if it does not look like a name."""
-    text = re.split(r"\s+(?:and|&|with)\s+|,\s+(?=[A-Z])", clean_text(text).strip(" .,;:"))[0]
+    text = re.sub(r"[^A-Za-z.'’\-,& ]", "", text)             # OCR specks after the name (RAMACHARAKA°, Name!)
+    text = re.split(r"\s+(?:and|&|with)\s+|,\s+(?=[A-Z])", clean_text(text).strip(" .,;:'’-"))[0]
     tokens = text.split(" ")
     return display_name(text) if looks_like_name(tokens) else None
 
@@ -215,15 +216,15 @@ def pdf_info(pdf):
     return {"title": get("Title"), "author": get("Author")}
 
 
-def ocr_first_pages(pdf, first, last, lang):
+def ocr_first_pages(pdf, first, last, lang, dpi=300):
     """OCR of single pages with pdftoppm + tesseract, for pages that have no text layer. Returns [(page, text)]."""
     if OCR_FUNC:
-        return OCR_FUNC(pdf, first, last, lang)
+        return OCR_FUNC(pdf, first, last, lang)         # tests: one pass only
     if shutil.which("tesseract") is None or shutil.which("pdftoppm") is None:
         return None
     out = []
     with tempfile.TemporaryDirectory() as tmp:
-        subprocess.run(["pdftoppm", "-r", "300", "-gray", "-f", str(first), "-l", str(last), "-png", pdf, os.path.join(tmp, "p")],
+        subprocess.run(["pdftoppm", "-r", str(dpi), "-gray", "-f", str(first), "-l", str(last), "-png", pdf, os.path.join(tmp, "p")],
                        capture_output=True, timeout=600)
         for name in sorted(os.listdir(tmp)):
             m = re.match(r"p-0*(\d+)\.png$", name)
@@ -279,17 +280,23 @@ def detect_from_pdf(cfg, pdf, hint=None):
     allpages = [(p, t) for p, t, _ in pages]
     if not page.get("author") and blank and cfg["ocr"]["enabled"]:
         if not (res["title"] and info_author):                     # nothing more to learn if metadata already gives both
-            ocr = ocr_first_pages(pdf, min(blank), max(blank), cfg["ocr"]["language"])
-            if ocr is None:
-                notes.append("the first pages have no text layer and tesseract is not installed, so they could not be read")
-            else:
+            for dpi in ((300,) if OCR_FUNC else (300, 200, 600)):      # display type reads differently at each size: first pass that finds a 'By' line wins
+                ocr = ocr_first_pages(pdf, min(blank), max(blank), cfg["ocr"]["language"], dpi)
+                if ocr is None:
+                    notes.append("the first pages have no text layer and tesseract is not installed, so they could not be read")
+                    break
                 got = dict(ocr)
                 allpages = sorted([(p, t) for p, t, _ in pages] + [(p, got[p]) for p in blank if p in got])
                 page = read_title_page(allpages, hint)
                 ocr_used = True
+                if page.get("author_source") == "by-line":
+                    notes.append("first pages read by OCR at %d dpi" % dpi)
+                    break
     src = "first-page-ocr" if ocr_used and page.get("page") in blank else "first-page-text"
     if page.get("author"):
         if src == "first-page-ocr":
+            page["author"], fixes = correct_ocr_name(page["author"], [t for _, t in allpages])
+            notes.extend(fixes)
             notes.append("author %r was read by OCR from pdf page %s; check it. A file named like the PDF with .json (author field) overrides it" % (page["author"], page["page"]))
         res["author"] = page["author"]
         res["author_source"] = ("%s (%s, pdf page %s)" % (src, page["author_source"], page["page"]))
@@ -305,6 +312,29 @@ def detect_from_pdf(cfg, pdf, hint=None):
     return res
 
 
+def correct_ocr_name(name, texts):
+    """OCR can misread letters of display type ('Yoct' for 'Yogi'). A name word of 4+ letters that appears only once on the first
+    pages (in the author line) but has a same-length twin that appears 2+ times there, sharing the first letter and at least half
+    of the letters in the same positions, is replaced by that twin. Returns (name, [what was changed])."""
+    words = {}
+    for t in texts:
+        for w in re.findall(r"[A-Za-z][A-Za-z'’]{2,}", t):
+            words[w.lower()] = words.get(w.lower(), 0) + 1
+    out, notes = [], []
+    for tok in name.split(" "):
+        low = re.sub(r"[^a-z]", "", tok.lower())
+        twins = [(n, w) for w, n in words.items() if len(low) >= 4 and words.get(low, 0) <= 1 and w != low and len(w) == len(low) and w[0] == low[0]
+                 and n >= 2 and sum(a == b for a, b in zip(w, low)) * 2 >= len(low)]
+        if twins:
+            n, w = max(twins)
+            fixed = w.upper() if tok.isupper() else w[:1].upper() + w[1:]
+            notes.append("OCR read the name word %r once; the first pages show %r %d times, so %r was used" % (tok, w, n, fixed))
+            out.append(fixed)
+        else:
+            out.append(tok)
+    return " ".join(out), notes
+
+
 def cached_detect(cfg, pdf, hint=None):
     h = hashlib.sha256()
     with open(pdf, "rb") as f: