Structure: fuzzy CHAPTER keyword for OCR, cap run-on titles, prefer contents-page title when it has fewer unknown words; metadata: multi-resolution OCR of title pages, OCR name correction from repeated words; ingest: qpdf optional

_pipeline/structure.py

@@ -1,4 +1,5 @@
 """Stage 3, structure: find chapter boundaries from the text layer. Audible style file list is made by the script stage."""
+import difflib
 import re
 
 import bookio
@@ -7,7 +8,7 @@ import pdftext
 
 WORDS = "ONE TWO THREE FOUR FIVE SIX SEVEN EIGHT NINE TEN ELEVEN TWELVE THIRTEEN FOURTEEN FIFTEEN SIXTEEN SEVENTEEN EIGHTEEN NINETEEN TWENTY".split()
 ROMAN = {"I": 1, "V": 5, "X": 10, "L": 50}
-HEADING_RE = re.compile(r"^\s*CHAPTER\b[\s.:]*(.*?)[\s.,:;]*$")
+HEADING_RE = re.compile(r"^\s*([A-Z]{6,9}|CHAPTER)\b[\s.:]*(.*?)[\s.,:;]*$")
 
 
 def chapter_number(token):
@@ -23,6 +24,12 @@ def chapter_number(token):
     return None
 
 
+def is_chapter_word(word):
+    """CHAPTER, or what OCR makes of it (CUAPTER, CIEAPTER, OUAPTER): upper-case letters, 6 to 9 of them, 70% like CHAPTER."""
+    return word == "CHAPTER" or (re.fullmatch(r"[A-Z]{6,9}", word) is not None and difflib.SequenceMatcher(None, word, "CHAPTER").ratio() >= 0.7
+                                 and word not in ("CHAPTERS", "CAPTAIN", "CHARTER", "CHAPLET"))
+
+
 def heading_number(line, expected):
     """(number, guessed) for a 'CHAPTER <number>' line, else None. The number may be a roman numeral, digits or a word.
     OCR often damages the numeral ('I I r', 'X V', 'X f I'): a short unreadable token after an upper-case CHAPTER is
@@ -30,7 +37,9 @@ def heading_number(line, expected):
     m = HEADING_RE.match(line)
     if not m:
         return None
-    rest = m.group(1).strip()
+    if not is_chapter_word(m.group(1)):
+        return None
+    rest = m.group(2).strip()
     if re.fullmatch(r"[A-Z]+|\d+", rest):
         n = chapter_number(rest)
         if n is not None:
@@ -68,6 +77,12 @@ def find_headings(units):
             while k < len(lines) and lines[k].strip():
                 title.append(lines[k].strip())
                 k += 1
+            if len(" ".join(title)) > 100:        # no blank line after the title: keep the leading upper-case lines only
+                keep = 1
+                while keep < len(title) and keep < 3 and pageparts._mostly_upper(title[keep]):
+                    keep += 1
+                k = k - (len(title) - keep)
+                title = title[:keep]
             h = {"unit": ui, "line": li, "title_end": k, "number": n, "title": bookio.title_case(" ".join(title))}
             if guessed:
                 h["guessed"] = True
@@ -100,6 +115,29 @@ def toc_chapters(units, before_unit):
     return out
 
 
+def unknown_words(text):
+    """How many words of a title the spelling dictionary does not know (used to choose between two readings of one title)."""
+    import cleanup
+    return sum(1 for w in re.findall(r"[A-Za-z]+", text) if len(w) > 2 and not cleanup.known(w))
+
+
+def toc_chapter_lines(units, before_unit):
+    """Contents in the form 'CHAPTER V.' on one line and 'Title ........ 20' on the next: {number: (title, page)}."""
+    lines = []
+    for u in units[:before_unit]:
+        lines += [ln for ln in u["text"].split("\n") if ln.strip()]
+    out, last = {}, 0
+    for i, ln in enumerate(lines[:-1]):
+        got = heading_number(ln, last + 1)
+        if got is None or not re.search(r"\.{3,}", lines[i + 1]):
+            continue
+        title = re.split(r"\s*\.{3,}", lines[i + 1].strip())[0].strip(" .,")
+        if len(re.sub(r"[^A-Za-z]", "", title)) >= 4 and got[0] not in out:
+            out[got[0]] = (bookio.title_case(re.sub(r"\s+", " ", title)), None)
+            last = got[0]
+    return out
+
+
 def book_pages(units, title):
     """Printed page number of each unit (None if unknown). Printed numbers are read from headers/footers;
     a unit without one takes its neighbor's number plus or minus its distance."""
@@ -140,6 +178,8 @@ def run(book_dir, book, pdf):
             warnings.append(("heading-replaced", "chapter %d" % h["number"], "an earlier CHAPTER line with the same or a higher number (a contents entry or misread) was dropped in favour of the heading on pdf page %d" % units[h["unit"]]["pdf_page"]))
     pages, printed = book_pages(units, book["title"])
     toc = {n: (t, p) for n, t, p in toc_chapters(units, heads[0]["unit"])}
+    for n, tp in toc_chapter_lines(units, heads[0]["unit"]).items():
+        toc.setdefault(n, tp)
     old = {c["number"]: c for c in book.get("chapters", [])}
     chapters = []
     for k, h in enumerate(heads):
@@ -169,7 +209,11 @@ def run(book_dir, book, pdf):
             if t_page is not None and t_page != ch["book_pages"][0]:
                 warnings.append(("toc-mismatch", "chapter %d" % h["number"],
                                  "table of contents says page %s, structure found page %s" % (t_page, ch["book_pages"][0])))
-            if pageparts.similar(t_title, h["title"]) < 0.9:
+            sim = pageparts.similar(t_title, h["title"])
+            if 0.5 <= sim < 1 and t_page is None and not h.get("synthetic") and unknown_words(t_title) < unknown_words(h["title"]):
+                warnings.append(("title-from-contents", "chapter %d" % h["number"], "heading read '%s' (OCR damage?); the contents page says '%s' with fewer unknown words, which is used" % (h["title"], t_title)))
+                ch["title"] = t_title
+            elif sim < 0.9:
                 warnings.append(("toc-mismatch", "chapter %d" % h["number"],
                                  "table of contents title '%s' differs from heading '%s'" % (t_title, h["title"])))
         chapters.append(ch)