Structure: tolerate OCR-damaged CHAPTER numerals, drop contents-page entries, one synthetic chapter when a book has no headings; metadata: OCR at 300 dpi, title-page text logged, OCR-read author flagged

_pipeline/cleanup.py

@@ -87,7 +87,7 @@ def unit_lines(unit, title, heading, first_unit, notes):
     if f:
         drop.add(f[0])
         notes.append(("page-number-removed", where, "footer page number %d" % f[1]))
-    if first_unit and heading:
+    if first_unit and heading and not heading.get("synthetic"):
         drop.update(range(0, heading["title_end"]))
         notes.append(("heading-removed", where, "chapter heading 'CHAPTER %d / %s' (spoken header is added in the script)" % (heading["number"], heading["title"])))
     # last line of running text: the lines after it (all capitals only) are picture captions, not headings

_pipeline/meta.py

@@ -223,7 +223,7 @@ def ocr_first_pages(pdf, first, last, lang):
         return None
     out = []
     with tempfile.TemporaryDirectory() as tmp:
-        subprocess.run(["pdftoppm", "-r", "200", "-gray", "-f", str(first), "-l", str(last), "-png", pdf, os.path.join(tmp, "p")],
+        subprocess.run(["pdftoppm", "-r", "300", "-gray", "-f", str(first), "-l", str(last), "-png", pdf, os.path.join(tmp, "p")],
                        capture_output=True, timeout=600)
         for name in sorted(os.listdir(tmp)):
             m = re.match(r"p-0*(\d+)\.png$", name)
@@ -276,6 +276,7 @@ def detect_from_pdf(cfg, pdf, hint=None):
     pages, blank = first_pages_text(cfg, pdf, n, notes)
     page = read_title_page([(p, t) for p, t, _ in pages], hint)
     ocr_used = False
+    allpages = [(p, t) for p, t, _ in pages]
     if not page.get("author") and blank and cfg["ocr"]["enabled"]:
         if not (res["title"] and info_author):                     # nothing more to learn if metadata already gives both
             ocr = ocr_first_pages(pdf, min(blank), max(blank), cfg["ocr"]["language"])
@@ -288,12 +289,17 @@ def detect_from_pdf(cfg, pdf, hint=None):
                 ocr_used = True
     src = "first-page-ocr" if ocr_used and page.get("page") in blank else "first-page-text"
     if page.get("author"):
+        if src == "first-page-ocr":
+            notes.append("author %r was read by OCR from pdf page %s; check it. A file named like the PDF with .json (author field) overrides it" % (page["author"], page["page"]))
         res["author"] = page["author"]
         res["author_source"] = ("%s (%s, pdf page %s)" % (src, page["author_source"], page["page"]))
     elif info_author:
         res["author"], res["author_source"] = display_name(info_author), "pdf-metadata"
     if not res["title"] and page.get("title") and page.get("author"):
         res["title"], res["title_source"] = bookio.title_case(page["title"]), "%s (pdf page %s)" % (src, page["page"])
+    if page.get("page") is not None:
+        txt = dict(allpages).get(page["page"], "")
+        res["excerpt"] = " / ".join(s for _, s in page_lines(txt)[:14])[:600]
     res["_page_title"] = bookio.title_case(page["title"]) if page.get("title") else ""
     res["_page_src"] = "%s (pdf page %s)" % (src, page.get("page"))
     return res
@@ -356,5 +362,5 @@ def resolve(cfg, pdf, title=None, author=None, sidecar=None, name_hints=None):
         else:
             a, as_ = UNKNOWN_AUTHOR, "default"
             warnings.append(("metadata-author", os.path.basename(pdf), "no author found in the PDF or its file name; using 'Unknown' (folder 'unknown'). Add %s.json with {\"author\": \"...\"} next to the PDF to set it" % os.path.splitext(os.path.basename(pdf))[0]))
-    log = ["metadata: title %r from %s" % (t, ts), "metadata: author %r from %s" % (a, as_)] + ["metadata: " + n for n in det.get("notes", [])]
+    log = ["metadata: title %r from %s" % (t, ts), "metadata: author %r from %s" % (a, as_)] + ["metadata: " + n for n in det.get("notes", [])] + (["metadata: title page text: " + det["excerpt"]] if det.get("excerpt") else [])
     return {"title": t, "author": a, "title_source": ts, "author_source": as_, "warnings": warnings, "log": log}

_pipeline/structure.py

@@ -7,7 +7,7 @@ import pdftext
 
 WORDS = "ONE TWO THREE FOUR FIVE SIX SEVEN EIGHT NINE TEN ELEVEN TWELVE THIRTEEN FOURTEEN FIFTEEN SIXTEEN SEVENTEEN EIGHTEEN NINETEEN TWENTY".split()
 ROMAN = {"I": 1, "V": 5, "X": 10, "L": 50}
-HEADING_RE = re.compile(r"^\s*CHAPTER\s+([A-Z]+|\d+)\s*$")
+HEADING_RE = re.compile(r"^\s*CHAPTER\b[\s.:]*(.*?)[\s.,:;]*$")
 
 
 def chapter_number(token):
@@ -23,8 +23,31 @@ def chapter_number(token):
     return None
 
 
+def heading_number(line, expected):
+    """(number, guessed) for a 'CHAPTER <number>' line, else None. The number may be a roman numeral, digits or a word.
+    OCR often damages the numeral ('I I r', 'X V', 'X f I'): a short unreadable token after an upper-case CHAPTER is
+    taken to be the next chapter in sequence (guessed=True, and a warning is written by run())."""
+    m = HEADING_RE.match(line)
+    if not m:
+        return None
+    rest = m.group(1).strip()
+    if re.fullmatch(r"[A-Z]+|\d+", rest):
+        n = chapter_number(rest)
+        if n is not None:
+            return n, False
+    collapsed = re.sub(r"\s+", "", rest)
+    if 0 < len(rest) <= 8 and re.fullmatch(r"[A-Za-z0-9|!.,]+", collapsed):
+        n = chapter_number(collapsed.upper().strip(".,"))
+        if n is not None and not (" " in rest and n != expected):     # 'I I' is more likely III than II when III is next
+            return n, False
+        return expected, True
+    return None
+
+
 def find_headings(units):
-    """'CHAPTER ONE' then the title line(s) up to a blank line, within the first 12 lines of a unit."""
+    """'CHAPTER ONE' then the title line(s) up to a blank line, within the first 12 lines of a unit.
+    A heading whose number does not rise (a table-of-contents entry that came before the real chapter) is replaced by the
+    later one. With no heading at all the whole body is one chapter ('synthetic')."""
     found = []
     for ui, u in enumerate(units):
         lines = u["text"].split("\n")
@@ -34,10 +57,10 @@ def find_headings(units):
                 shown += 1
             if shown > 12:
                 break
-            m = HEADING_RE.match(ln)
+            got = heading_number(ln, (found[-1]["number"] + 1) if found else 1)
-            n = chapter_number(m.group(1)) if m else None
+            if got is None:
-            if n is None:
                 continue
+            n, guessed = got
             k = li + 1
             while k < len(lines) and not lines[k].strip():
                 k += 1
@@ -45,9 +68,18 @@ def find_headings(units):
             while k < len(lines) and lines[k].strip():
                 title.append(lines[k].strip())
                 k += 1
-            found.append({"unit": ui, "line": li, "title_end": k, "number": n,
+            h = {"unit": ui, "line": li, "title_end": k, "number": n, "title": bookio.title_case(" ".join(title))}
-                          "title": bookio.title_case(" ".join(title))})
+            if guessed:
+                h["guessed"] = True
+            while found and found[-1]["number"] >= n:      # the earlier one was a contents entry or a misread
+                found.pop()
+                h["replaced_earlier"] = True
+            found.append(h)
             break
+    if not found:
+        for ui, u in enumerate(units):
+            if len(re.sub(r"\s", "", u["text"])) >= 500:
+                return [{"unit": ui, "line": -1, "title_end": 0, "number": 1, "title": "The Complete Text", "synthetic": True}]
     return found
 
 
@@ -98,7 +130,14 @@ def run(book_dir, book, pdf):
     heads = find_headings(units)
     warnings = []
     if not heads:
-        raise SystemExit("no chapter headings found")
+        raise SystemExit("no chapter headings found and no page with a body of text to treat as a single chapter")
+    for h in heads:
+        if h.get("synthetic"):
+            warnings.append(("no-chapter-headings", "book", "no 'CHAPTER <number>' headings found; the text from pdf page %d on is treated as one chapter" % units[h["unit"]]["pdf_page"]))
+        if h.get("guessed"):
+            warnings.append(("chapter-number-guessed", "chapter %d" % h["number"], "the number after CHAPTER on pdf page %d was unreadable (OCR); taken as the next in sequence" % units[h["unit"]]["pdf_page"]))
+        if h.get("replaced_earlier"):
+            warnings.append(("heading-replaced", "chapter %d" % h["number"], "an earlier CHAPTER line with the same or a higher number (a contents entry or misread) was dropped in favour of the heading on pdf page %d" % units[h["unit"]]["pdf_page"]))
     pages, printed = book_pages(units, book["title"])
     toc = {n: (t, p) for n, t, p in toc_chapters(units, heads[0]["unit"])}
     old = {c["number"]: c for c in book.get("chapters", [])}

_pipeline/tests/test_meta.py

@@ -87,6 +87,8 @@ class MetaTests(unittest.TestCase):
         self.assertEqual((m["title"], m["author"]), ("Science of Breath", "Yogi Ramacharaka"))
         self.assertIn("first-page-ocr", m["author_source"])
         self.assertIn("first-page-ocr", m["title_source"])
+        self.assertTrue(any("read by OCR" in w[2] for w in m["warnings"]))
+        self.assertTrue(any("title page text:" in line for line in m["log"]))
         meta.resolve(self.cfg, p)
         self.assertEqual(len(calls), 1, "the detection from the PDF is cached")
 

_pipeline/tests/test_structure.py

@@ -0,0 +1,56 @@
+"""Chapter heading detection, including damaged OCR headings, contents pages and books without chapters."""
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+
+import structure
+
+
+def unit(text, page=1):
+    return {"text": text, "pdf_page": page}
+
+
+BODY = ("A line of ordinary body text that goes on for a while. " * 12)
+
+
+class HeadingTests(unittest.TestCase):
+    def numbers(self, units):
+        return [(h["number"], h["title"]) for h in structure.find_headings(units)]
+
+    def test_clean_headings_still_work(self):
+        units = [unit("CHAPTER ONE\nThe Start\n\n" + BODY), unit(BODY), unit("CHAPTER TWO\n\nThe Middle\n\n" + BODY), unit("CHAPTER 3\nEnd\n\n" + BODY)]
+        self.assertEqual(self.numbers(units), [(1, "The Start"), (2, "The Middle"), (3, "End")])
+
+    def test_ocr_damaged_numerals_are_taken_in_sequence(self):
+        units = [unit("CHAPTER I.\nBreath\n\n" + BODY), unit("CHAPTER II.\nTheory\n\n" + BODY),
+                 unit("CHAPTER I I r.\nMore\n\n" + BODY), unit("CHAPTER IV.\nFour\n\n" + BODY),
+                 unit("CHAPTER X V\nFifteen\n\n" + BODY)]
+        got = structure.find_headings(units)
+        self.assertEqual([h["number"] for h in got], [1, 2, 3, 4, 5])        # "X V" is not 15 here: 5 is next, so it was a gap in the numerals
+        self.assertEqual([h.get("guessed", False) for h in got], [False, False, True, False, True])
+
+    def test_contents_entry_is_replaced_by_the_real_heading(self):
+        units = [unit("CONTENTS\nCHAPTER I.\nBreath\nCHAPTER II.\nTheory\n"), unit("CHAPTER I.\nBreath\n\n" + BODY), unit("CHAPTER II.\nTheory\n\n" + BODY)]
+        got = structure.find_headings(units)
+        self.assertEqual([(h["number"], h["unit"]) for h in got], [(1, 1), (2, 2)])
+        self.assertTrue(got[0]["replaced_earlier"])
+
+    def test_sentences_starting_with_chapter_are_not_headings(self):
+        units = [unit("CHAPTER ONE\nStart\n\n" + BODY), unit("CHAPTER of the book is long and winding, said the guide.\n" + BODY)]
+        self.assertEqual(len(structure.find_headings(units)), 1)
+
+    def test_no_headings_gives_one_synthetic_chapter_from_the_first_body_page(self):
+        units = [unit("Title\nBy Someone"), unit("Short"), unit(BODY, 3), unit(BODY, 4)]
+        got = structure.find_headings(units)
+        self.assertEqual(len(got), 1)
+        self.assertTrue(got[0]["synthetic"])
+        self.assertEqual((got[0]["unit"], got[0]["number"]), (2, 1))
+
+    def test_nothing_readable_gives_no_headings(self):
+        self.assertEqual(structure.find_headings([unit(""), unit("x")]), [])
+
+
+if __name__ == "__main__":
+    unittest.main()