Structure: tolerate OCR-damaged CHAPTER numerals, drop contents-page entries, one synthetic chapter when a book has no headings; metadata: OCR at 300 dpi, title-page text logged, OCR-read author flagged

_pipeline/tests/test_meta.py

@@ -87,6 +87,8 @@ class MetaTests(unittest.TestCase):
         self.assertEqual((m["title"], m["author"]), ("Science of Breath", "Yogi Ramacharaka"))
         self.assertIn("first-page-ocr", m["author_source"])
         self.assertIn("first-page-ocr", m["title_source"])
+        self.assertTrue(any("read by OCR" in w[2] for w in m["warnings"]))
+        self.assertTrue(any("title page text:" in line for line in m["log"]))
         meta.resolve(self.cfg, p)
         self.assertEqual(len(calls), 1, "the detection from the PDF is cached")
 

_pipeline/tests/test_structure.py

@@ -0,0 +1,56 @@
+"""Chapter heading detection, including damaged OCR headings, contents pages and books without chapters."""
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+
+import structure
+
+
+def unit(text, page=1):
+    return {"text": text, "pdf_page": page}
+
+
+BODY = ("A line of ordinary body text that goes on for a while. " * 12)
+
+
+class HeadingTests(unittest.TestCase):
+    def numbers(self, units):
+        return [(h["number"], h["title"]) for h in structure.find_headings(units)]
+
+    def test_clean_headings_still_work(self):
+        units = [unit("CHAPTER ONE\nThe Start\n\n" + BODY), unit(BODY), unit("CHAPTER TWO\n\nThe Middle\n\n" + BODY), unit("CHAPTER 3\nEnd\n\n" + BODY)]
+        self.assertEqual(self.numbers(units), [(1, "The Start"), (2, "The Middle"), (3, "End")])
+
+    def test_ocr_damaged_numerals_are_taken_in_sequence(self):
+        units = [unit("CHAPTER I.\nBreath\n\n" + BODY), unit("CHAPTER II.\nTheory\n\n" + BODY),
+                 unit("CHAPTER I I r.\nMore\n\n" + BODY), unit("CHAPTER IV.\nFour\n\n" + BODY),
+                 unit("CHAPTER X V\nFifteen\n\n" + BODY)]
+        got = structure.find_headings(units)
+        self.assertEqual([h["number"] for h in got], [1, 2, 3, 4, 5])        # "X V" is not 15 here: 5 is next, so it was a gap in the numerals
+        self.assertEqual([h.get("guessed", False) for h in got], [False, False, True, False, True])
+
+    def test_contents_entry_is_replaced_by_the_real_heading(self):
+        units = [unit("CONTENTS\nCHAPTER I.\nBreath\nCHAPTER II.\nTheory\n"), unit("CHAPTER I.\nBreath\n\n" + BODY), unit("CHAPTER II.\nTheory\n\n" + BODY)]
+        got = structure.find_headings(units)
+        self.assertEqual([(h["number"], h["unit"]) for h in got], [(1, 1), (2, 2)])
+        self.assertTrue(got[0]["replaced_earlier"])
+
+    def test_sentences_starting_with_chapter_are_not_headings(self):
+        units = [unit("CHAPTER ONE\nStart\n\n" + BODY), unit("CHAPTER of the book is long and winding, said the guide.\n" + BODY)]
+        self.assertEqual(len(structure.find_headings(units)), 1)
+
+    def test_no_headings_gives_one_synthetic_chapter_from_the_first_body_page(self):
+        units = [unit("Title\nBy Someone"), unit("Short"), unit(BODY, 3), unit(BODY, 4)]
+        got = structure.find_headings(units)
+        self.assertEqual(len(got), 1)
+        self.assertTrue(got[0]["synthetic"])
+        self.assertEqual((got[0]["unit"], got[0]["number"]), (2, 1))
+
+    def test_nothing_readable_gives_no_headings(self):
+        self.assertEqual(structure.find_headings([unit(""), unit("x")]), [])
+
+
+if __name__ == "__main__":
+    unittest.main()