Structure: no-heading fallback also works for short books (body threshold relative to the median page)

_pipeline/structure.py

@@ -92,8 +92,11 @@ def find_headings(units):
             found.append(h)
             break
     if not found:
-        for ui, u in enumerate(units):
+        sizes = [len(re.sub(r"\s", "", u["text"])) for u in units]
-            if len(re.sub(r"\s", "", u["text"])) >= 500:
+        filled = sorted(n for n in sizes if n >= 20)
+        floor = max(40, 0.4 * filled[len(filled) // 2]) if filled else 0      # a page with 40% of the median text or more is body
+        for ui, n in enumerate(sizes):
+            if filled and n >= floor:
                 return [{"unit": ui, "line": -1, "title_end": 0, "number": 1, "title": "The Complete Text", "synthetic": True}]
     return found
 

_pipeline/tests/test_structure.py

@@ -72,6 +72,12 @@ class HeadingTests(unittest.TestCase):
         self.assertEqual(structure.unknown_words("The Four Metiiods of Respiration"), 1)
         self.assertEqual(structure.unknown_words("The Four Methods of Respiration"), 0)
 
+    def test_short_book_without_headings_still_gets_a_chapter(self):
+        speech = "Four score and seven years ago our fathers brought forth on this continent a new nation. " * 3
+        units = [unit("THE ADDRESS"), unit(""), unit("DELIVERED BY A. LINCOLN\nAT THE DEDICATION"), unit(speech, 4), unit(speech, 5), unit(speech, 6)]
+        got = structure.find_headings(units)
+        self.assertEqual((len(got), got[0]["unit"]), (1, 3))
+
     def test_nothing_readable_gives_no_headings(self):
         self.assertEqual(structure.find_headings([unit(""), unit("x")]), [])