diff --git a/_pipeline/structure.py b/_pipeline/structure.py index 0c99f5a..adc2d06 100644 --- a/_pipeline/structure.py +++ b/_pipeline/structure.py @@ -92,8 +92,11 @@ def find_headings(units): found.append(h) break if not found: - for ui, u in enumerate(units): - if len(re.sub(r"\s", "", u["text"])) >= 500: + sizes = [len(re.sub(r"\s", "", u["text"])) for u in units] + filled = sorted(n for n in sizes if n >= 20) + floor = max(40, 0.4 * filled[len(filled) // 2]) if filled else 0 # a page with 40% of the median text or more is body + for ui, n in enumerate(sizes): + if filled and n >= floor: return [{"unit": ui, "line": -1, "title_end": 0, "number": 1, "title": "The Complete Text", "synthetic": True}] return found diff --git a/_pipeline/tests/test_structure.py b/_pipeline/tests/test_structure.py index 6a40e10..5a7d6f3 100644 --- a/_pipeline/tests/test_structure.py +++ b/_pipeline/tests/test_structure.py @@ -72,6 +72,12 @@ class HeadingTests(unittest.TestCase): self.assertEqual(structure.unknown_words("The Four Metiiods of Respiration"), 1) self.assertEqual(structure.unknown_words("The Four Methods of Respiration"), 0) + def test_short_book_without_headings_still_gets_a_chapter(self): + speech = "Four score and seven years ago our fathers brought forth on this continent a new nation. " * 3 + units = [unit("THE ADDRESS"), unit(""), unit("DELIVERED BY A. LINCOLN\nAT THE DEDICATION"), unit(speech, 4), unit(speech, 5), unit(speech, 6)] + got = structure.find_headings(units) + self.assertEqual((len(got), got[0]["unit"]), (1, 3)) + def test_nothing_readable_gives_no_headings(self): self.assertEqual(structure.find_headings([unit(""), unit("x")]), [])