Audiobook Producer
7f0b598441
Root cause: clean-up found paragraph breaks from the layout indent of pdftotext -layout
(columns, relative to the smallest indent in the whole chapter). Chapter 1 happened to
have a clean left margin; later pages are skewed and drift 1-2 columns, drop caps push
the first lines right by varying amounts, and some pages wrap text around a label, so
continuation lines were taken as paragraph starts (a blank line inside a sentence).
Fix (all deterministic, all logged to warnings.log):
- pdftext: read each line's position in points (pdftotext -bbox-layout) and attach it to
the layout line it belongs to (units() now carries 'geom'; merged/partial lines get none).
- cleanup: a paragraph starts when a line sits >= 4.5 pt right of the line before it, or
>= 2.5 pt lower than the page's normal line pitch. A line without a position or starting
in lower case never starts a paragraph. Text wrapped beside a drop cap is not a start.
Without positions the old indent rule is still used.
- cleanup: drop cap + small caps on ONE layout line ('F OR MANY YEARS') is repaired like
the two-line form (chapters 4 and 5).
- pageparts.header_zone: the running header is found even when text follows it without a
blank line, and the right-hand header (chapter title, page number sometimes misread as
't5' or 'a') is recognised from the chapter title.
- cleanup: all-capitals lines below the text are captions (removed); an all-capitals line
above the text is kept as a section heading in title case ('The Deferred Preface');
'Right: MARK TWIN' style captions removed; '5. JOY' labels set inside a sentence removed.
- cleanup: paragraph starting with a lone quote mark + lower-case word joins the paragraph
before it; paragraph-leading ' + capital becomes '"' (not elisions such as 'Tis); a bullet
in place of an opening quote becomes '"'; closing quote with no opening quote, and a '1'
that is probably the letter I, are logged only.
- cleanup: mid-line 'worth- while' joined only when certain, otherwise logged.
- tests: new _pipeline/tests/test_cleanup.py (36 tests, 27 fail on the previous code);
test_pipeline.py helper test_piece renamed make_test_piece so pytest stops collecting it.
Chapter 1 text is byte-identical to the approved pilot text.
Re-run for chapters 1-6: cleanup, figures, script, license. Text, script, warnings and
book.json for chapters 2-6 added.
304 lines
15 KiB
Python
304 lines
15 KiB
Python
"""Tests for the text stages: paragraph detection from line positions, drop caps, headers, captions, quotes, hyphens.
|
|
Run: python3 -m unittest discover -s _pipeline/tests -v (from the repo root), or python3 -m pytest _pipeline/tests"""
|
|
import os
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, os.path.join(HERE, ".."))
|
|
|
|
import bookio
|
|
import cleanup
|
|
import pageparts
|
|
import pdftext
|
|
import structure
|
|
|
|
REPO = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
BOOK = os.path.join(REPO, "books", "clark-glenn", "the-man-who-tapped-the-secrets-of-the-universe")
|
|
PDF = os.path.join(BOOK, "The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf")
|
|
|
|
|
|
def line(text, x=None, y=None, unit=(1, "L"), indent=0):
|
|
return {"text": text, "indent": indent, "where": "t", "unit": unit, "x": x, "y": y}
|
|
|
|
|
|
def page(rows, unit=(1, "L")):
|
|
"""rows: [(text, x, y)] -> line dicts of one page unit."""
|
|
return [line(t, x, y, unit) for t, x, y in rows]
|
|
|
|
|
|
def paras(lines):
|
|
notes = []
|
|
out = cleanup.paragraphs_from(lines, set(), notes)
|
|
return [[l["text"] for l in p] for p in out], notes
|
|
|
|
|
|
class ParagraphGeometryTests(unittest.TestCase):
|
|
def test_indent_starts_a_paragraph_on_a_skewed_page(self):
|
|
# The scan drifts left down the page (43.0 -> 38.6) and the first line of a paragraph sits ~9 pt right.
|
|
rows = [("First paragraph line one", 43.0, 100), ("line two continues", 41.5, 113), ("and ends here.", 41.5, 126),
|
|
("Second paragraph starts", 50.5, 147), ("and continues", 40.1, 160), ("on this line.", 40.1, 173),
|
|
("Third paragraph begins", 47.8, 191), ("and ends.", 38.6, 204)]
|
|
got, _ = paras(page(rows))
|
|
self.assertEqual([len(p) for p in got], [3, 3, 2])
|
|
|
|
def test_continuation_line_one_column_right_is_not_a_break(self):
|
|
# Layout indent of the old rule was 0 then 1: a 1-column wobble (about 2 pt) must not split a paragraph.
|
|
rows = [("Para one a", 40.0, 100), ("para one b", 40.0, 113), ("para one c", 42.0, 126), ("para one d", 42.0, 139)]
|
|
got, _ = paras(page(rows))
|
|
self.assertEqual(len(got), 1)
|
|
|
|
def test_vertical_gap_starts_a_paragraph_without_indent(self):
|
|
rows = [("End of the last paragraph.", 40.0, 100), ("A new paragraph with no indent", 40.5, 100 + 13 + 4.5),
|
|
("continues here.", 40.5, 130.5)]
|
|
got, _ = paras(page(rows))
|
|
self.assertEqual(len(got), 2)
|
|
|
|
def test_lower_case_line_never_starts_a_paragraph(self):
|
|
rows = [("Text line one", 40.0, 100), ("ecstasy. wrapped beside a label", 52.0, 113), ("more text", 40.0, 126)]
|
|
got, _ = paras(page(rows))
|
|
self.assertEqual(len(got), 1)
|
|
|
|
def test_line_without_position_is_a_continuation(self):
|
|
rows = [("Text line one", 40.0, 100), ("5. JOY qualities merged line", None, None), ("Next line", 40.0, 126)]
|
|
got, _ = paras(page(rows))
|
|
self.assertEqual(len(got), 1)
|
|
|
|
def test_first_line_of_next_page_continues_or_starts_by_indent(self):
|
|
p1 = page([("one", 40.0, 100), ("two", 40.0, 113), ("three, cut by the page end", 40.0, 126)], (1, "L"))
|
|
cont = page([("continues the sentence", 40.5, 100), ("next line", 40.0, 113), ("and another", 40.0, 126)], (1, "R"))
|
|
got, _ = paras(p1 + cont)
|
|
self.assertEqual(len(got), 1)
|
|
new = page([("Starts a paragraph", 49.5, 100), ("next line", 40.0, 113), ("and another", 40.0, 126)], (1, "R"))
|
|
got, _ = paras(p1 + new)
|
|
self.assertEqual(len(got), 2)
|
|
|
|
def test_same_line_drop_cap_and_its_wrapped_lines(self):
|
|
# 'F OR MANY YEARS' then three lines pushed right by the drop cap, one of them starting with a capital.
|
|
rows = [("F OR MANY YEARS he lectured upon", 40.6, 100), ("self-multiplication of the", 68.0, 113),
|
|
("Business to the officers", 68.0, 126), ("Machines Corporation, in the effort", 42.0, 139),
|
|
("Second paragraph", 51.0, 160), ("goes on.", 40.0, 173)]
|
|
notes = []
|
|
out = cleanup.paragraphs_from(page(rows), set(), notes)
|
|
self.assertEqual(len(out), 2)
|
|
self.assertEqual(len(out[0]), 4)
|
|
self.assertTrue(out[0][0]["smallcaps"])
|
|
self.assertTrue(out[0][0]["text"].startswith("FOR MANY YEARS"))
|
|
self.assertEqual(cleanup.smallcaps_lead(out[0][0]["text"]), "For many years he lectured upon")
|
|
self.assertEqual(notes[0][0], "dropcap")
|
|
|
|
def test_drop_cap_on_its_own_line(self):
|
|
rows = [("I", 38.2, 90), ("KNEW by the cut of the Van Dyke beard", 60.6, 92), ("an artist. I could tell by the", 56.2, 105),
|
|
("eyes that I was talking to", 56.2, 118), ("of speaking revealed in him", 38.2, 131), ("Next paragraph", 47.0, 152)]
|
|
got, _ = paras(page(rows))
|
|
self.assertEqual(len(got), 2)
|
|
self.assertEqual(got[0][0], "I KNEW by the cut of the Van Dyke beard")
|
|
|
|
def test_old_rule_still_used_without_positions(self):
|
|
lines = [line("Alpha", indent=0), line("beta", indent=0), line("Gamma", indent=2), line("delta", indent=0)]
|
|
got, _ = paras(lines)
|
|
self.assertEqual(len(got), 2)
|
|
|
|
|
|
class AlignTests(unittest.TestCase):
|
|
def test_align_matches_body_lines_and_skips_unmatched(self):
|
|
raw = ["2 THE MAN WHO TAPPED", "", " In the field of religion I had", "found several such men."]
|
|
boxes = [{"text": "2", "x": 1, "y": 1, "x2": 2, "y2": 2}, {"text": "T HE M AN", "x": 9, "y": 1, "x2": 20, "y2": 2},
|
|
{"text": "In the field of religion I had", "x": 50.0, "y": 30, "x2": 300, "y2": 40},
|
|
{"text": "found several such men.", "x": 41.0, "y": 43, "x2": 300, "y2": 53}]
|
|
got = pdftext.align(raw, boxes)
|
|
self.assertIsNone(got[0])
|
|
self.assertIsNone(got[1])
|
|
self.assertEqual(got[2]["x"], 50.0)
|
|
self.assertEqual(got[3]["x"], 41.0)
|
|
|
|
def test_partial_match_gets_no_position(self):
|
|
raw = ["5. JOY qualities of your consciousness which we know by"]
|
|
boxes = [{"text": "qualities of your consciousness which we know by", "x": 112.0, "y": 378, "x2": 300, "y2": 388}]
|
|
self.assertEqual(pdftext.align(raw, boxes), [None])
|
|
|
|
|
|
class HeaderTests(unittest.TestCase):
|
|
TITLE = "The Man Who Tapped the Secrets of the Universe"
|
|
|
|
def test_header_directly_followed_by_text_without_blank_line(self):
|
|
lines = [" 12 THE MAN WHO TAPPED THE SECRETS OFTHE UNIVERSE", " These secrets which Nature whispered to him", " have not only given him"]
|
|
self.assertEqual(pageparts.header_zone(lines, self.TITLE), (0, 1, 12))
|
|
|
|
def test_two_line_header(self):
|
|
lines = ["2 THE MAN WHO TAPPED THE SECRETS OF THE", " UNIVERSE", "", " In the field"]
|
|
self.assertEqual(pageparts.header_zone(lines, self.TITLE), (0, 2, 2))
|
|
|
|
def test_chapter_title_header_with_misread_page_number(self):
|
|
lines = [" WE MEET THE ARTIST t5", "", "", " Music, for example, financed"]
|
|
self.assertIsNone(pageparts.header_zone(lines, self.TITLE))
|
|
self.assertEqual(pageparts.header_zone(lines, self.TITLE, ["We Meet the Artist"]), (0, 1, None))
|
|
lines[0] = " WE MEET THE ARTIST a"
|
|
self.assertEqual(pageparts.header_zone(lines, self.TITLE, ["We Meet the Artist"]), (0, 1, None))
|
|
|
|
def test_body_text_is_not_a_header(self):
|
|
lines = [" He was born in Boston, Massachusetts, May 19th, 1871. He at-", "tended a village school until nearly ten, when"]
|
|
self.assertIsNone(pageparts.header_zone(lines, self.TITLE, ["We Meet the Artist"]))
|
|
|
|
|
|
class UnitLinesTests(unittest.TestCase):
|
|
def unit(self, text):
|
|
return {"pdf_page": 9, "half": "R", "text": text, "geom": None}
|
|
|
|
def run_unit(self, text):
|
|
notes = []
|
|
out = cleanup.unit_lines(self.unit(text), "Book Title", None, False, notes)
|
|
return [l["text"] for l in out], notes
|
|
|
|
def test_captions_below_the_text_are_removed(self):
|
|
text = " Some running text here.\nmore running text.\n\nPAGE 10: FRANKLIN DELANO ROOSEVELT\n BEFORE PEARL HARBOR\n"
|
|
got, notes = self.run_unit(text)
|
|
self.assertEqual(got, ["Some running text here.", "more running text."])
|
|
self.assertEqual([n[0] for n in notes], ["caption-removed", "caption-removed"])
|
|
|
|
def test_all_caps_heading_above_the_text_is_kept_in_title_case(self):
|
|
text = " THE DEFERRED PREFACE\n\n Now that you have finished reading this book,\nyou must begin again.\n"
|
|
got, notes = self.run_unit(text)
|
|
self.assertEqual(got[0], "The Deferred Preface")
|
|
self.assertEqual(notes[0][0], "heading-kept")
|
|
|
|
def test_caption_marker_not_in_capitals(self):
|
|
got, notes = self.run_unit("Text line of the book.\n\nRight: MARK TWIN\n")
|
|
self.assertEqual(got, ["Text line of the book."])
|
|
self.assertEqual(notes[0][0], "caption-removed")
|
|
|
|
|
|
class QuoteAndHyphenTests(unittest.TestCase):
|
|
def test_opening_quote_misread_as_apostrophe_at_paragraph_start(self):
|
|
notes = []
|
|
out = cleanup.fix_quotes("'The underlying principle of Balance is equality.", "w", notes)
|
|
self.assertEqual(out, '"The underlying principle of Balance is equality.')
|
|
self.assertEqual(notes[0][0], "quote-fixed")
|
|
|
|
def test_elision_is_not_an_opening_quote(self):
|
|
notes = []
|
|
self.assertEqual(cleanup.fix_quotes("'Tis the season.", "w", notes), "'Tis the season.")
|
|
self.assertEqual(notes, [])
|
|
|
|
def test_bullet_misread_as_opening_quote(self):
|
|
notes = []
|
|
self.assertEqual(cleanup.fix_quotes("\u2022That is exactly what J am planning,\" he replied.", "w", notes),
|
|
'"That is exactly what J am planning," he replied.')
|
|
|
|
def test_closing_quote_without_opening_is_logged_not_changed(self):
|
|
notes = []
|
|
text = 'When I am in this Studio I feel like Alice in Wonderland," I remarked, when we were together again."'
|
|
self.assertEqual(cleanup.fix_quotes(text, "w", notes), text)
|
|
self.assertEqual([n[0] for n in notes], ["quote-unmatched", "quote-unmatched"])
|
|
|
|
def test_balanced_speech_is_silent(self):
|
|
notes = []
|
|
text = '"Most sculptors make the mistake," he said, "of thinking of eyes as form."'
|
|
self.assertEqual(cleanup.fix_quotes(text, "w", notes), text)
|
|
self.assertEqual(notes, [])
|
|
|
|
def test_digit_one_is_logged_not_changed(self):
|
|
notes = []
|
|
text = '"What do you mean by beauty?" 1 asked.'
|
|
self.assertEqual(cleanup.fix_quotes(text, "w", notes), text)
|
|
self.assertEqual(notes[0][0], "digit-one")
|
|
|
|
def test_orphan_quote_paragraph_is_joined_to_the_one_before(self):
|
|
notes = []
|
|
paras_in = [('They looked at the product. They "milked\'', "a"), ('" every operation for themselves.', "b"), ("Next.", "c")]
|
|
out = cleanup.merge_orphan_quotes(paras_in, 3, notes)
|
|
self.assertEqual([t for t, _ in out], ['They looked at the product. They "milked" every operation for themselves.', "Next."])
|
|
self.assertEqual(notes[0][0], "paragraph-joined")
|
|
|
|
def test_a_paragraph_that_starts_a_quote_normally_is_not_joined(self):
|
|
out = cleanup.merge_orphan_quotes([("One.", "a"), ('"Two," he said.', "b"), ('" Three, capital', "c")], 1, [])
|
|
self.assertEqual(len(out), 3)
|
|
|
|
def test_inline_label_removed_only_inside_a_sentence(self):
|
|
notes = []
|
|
text = "No one can multiply himself by 1. HUMILITY himself. He must divide himself and use it in 4. DEEP PURPOSE the direction."
|
|
self.assertEqual(cleanup.strip_inline_labels(text, "w", notes),
|
|
"No one can multiply himself by himself. He must divide himself and use it in the direction.")
|
|
self.assertEqual(len(notes), 2)
|
|
keep = "In 1946 he had 3. Some sentence."
|
|
self.assertEqual(cleanup.strip_inline_labels(keep, "w", []), keep)
|
|
|
|
def test_midline_hyphen_joined_only_when_certain(self):
|
|
notes = []
|
|
inline = {"thought-world"}
|
|
out = cleanup.fix_midline_hyphens("only worth- while, the thought- world, and superficially- minded", "w", notes, inline)
|
|
self.assertEqual(out, "only worthwhile, the thought-world, and superficially- minded")
|
|
self.assertEqual([n[0] for n in notes], ["hyphen-joined", "hyphen-joined", "hyphen-unresolved"])
|
|
|
|
def test_line_end_hyphen_is_joined_inside_a_paragraph(self):
|
|
para = [{"text": "I feel like Alice in Won-", "where": "t"}, {"text": 'derland," I remarked.', "where": "t"}]
|
|
self.assertEqual(cleanup.join_lines(para, set(), []), 'I feel like Alice in Wonderland," I remarked.')
|
|
|
|
|
|
@unittest.skipUnless(os.path.exists(PDF), "Clark PDF not in this checkout")
|
|
class ClarkBookTests(unittest.TestCase):
|
|
"""Runs the real clean-up on chapters 1-6 and checks the numbers that were wrong before."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.tmp = tempfile.TemporaryDirectory()
|
|
book = bookio.read_book(BOOK)
|
|
cls.book = book
|
|
cls.notes = cleanup.run(cls.tmp.name, book, PDF, [1, 2, 3, 4, 5, 6])
|
|
cls.text = {}
|
|
for n in range(1, 7):
|
|
with open(os.path.join(cls.tmp.name, "text", "chapter-%02d.txt" % n), encoding="utf-8") as f:
|
|
cls.text[n] = f.read()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.tmp.cleanup()
|
|
|
|
def paragraphs(self, n):
|
|
return [p.strip() for p in self.text[n].split("\n\n") if p.strip()]
|
|
|
|
def test_chapter_one_is_unchanged(self):
|
|
with open(os.path.join(BOOK, "text", "chapter-01.txt"), encoding="utf-8") as f:
|
|
self.assertEqual(self.text[1], f.read())
|
|
|
|
def test_no_paragraph_break_inside_a_sentence(self):
|
|
for n in range(1, 7):
|
|
for p in self.paragraphs(n):
|
|
self.assertNotIn("\n", p)
|
|
for a, b in zip(self.paragraphs(n), self.paragraphs(n)[1:]):
|
|
self.assertFalse(re.search(r"[A-Za-z,]$", a) and re.match(r"[a-z]", b), (n, a[-30:], b[:30]))
|
|
|
|
def test_paragraphs_are_long_enough(self):
|
|
for n in range(1, 7):
|
|
words = [len(p.split()) for p in self.paragraphs(n)]
|
|
self.assertGreater(sum(words) / len(words), 40, "chapter %d" % n)
|
|
|
|
def test_no_line_ends_in_a_hyphen(self):
|
|
for n in range(1, 7):
|
|
self.assertFalse([p for p in self.paragraphs(n) if p.endswith("-")], "chapter %d" % n)
|
|
left = len(re.findall(r"[A-Za-z]- [a-z]", "".join(self.text.values())))
|
|
logged = len([k for k, _, m in self.notes if k == "hyphen-unresolved" and m.startswith("mid-line")])
|
|
self.assertEqual(left, logged) # the only 'word- word' left are the ones logged as unresolved
|
|
|
|
def test_drop_caps_are_repaired(self):
|
|
self.assertTrue(self.text[4].startswith("For many years he lectured"))
|
|
self.assertTrue(self.text[5].startswith("When I am in this Studio"))
|
|
for n in range(1, 7):
|
|
for p in self.paragraphs(n):
|
|
self.assertNotRegex(p[:12], r"^[A-Z] [A-Z]{2,}")
|
|
|
|
def test_no_running_headers_page_numbers_or_captions(self):
|
|
for n in range(1, 7):
|
|
self.assertNotRegex(self.text[n], r"THE MAN WHO|WE MEET THE|FIVE LAWS OF|VISION FROM THE|PAGE ?\d+:|Right: |LEFT:|\u2022")
|
|
|
|
def test_every_change_is_logged(self):
|
|
kinds = {k for k, _, _ in self.notes}
|
|
for k in ("dropcap", "smallcaps", "hyphen-joined", "header-removed", "caption-removed", "quote-fixed", "paragraph-joined"):
|
|
self.assertIn(k, kinds)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|