Fix paragraph breaks, drop caps, headers and captions for chapters 2-6 of the Clark book
Root cause: clean-up found paragraph breaks from the layout indent of pdftotext -layout
(columns, relative to the smallest indent in the whole chapter). Chapter 1 happened to
have a clean left margin; later pages are skewed and drift 1-2 columns, drop caps push
the first lines right by varying amounts, and some pages wrap text around a label, so
continuation lines were taken as paragraph starts (a blank line inside a sentence).
Fix (all deterministic, all logged to warnings.log):
- pdftext: read each line's position in points (pdftotext -bbox-layout) and attach it to
the layout line it belongs to (units() now carries 'geom'; merged/partial lines get none).
- cleanup: a paragraph starts when a line sits >= 4.5 pt right of the line before it, or
>= 2.5 pt lower than the page's normal line pitch. A line without a position or starting
in lower case never starts a paragraph. Text wrapped beside a drop cap is not a start.
Without positions the old indent rule is still used.
- cleanup: drop cap + small caps on ONE layout line ('F OR MANY YEARS') is repaired like
the two-line form (chapters 4 and 5).
- pageparts.header_zone: the running header is found even when text follows it without a
blank line, and the right-hand header (chapter title, page number sometimes misread as
't5' or 'a') is recognised from the chapter title.
- cleanup: all-capitals lines below the text are captions (removed); an all-capitals line
above the text is kept as a section heading in title case ('The Deferred Preface');
'Right: MARK TWIN' style captions removed; '5. JOY' labels set inside a sentence removed.
- cleanup: paragraph starting with a lone quote mark + lower-case word joins the paragraph
before it; paragraph-leading ' + capital becomes '"' (not elisions such as 'Tis); a bullet
in place of an opening quote becomes '"'; closing quote with no opening quote, and a '1'
that is probably the letter I, are logged only.
- cleanup: mid-line 'worth- while' joined only when certain, otherwise logged.
- tests: new _pipeline/tests/test_cleanup.py (36 tests, 27 fail on the previous code);
test_pipeline.py helper test_piece renamed make_test_piece so pytest stops collecting it.
Chapter 1 text is byte-identical to the approved pilot text.
Re-run for chapters 1-6: cleanup, figures, script, license. Text, script, warnings and
book.json for chapters 2-6 added.
This commit is contained in:
Audiobook Producer
+164
-11
@@ -75,9 +75,10 @@ def smallcaps_lead(text):
|
||||
def unit_lines(unit, title, heading, first_unit, notes):
|
||||
"""Body lines of one unit as dicts {text, indent, where}; removed parts are noted."""
|
||||
raw = [l.rstrip() for l in unit["text"].split("\n")]
|
||||
geom = unit.get("geom") or [None] * len(raw)
|
||||
where = "pdf p.%d%s" % (unit["pdf_page"], unit["half"] or "")
|
||||
drop = set()
|
||||
z = pageparts.header_zone(raw, title)
|
||||
z = pageparts.header_zone(raw, title, [heading["title"]] if heading else [])
|
||||
if z:
|
||||
drop.update(range(z[0], z[1]))
|
||||
what = "page number %d" % z[2] if z[2] is not None else "no page number"
|
||||
@@ -89,6 +90,9 @@ def unit_lines(unit, title, heading, first_unit, notes):
|
||||
if first_unit and heading:
|
||||
drop.update(range(0, heading["title_end"]))
|
||||
notes.append(("heading-removed", where, "chapter heading 'CHAPTER %d / %s' (spoken header is added in the script)" % (heading["number"], heading["title"])))
|
||||
# last line of running text: the lines after it (all capitals only) are picture captions, not headings
|
||||
body = [i for i, l in enumerate(raw) if i not in drop and any(c.islower() for c in l)]
|
||||
last_body = body[-1] if body else -1
|
||||
out = []
|
||||
for i, ln in enumerate(raw):
|
||||
if i in drop or not ln.strip():
|
||||
@@ -96,17 +100,87 @@ def unit_lines(unit, title, heading, first_unit, notes):
|
||||
if pageparts.CAPTION_RE.match(ln):
|
||||
notes.append(("caption-removed", where, "figure caption '%s' (listed in figures/skipped_figures.md)" % ln.strip()))
|
||||
continue
|
||||
out.append({"text": ln.strip(), "indent": len(ln) - len(ln.lstrip()), "where": where})
|
||||
if CAPTION_MIXED_RE.match(ln):
|
||||
notes.append(("caption-removed", where, "figure caption '%s' (marker not in capitals)" % ln.strip()))
|
||||
continue
|
||||
letters = [c for c in ln if c.isalpha()]
|
||||
if len(letters) >= 3 and not any(c.islower() for c in letters):
|
||||
if i > last_body:
|
||||
notes.append(("caption-removed", where, "all-capitals line '%s' below the text (picture caption or label)" % ln.strip()))
|
||||
continue
|
||||
spoken = bookio.title_case(" ".join(ln.split()))
|
||||
notes.append(("heading-kept", where, "all-capitals line above the text kept as a section heading: '%s' -> '%s'" % (ln.strip(), spoken)))
|
||||
ln = " " * (len(ln) - len(ln.lstrip())) + spoken
|
||||
g = geom[i] if i < len(geom) else None
|
||||
out.append({"text": ln.strip(), "indent": len(ln) - len(ln.lstrip()), "where": where,
|
||||
"unit": (unit["pdf_page"], unit["half"]), "x": g["x"] if g else None, "y": g["y"] if g else None})
|
||||
return out
|
||||
|
||||
|
||||
INDENT_PT = 4.5 # a paragraph's first line sits at least this many points right of the line before it
|
||||
GAP_PT = 2.5 # or this many points lower than the page's normal line spacing
|
||||
SAMELINE_CAP = re.compile(r"^([A-Z])\s{2,}([A-Z]{2,})\b")
|
||||
|
||||
|
||||
def unit_stats(lines):
|
||||
"""Per page unit: normal line pitch and the local left margin, read from line positions in points."""
|
||||
stats = {}
|
||||
for u in {l["unit"] for l in lines}:
|
||||
mine = [l for l in lines if l["unit"] == u and l["x"] is not None]
|
||||
gaps = sorted(b["y"] - a["y"] for a, b in zip(mine, mine[1:]) if 8 < b["y"] - a["y"] < 20)
|
||||
stats[u] = {"pitch": gaps[(len(gaps) - 1) // 2] if gaps else 13.0, "lines": mine}
|
||||
return stats
|
||||
|
||||
|
||||
def local_margin(stats, ln):
|
||||
"""Left margin near the first line of a unit: the leftmost of its next few lines (the page may be skewed)."""
|
||||
near = [l["x"] for l in stats[ln["unit"]]["lines"] if l is not ln and 0 < l["y"] - ln["y"] < 90]
|
||||
return min(near) if near else ln["x"]
|
||||
|
||||
|
||||
def starts_paragraph(prev, ln, stats):
|
||||
"""True if the line opens a paragraph. Evidence is geometric: indented relative to the line before it, or
|
||||
set lower than normal line spacing. A line with no position, or one that begins in lower case, never starts one."""
|
||||
if ln["x"] is None or ln["text"][:1].islower():
|
||||
return False
|
||||
if prev is not None and prev["unit"] == ln["unit"] and prev["x"] is not None:
|
||||
return ln["x"] - prev["x"] >= INDENT_PT or ln["y"] - prev["y"] >= stats[ln["unit"]]["pitch"] + GAP_PT
|
||||
return ln["x"] - local_margin(stats, ln) >= INDENT_PT
|
||||
|
||||
|
||||
def paragraphs_from(lines, inline, notes):
|
||||
"""Group lines into paragraphs by first-line indent, repair drop caps and hyphenation."""
|
||||
"""Group lines into paragraphs and repair drop caps. With line positions (the normal case) a paragraph break is
|
||||
an indent or a vertical gap; without them (older callers) it is the layout indent of the first line."""
|
||||
geometric = any(l.get("x") is not None for l in lines)
|
||||
stats = unit_stats(lines) if geometric else None
|
||||
base = min(l["indent"] for l in lines)
|
||||
i, paras, cur, cap_indent = 0, [], None, None
|
||||
i, paras, cur, cap_indent, wrap, prev = 0, [], None, None, False, None
|
||||
while i < len(lines):
|
||||
ln = dict(lines[i])
|
||||
is_cap = split_dropcap(lines, i)
|
||||
same = SAMELINE_CAP.match(ln["text"]) if not is_cap else None
|
||||
if geometric:
|
||||
if wrap and ln["x"] is not None and ln["x"] - local_margin(stats, ln) >= INDENT_PT and not same:
|
||||
is_start = False # still beside the drop cap
|
||||
else:
|
||||
wrap = False
|
||||
is_start = cur is None or starts_paragraph(prev, ln, stats)
|
||||
else:
|
||||
is_start = ln["indent"] - base >= 1 and not (cap_indent is not None and ln["indent"] == cap_indent)
|
||||
if same and (cur is None or is_start):
|
||||
letter, rest = same.group(1), ln["text"][same.end(1):].strip()
|
||||
first = same.group(2)
|
||||
if known(letter + first) or not (letter in "AI" and known(first)):
|
||||
merged, how = letter + rest, "joined"
|
||||
else:
|
||||
merged, how = letter + " " + rest, "kept as its own word"
|
||||
notes.append(("dropcap", ln["where"], "drop cap '%s' + '%s' on one line: %s" % (letter, rest[:12], how)))
|
||||
ln.update(text=merged, indent=base + 1, smallcaps=True)
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
wrap, prev = True, lines[i]
|
||||
i += 1
|
||||
continue
|
||||
if is_cap:
|
||||
letter, frag = ln["text"], lines[i + 1]
|
||||
first = re.match(r"[A-Z]+", frag["text"]).group(0)
|
||||
@@ -117,20 +191,21 @@ def paragraphs_from(lines, inline, notes):
|
||||
merged = letter + " " + frag["text"]
|
||||
how = "kept as its own word"
|
||||
notes.append(("dropcap", ln["where"], "drop cap '%s' + '%s': %s" % (letter, frag["text"][:12], how)))
|
||||
ln = {"text": merged, "indent": base + 1, "where": ln["where"], "smallcaps": True}
|
||||
ln = dict(frag, text=merged, indent=base + 1, where=ln["where"], smallcaps=True)
|
||||
cap_indent = frag["indent"]
|
||||
i += 2
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
wrap, prev = True, frag
|
||||
continue
|
||||
starts = ln["indent"] - base >= 1 and not (cap_indent is not None and ln["indent"] == cap_indent)
|
||||
if ln["indent"] != cap_indent:
|
||||
cap_indent = None
|
||||
if starts or cur is None:
|
||||
if is_start or cur is None:
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
else:
|
||||
cur.append(ln)
|
||||
prev = lines[i]
|
||||
i += 1
|
||||
return paras
|
||||
|
||||
@@ -151,11 +226,83 @@ def join_lines(para, inline, notes):
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
ELISION = {"Tis", "Twas", "Twere", "Twill", "Twould"}
|
||||
CAPTION_MIXED_RE = re.compile(r"^\s*(?:Left|Right|Above|Below|Opposite|Facing)\s*:\s*[A-Z][A-Z .,'\-]+\s*$")
|
||||
MIDLINE_HYPHEN_RE = re.compile(r"\b([A-Za-z]+)- ([a-z]+)\b")
|
||||
LABEL_RE = re.compile(r"\s\d\.\s+[A-Z]{3,}(?:\s+[A-Z]{3,})*(?=\s+[a-z])")
|
||||
DIGIT_ONE_RE = re.compile(r"(?<![\d.,$#/-])\b1 (?=[a-z]+\b)")
|
||||
|
||||
|
||||
def strip_inline_labels(text, where, notes):
|
||||
"""The book sets '5. JOY' style labels into the middle of a sentence (a typeset inset, not words of the sentence).
|
||||
Left in, they would be read aloud inside the sentence, so they are removed and logged."""
|
||||
def drop(m):
|
||||
notes.append(("inline-label-removed", where, "typeset label '%s' sat inside a sentence" % m.group(0).strip()))
|
||||
return ""
|
||||
return LABEL_RE.sub(drop, text)
|
||||
|
||||
|
||||
def fix_midline_hyphens(text, where, notes, inline):
|
||||
"""'worth- while' inside a line: the scan kept a line-end hyphen after the line was re-flowed.
|
||||
Joined only when the answer is certain (the hyphenated form occurs elsewhere in the book, or the joined
|
||||
word is a dictionary word). Otherwise logged and left as it is."""
|
||||
def fix(m):
|
||||
joined, reason, ok = join_hyphen(m.group(1), m.group(2), inline)
|
||||
certain = ok and (reason.startswith("hyphenated form") or reason.startswith("joined word"))
|
||||
notes.append(("hyphen-joined" if certain else "hyphen-unresolved", where,
|
||||
"mid-line '%s- %s' -> %s (%s)" % (m.group(1), m.group(2), "'%s'" % joined if certain else "not changed", reason)))
|
||||
return joined if certain else m.group(0)
|
||||
return MIDLINE_HYPHEN_RE.sub(fix, text)
|
||||
|
||||
|
||||
def merge_orphan_quotes(paras, chapter, notes):
|
||||
"""[(text, where)] -> same, where a paragraph that opens with a lone quote mark and a lower-case word is
|
||||
joined to the paragraph before it. No paragraph starts like that: the page broke the line after an opening
|
||||
quoted word, as in: They "milked' / " every operation. The stray ' closing the quoted word is made a '"'."""
|
||||
out = []
|
||||
for text, where in paras:
|
||||
if out and re.match(r'"\s+[a-z]', text):
|
||||
prev, prev_where = out[-1]
|
||||
fixed = prev
|
||||
if prev.count('"') % 2 == 1 and re.search(r"[A-Za-z]'$", prev):
|
||||
fixed = prev[:-1] + '"'
|
||||
out[-1] = (fixed + " " + text[1:].strip(), prev_where)
|
||||
notes.append(("paragraph-joined", where, "paragraph began with a lone quote mark and a lower-case word, "
|
||||
"so it continues the paragraph before it ('...%s' + '%s...')" % (prev[-14:], text[:14])))
|
||||
else:
|
||||
out.append((text, where))
|
||||
return out
|
||||
|
||||
|
||||
def fix_quotes(text, where, notes):
|
||||
"""A paragraph with an unclosed double quote that ends in punctuation + ' : the ' is a misread closing quote."""
|
||||
"""Quote marks the scan got wrong. Fixed only when the rule is certain; otherwise the paragraph is logged.
|
||||
- an unclosed double quote that ends in punctuation + ' : the ' is a misread closing quote.
|
||||
- a paragraph that opens with ' + capital letter (not an elision such as 'Tis): a misread opening
|
||||
double quote, as at the start of each paragraph of a long speech.
|
||||
- a closing double quote (after punctuation) with no opening quote before it in the paragraph is logged only."""
|
||||
if text.count('"') % 2 == 1 and re.search(r"[.?!]'$", text):
|
||||
notes.append(("quote-fixed", where, "closing quote misread as apostrophe at '...%s'" % text[-12:]))
|
||||
return text[:-1] + '"'
|
||||
text = text[:-1] + '"'
|
||||
if re.match(r"\u2022[A-Z]", text):
|
||||
notes.append(("quote-fixed", where, "opening quote misread as a bullet at '%s...'" % text[:16]))
|
||||
text = '"' + text[1:]
|
||||
m = re.match(r"'([A-Z][a-z]*)", text)
|
||||
if m and m.group(1) not in ELISION:
|
||||
notes.append(("quote-fixed", where, "opening quote misread as apostrophe at '%s...'" % text[:16]))
|
||||
text = '"' + text[1:]
|
||||
opened = False
|
||||
for q in re.finditer('"', text):
|
||||
before = text[q.start() - 1] if q.start() else ""
|
||||
if before in ("", " ", "(", "\u2014"):
|
||||
opened = True
|
||||
elif before == "'":
|
||||
opened = not opened
|
||||
elif opened:
|
||||
opened = False
|
||||
else:
|
||||
notes.append(("quote-unmatched", where, "closing quote with no opening quote before it: '...%s' (not changed)" % text[max(0, q.start() - 14):q.start() + 1]))
|
||||
for m in DIGIT_ONE_RE.finditer(text):
|
||||
notes.append(("digit-one", where, "'1' before a lowercase word, probably the letter I misread: '...%s' (not changed)" % text[max(0, m.start() - 8):m.end() + 8]))
|
||||
return text
|
||||
|
||||
|
||||
@@ -180,14 +327,20 @@ def run(book_dir, book, pdf, chapter_numbers):
|
||||
for ui in range(ch["first_unit"], ch["last_unit"] + 1):
|
||||
lines += unit_lines(units[ui], book["title"], heads[ch["number"]], ui == ch["first_unit"], notes)
|
||||
paras = paragraphs_from(lines, inline, notes)
|
||||
texts = []
|
||||
for n, p in enumerate(paras, start=1):
|
||||
joined = []
|
||||
for p in paras:
|
||||
t = join_lines(p, inline, notes)
|
||||
if p[0].get("smallcaps"):
|
||||
before, t = t, smallcaps_lead(t)
|
||||
if before != t:
|
||||
notes.append(("smallcaps", p[0]["where"], "'%s...' -> '%s...'" % (before[:14], t[:14])))
|
||||
joined.append((t, p[0]["where"]))
|
||||
joined = merge_orphan_quotes(joined, ch["number"], notes)
|
||||
texts = []
|
||||
for n, (t, _) in enumerate(joined, start=1):
|
||||
where = "ch%d para %d" % (ch["number"], n)
|
||||
t = strip_inline_labels(t, where, notes)
|
||||
t = fix_midline_hyphens(t, where, notes, inline)
|
||||
t = fix_quotes(t, where, notes)
|
||||
odd_words(t, where, notes)
|
||||
texts.append(t)
|
||||
|
||||
+19
-12
@@ -16,20 +16,28 @@ def similar(a, b):
|
||||
return SequenceMatcher(None, norm(a), norm(b)).ratio()
|
||||
|
||||
|
||||
def header_zone(lines, title):
|
||||
def _mostly_upper(text):
|
||||
letters = [c for c in text if c.isalpha()]
|
||||
return (not letters) or sum(c.isupper() for c in letters) / len(letters) > 0.7
|
||||
|
||||
|
||||
def header_zone(lines, title, extra_titles=()):
|
||||
"""The running header at the top of a unit.
|
||||
Returns (first_line, last_line_exclusive, page_number or None) or None.
|
||||
A header is the first block of lines (up to a blank line) that either carries a page number
|
||||
next to uppercase text, or reads as the book title."""
|
||||
The header starts at the first non-empty line and runs over the following lines while they are
|
||||
upper case (at most 3 lines in all), whether or not a blank line separates it from the text.
|
||||
It is a header if it carries a page number next to uppercase text, or reads as the book title or as
|
||||
the chapter title (extra_titles: the right-hand page repeats the chapter title, and its page number
|
||||
may be misread by the scan as a letter, so no number is found)."""
|
||||
i = 0
|
||||
while i < len(lines) and not lines[i].strip():
|
||||
i += 1
|
||||
j = i
|
||||
while j < len(lines) and lines[j].strip():
|
||||
if i >= len(lines):
|
||||
return None
|
||||
j = i + 1
|
||||
while j < len(lines) and lines[j].strip() and j - i < 3 and _mostly_upper(lines[j]):
|
||||
j += 1
|
||||
zone = lines[i:j]
|
||||
if not zone:
|
||||
return None
|
||||
number, rest = None, []
|
||||
for ln in zone:
|
||||
m = _NUM_LEAD.match(ln) or None
|
||||
@@ -46,12 +54,11 @@ def header_zone(lines, title):
|
||||
continue
|
||||
rest.append(ln)
|
||||
text = " ".join(rest)
|
||||
letters = [c for c in text if c.isalpha()]
|
||||
mostly_upper = (not letters) or sum(c.isupper() for c in letters) / len(letters) > 0.7
|
||||
if number is not None and mostly_upper and len(zone) <= 3:
|
||||
if number is not None and _mostly_upper(text) and len(zone) <= 3:
|
||||
return i, j, number
|
||||
if title and similar(text, title) >= 0.9:
|
||||
return i, j, None
|
||||
for t in (title,) + tuple(extra_titles):
|
||||
if t and similar(text, t) >= 0.9:
|
||||
return i, j, None
|
||||
return None
|
||||
|
||||
|
||||
|
||||
+49
-2
@@ -4,6 +4,7 @@ import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from html import unescape
|
||||
|
||||
|
||||
def _run(cmd):
|
||||
@@ -42,14 +43,60 @@ def chars_per_page(pdf):
|
||||
return out
|
||||
|
||||
|
||||
_LINE_RE = re.compile(r'<line xMin="([\d.]+)" yMin="([\d.]+)" xMax="([\d.]+)" yMax="([\d.]+)">(.*?)</line>', re.S)
|
||||
|
||||
|
||||
def page_lines(pdf, page):
|
||||
"""Text lines of one PDF page with their position in points, top to bottom:
|
||||
[{'x', 'y', 'x2', 'y2', 'text'}]. Coordinates are those of the displayed page (poppler bbox-layout)."""
|
||||
xml = _run(["pdftotext", "-bbox-layout", "-f", str(page), "-l", str(page), pdf, "-"])
|
||||
out = []
|
||||
for m in _LINE_RE.finditer(xml):
|
||||
words = [unescape(w) for w in re.findall(r">([^<]+)</word>", m.group(5))]
|
||||
out.append({"x": float(m.group(1)), "y": float(m.group(2)), "x2": float(m.group(3)),
|
||||
"y2": float(m.group(4)), "text": " ".join(words)})
|
||||
return sorted(out, key=lambda ln: (ln["y"], ln["x"]))
|
||||
|
||||
|
||||
def _key(text):
|
||||
return re.sub(r"[^A-Za-z0-9]", "", text)
|
||||
|
||||
|
||||
def align(raw_lines, boxes, window=8):
|
||||
"""Give each line of a layout text (pdftotext -layout) the position of the same line from page_lines.
|
||||
Returns a list parallel to raw_lines: the box dict, or None for blank lines and lines that cannot be
|
||||
matched one to one (merged columns, small-caps headers). A line that only partly matches a box is left
|
||||
as None: its position would be wrong."""
|
||||
result = [None] * len(raw_lines)
|
||||
j = 0
|
||||
for i, line in enumerate(raw_lines):
|
||||
k = _key(line)
|
||||
if not k:
|
||||
continue
|
||||
for jj in range(j, min(j + window, len(boxes))):
|
||||
b = _key(boxes[jj]["text"])
|
||||
if b == k:
|
||||
result[i] = boxes[jj]
|
||||
j = jj + 1
|
||||
break
|
||||
if len(b) >= 8 and (k.endswith(b) or k.startswith(b)):
|
||||
j = jj + 1
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def units(pdf):
|
||||
"""Reading units in order: [{'pdf_page', 'half', 'text'}]. Text keeps the layout (indentation, hyphens)."""
|
||||
"""Reading units in order: [{'pdf_page', 'half', 'text', 'geom'}]. Text keeps the layout (indentation, hyphens).
|
||||
'geom' is parallel to text.split('\\n'): the position of each line on the page (see align)."""
|
||||
result = []
|
||||
for p, (w, h) in enumerate(page_geometry(pdf), start=1):
|
||||
boxes = page_lines(pdf, p)
|
||||
for half, x, width in halves(w, h):
|
||||
text = _run(["pdftotext", "-layout", "-f", str(p), "-l", str(p),
|
||||
"-x", str(int(x)), "-y", "0", "-W", str(int(width)), "-H", str(int(h)), pdf, "-"])
|
||||
result.append({"pdf_page": p, "half": half, "text": text.replace("\f", "")})
|
||||
text = text.replace("\f", "")
|
||||
mine = [b for b in boxes if x <= b["x"] < x + width] if half else boxes
|
||||
result.append({"pdf_page": p, "half": half, "text": text, "geom": align(text.split("\n"), mine)})
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Tests for the text stages: paragraph detection from line positions, drop caps, headers, captions, quotes, hyphens.
|
||||
Run: python3 -m unittest discover -s _pipeline/tests -v (from the repo root), or python3 -m pytest _pipeline/tests"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
|
||||
import bookio
|
||||
import cleanup
|
||||
import pageparts
|
||||
import pdftext
|
||||
import structure
|
||||
|
||||
REPO = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||
BOOK = os.path.join(REPO, "books", "clark-glenn", "the-man-who-tapped-the-secrets-of-the-universe")
|
||||
PDF = os.path.join(BOOK, "The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf")
|
||||
|
||||
|
||||
def line(text, x=None, y=None, unit=(1, "L"), indent=0):
|
||||
return {"text": text, "indent": indent, "where": "t", "unit": unit, "x": x, "y": y}
|
||||
|
||||
|
||||
def page(rows, unit=(1, "L")):
|
||||
"""rows: [(text, x, y)] -> line dicts of one page unit."""
|
||||
return [line(t, x, y, unit) for t, x, y in rows]
|
||||
|
||||
|
||||
def paras(lines):
|
||||
notes = []
|
||||
out = cleanup.paragraphs_from(lines, set(), notes)
|
||||
return [[l["text"] for l in p] for p in out], notes
|
||||
|
||||
|
||||
class ParagraphGeometryTests(unittest.TestCase):
|
||||
def test_indent_starts_a_paragraph_on_a_skewed_page(self):
|
||||
# The scan drifts left down the page (43.0 -> 38.6) and the first line of a paragraph sits ~9 pt right.
|
||||
rows = [("First paragraph line one", 43.0, 100), ("line two continues", 41.5, 113), ("and ends here.", 41.5, 126),
|
||||
("Second paragraph starts", 50.5, 147), ("and continues", 40.1, 160), ("on this line.", 40.1, 173),
|
||||
("Third paragraph begins", 47.8, 191), ("and ends.", 38.6, 204)]
|
||||
got, _ = paras(page(rows))
|
||||
self.assertEqual([len(p) for p in got], [3, 3, 2])
|
||||
|
||||
def test_continuation_line_one_column_right_is_not_a_break(self):
|
||||
# Layout indent of the old rule was 0 then 1: a 1-column wobble (about 2 pt) must not split a paragraph.
|
||||
rows = [("Para one a", 40.0, 100), ("para one b", 40.0, 113), ("para one c", 42.0, 126), ("para one d", 42.0, 139)]
|
||||
got, _ = paras(page(rows))
|
||||
self.assertEqual(len(got), 1)
|
||||
|
||||
def test_vertical_gap_starts_a_paragraph_without_indent(self):
|
||||
rows = [("End of the last paragraph.", 40.0, 100), ("A new paragraph with no indent", 40.5, 100 + 13 + 4.5),
|
||||
("continues here.", 40.5, 130.5)]
|
||||
got, _ = paras(page(rows))
|
||||
self.assertEqual(len(got), 2)
|
||||
|
||||
def test_lower_case_line_never_starts_a_paragraph(self):
|
||||
rows = [("Text line one", 40.0, 100), ("ecstasy. wrapped beside a label", 52.0, 113), ("more text", 40.0, 126)]
|
||||
got, _ = paras(page(rows))
|
||||
self.assertEqual(len(got), 1)
|
||||
|
||||
def test_line_without_position_is_a_continuation(self):
|
||||
rows = [("Text line one", 40.0, 100), ("5. JOY qualities merged line", None, None), ("Next line", 40.0, 126)]
|
||||
got, _ = paras(page(rows))
|
||||
self.assertEqual(len(got), 1)
|
||||
|
||||
def test_first_line_of_next_page_continues_or_starts_by_indent(self):
|
||||
p1 = page([("one", 40.0, 100), ("two", 40.0, 113), ("three, cut by the page end", 40.0, 126)], (1, "L"))
|
||||
cont = page([("continues the sentence", 40.5, 100), ("next line", 40.0, 113), ("and another", 40.0, 126)], (1, "R"))
|
||||
got, _ = paras(p1 + cont)
|
||||
self.assertEqual(len(got), 1)
|
||||
new = page([("Starts a paragraph", 49.5, 100), ("next line", 40.0, 113), ("and another", 40.0, 126)], (1, "R"))
|
||||
got, _ = paras(p1 + new)
|
||||
self.assertEqual(len(got), 2)
|
||||
|
||||
def test_same_line_drop_cap_and_its_wrapped_lines(self):
|
||||
# 'F OR MANY YEARS' then three lines pushed right by the drop cap, one of them starting with a capital.
|
||||
rows = [("F OR MANY YEARS he lectured upon", 40.6, 100), ("self-multiplication of the", 68.0, 113),
|
||||
("Business to the officers", 68.0, 126), ("Machines Corporation, in the effort", 42.0, 139),
|
||||
("Second paragraph", 51.0, 160), ("goes on.", 40.0, 173)]
|
||||
notes = []
|
||||
out = cleanup.paragraphs_from(page(rows), set(), notes)
|
||||
self.assertEqual(len(out), 2)
|
||||
self.assertEqual(len(out[0]), 4)
|
||||
self.assertTrue(out[0][0]["smallcaps"])
|
||||
self.assertTrue(out[0][0]["text"].startswith("FOR MANY YEARS"))
|
||||
self.assertEqual(cleanup.smallcaps_lead(out[0][0]["text"]), "For many years he lectured upon")
|
||||
self.assertEqual(notes[0][0], "dropcap")
|
||||
|
||||
def test_drop_cap_on_its_own_line(self):
|
||||
rows = [("I", 38.2, 90), ("KNEW by the cut of the Van Dyke beard", 60.6, 92), ("an artist. I could tell by the", 56.2, 105),
|
||||
("eyes that I was talking to", 56.2, 118), ("of speaking revealed in him", 38.2, 131), ("Next paragraph", 47.0, 152)]
|
||||
got, _ = paras(page(rows))
|
||||
self.assertEqual(len(got), 2)
|
||||
self.assertEqual(got[0][0], "I KNEW by the cut of the Van Dyke beard")
|
||||
|
||||
def test_old_rule_still_used_without_positions(self):
|
||||
lines = [line("Alpha", indent=0), line("beta", indent=0), line("Gamma", indent=2), line("delta", indent=0)]
|
||||
got, _ = paras(lines)
|
||||
self.assertEqual(len(got), 2)
|
||||
|
||||
|
||||
class AlignTests(unittest.TestCase):
|
||||
def test_align_matches_body_lines_and_skips_unmatched(self):
|
||||
raw = ["2 THE MAN WHO TAPPED", "", " In the field of religion I had", "found several such men."]
|
||||
boxes = [{"text": "2", "x": 1, "y": 1, "x2": 2, "y2": 2}, {"text": "T HE M AN", "x": 9, "y": 1, "x2": 20, "y2": 2},
|
||||
{"text": "In the field of religion I had", "x": 50.0, "y": 30, "x2": 300, "y2": 40},
|
||||
{"text": "found several such men.", "x": 41.0, "y": 43, "x2": 300, "y2": 53}]
|
||||
got = pdftext.align(raw, boxes)
|
||||
self.assertIsNone(got[0])
|
||||
self.assertIsNone(got[1])
|
||||
self.assertEqual(got[2]["x"], 50.0)
|
||||
self.assertEqual(got[3]["x"], 41.0)
|
||||
|
||||
def test_partial_match_gets_no_position(self):
|
||||
raw = ["5. JOY qualities of your consciousness which we know by"]
|
||||
boxes = [{"text": "qualities of your consciousness which we know by", "x": 112.0, "y": 378, "x2": 300, "y2": 388}]
|
||||
self.assertEqual(pdftext.align(raw, boxes), [None])
|
||||
|
||||
|
||||
class HeaderTests(unittest.TestCase):
|
||||
TITLE = "The Man Who Tapped the Secrets of the Universe"
|
||||
|
||||
def test_header_directly_followed_by_text_without_blank_line(self):
|
||||
lines = [" 12 THE MAN WHO TAPPED THE SECRETS OFTHE UNIVERSE", " These secrets which Nature whispered to him", " have not only given him"]
|
||||
self.assertEqual(pageparts.header_zone(lines, self.TITLE), (0, 1, 12))
|
||||
|
||||
def test_two_line_header(self):
|
||||
lines = ["2 THE MAN WHO TAPPED THE SECRETS OF THE", " UNIVERSE", "", " In the field"]
|
||||
self.assertEqual(pageparts.header_zone(lines, self.TITLE), (0, 2, 2))
|
||||
|
||||
def test_chapter_title_header_with_misread_page_number(self):
|
||||
lines = [" WE MEET THE ARTIST t5", "", "", " Music, for example, financed"]
|
||||
self.assertIsNone(pageparts.header_zone(lines, self.TITLE))
|
||||
self.assertEqual(pageparts.header_zone(lines, self.TITLE, ["We Meet the Artist"]), (0, 1, None))
|
||||
lines[0] = " WE MEET THE ARTIST a"
|
||||
self.assertEqual(pageparts.header_zone(lines, self.TITLE, ["We Meet the Artist"]), (0, 1, None))
|
||||
|
||||
def test_body_text_is_not_a_header(self):
|
||||
lines = [" He was born in Boston, Massachusetts, May 19th, 1871. He at-", "tended a village school until nearly ten, when"]
|
||||
self.assertIsNone(pageparts.header_zone(lines, self.TITLE, ["We Meet the Artist"]))
|
||||
|
||||
|
||||
class UnitLinesTests(unittest.TestCase):
|
||||
def unit(self, text):
|
||||
return {"pdf_page": 9, "half": "R", "text": text, "geom": None}
|
||||
|
||||
def run_unit(self, text):
|
||||
notes = []
|
||||
out = cleanup.unit_lines(self.unit(text), "Book Title", None, False, notes)
|
||||
return [l["text"] for l in out], notes
|
||||
|
||||
def test_captions_below_the_text_are_removed(self):
|
||||
text = " Some running text here.\nmore running text.\n\nPAGE 10: FRANKLIN DELANO ROOSEVELT\n BEFORE PEARL HARBOR\n"
|
||||
got, notes = self.run_unit(text)
|
||||
self.assertEqual(got, ["Some running text here.", "more running text."])
|
||||
self.assertEqual([n[0] for n in notes], ["caption-removed", "caption-removed"])
|
||||
|
||||
def test_all_caps_heading_above_the_text_is_kept_in_title_case(self):
|
||||
text = " THE DEFERRED PREFACE\n\n Now that you have finished reading this book,\nyou must begin again.\n"
|
||||
got, notes = self.run_unit(text)
|
||||
self.assertEqual(got[0], "The Deferred Preface")
|
||||
self.assertEqual(notes[0][0], "heading-kept")
|
||||
|
||||
def test_caption_marker_not_in_capitals(self):
|
||||
got, notes = self.run_unit("Text line of the book.\n\nRight: MARK TWIN\n")
|
||||
self.assertEqual(got, ["Text line of the book."])
|
||||
self.assertEqual(notes[0][0], "caption-removed")
|
||||
|
||||
|
||||
class QuoteAndHyphenTests(unittest.TestCase):
|
||||
def test_opening_quote_misread_as_apostrophe_at_paragraph_start(self):
|
||||
notes = []
|
||||
out = cleanup.fix_quotes("'The underlying principle of Balance is equality.", "w", notes)
|
||||
self.assertEqual(out, '"The underlying principle of Balance is equality.')
|
||||
self.assertEqual(notes[0][0], "quote-fixed")
|
||||
|
||||
def test_elision_is_not_an_opening_quote(self):
|
||||
notes = []
|
||||
self.assertEqual(cleanup.fix_quotes("'Tis the season.", "w", notes), "'Tis the season.")
|
||||
self.assertEqual(notes, [])
|
||||
|
||||
def test_bullet_misread_as_opening_quote(self):
|
||||
notes = []
|
||||
self.assertEqual(cleanup.fix_quotes("\u2022That is exactly what J am planning,\" he replied.", "w", notes),
|
||||
'"That is exactly what J am planning," he replied.')
|
||||
|
||||
def test_closing_quote_without_opening_is_logged_not_changed(self):
|
||||
notes = []
|
||||
text = 'When I am in this Studio I feel like Alice in Wonderland," I remarked, when we were together again."'
|
||||
self.assertEqual(cleanup.fix_quotes(text, "w", notes), text)
|
||||
self.assertEqual([n[0] for n in notes], ["quote-unmatched", "quote-unmatched"])
|
||||
|
||||
def test_balanced_speech_is_silent(self):
|
||||
notes = []
|
||||
text = '"Most sculptors make the mistake," he said, "of thinking of eyes as form."'
|
||||
self.assertEqual(cleanup.fix_quotes(text, "w", notes), text)
|
||||
self.assertEqual(notes, [])
|
||||
|
||||
def test_digit_one_is_logged_not_changed(self):
|
||||
notes = []
|
||||
text = '"What do you mean by beauty?" 1 asked.'
|
||||
self.assertEqual(cleanup.fix_quotes(text, "w", notes), text)
|
||||
self.assertEqual(notes[0][0], "digit-one")
|
||||
|
||||
def test_orphan_quote_paragraph_is_joined_to_the_one_before(self):
|
||||
notes = []
|
||||
paras_in = [('They looked at the product. They "milked\'', "a"), ('" every operation for themselves.', "b"), ("Next.", "c")]
|
||||
out = cleanup.merge_orphan_quotes(paras_in, 3, notes)
|
||||
self.assertEqual([t for t, _ in out], ['They looked at the product. They "milked" every operation for themselves.', "Next."])
|
||||
self.assertEqual(notes[0][0], "paragraph-joined")
|
||||
|
||||
def test_a_paragraph_that_starts_a_quote_normally_is_not_joined(self):
|
||||
out = cleanup.merge_orphan_quotes([("One.", "a"), ('"Two," he said.', "b"), ('" Three, capital', "c")], 1, [])
|
||||
self.assertEqual(len(out), 3)
|
||||
|
||||
def test_inline_label_removed_only_inside_a_sentence(self):
|
||||
notes = []
|
||||
text = "No one can multiply himself by 1. HUMILITY himself. He must divide himself and use it in 4. DEEP PURPOSE the direction."
|
||||
self.assertEqual(cleanup.strip_inline_labels(text, "w", notes),
|
||||
"No one can multiply himself by himself. He must divide himself and use it in the direction.")
|
||||
self.assertEqual(len(notes), 2)
|
||||
keep = "In 1946 he had 3. Some sentence."
|
||||
self.assertEqual(cleanup.strip_inline_labels(keep, "w", []), keep)
|
||||
|
||||
def test_midline_hyphen_joined_only_when_certain(self):
|
||||
notes = []
|
||||
inline = {"thought-world"}
|
||||
out = cleanup.fix_midline_hyphens("only worth- while, the thought- world, and superficially- minded", "w", notes, inline)
|
||||
self.assertEqual(out, "only worthwhile, the thought-world, and superficially- minded")
|
||||
self.assertEqual([n[0] for n in notes], ["hyphen-joined", "hyphen-joined", "hyphen-unresolved"])
|
||||
|
||||
def test_line_end_hyphen_is_joined_inside_a_paragraph(self):
|
||||
para = [{"text": "I feel like Alice in Won-", "where": "t"}, {"text": 'derland," I remarked.', "where": "t"}]
|
||||
self.assertEqual(cleanup.join_lines(para, set(), []), 'I feel like Alice in Wonderland," I remarked.')
|
||||
|
||||
|
||||
@unittest.skipUnless(os.path.exists(PDF), "Clark PDF not in this checkout")
|
||||
class ClarkBookTests(unittest.TestCase):
|
||||
"""Runs the real clean-up on chapters 1-6 and checks the numbers that were wrong before."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
book = bookio.read_book(BOOK)
|
||||
cls.book = book
|
||||
cls.notes = cleanup.run(cls.tmp.name, book, PDF, [1, 2, 3, 4, 5, 6])
|
||||
cls.text = {}
|
||||
for n in range(1, 7):
|
||||
with open(os.path.join(cls.tmp.name, "text", "chapter-%02d.txt" % n), encoding="utf-8") as f:
|
||||
cls.text[n] = f.read()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def paragraphs(self, n):
|
||||
return [p.strip() for p in self.text[n].split("\n\n") if p.strip()]
|
||||
|
||||
def test_chapter_one_is_unchanged(self):
|
||||
with open(os.path.join(BOOK, "text", "chapter-01.txt"), encoding="utf-8") as f:
|
||||
self.assertEqual(self.text[1], f.read())
|
||||
|
||||
def test_no_paragraph_break_inside_a_sentence(self):
|
||||
for n in range(1, 7):
|
||||
for p in self.paragraphs(n):
|
||||
self.assertNotIn("\n", p)
|
||||
for a, b in zip(self.paragraphs(n), self.paragraphs(n)[1:]):
|
||||
self.assertFalse(re.search(r"[A-Za-z,]$", a) and re.match(r"[a-z]", b), (n, a[-30:], b[:30]))
|
||||
|
||||
def test_paragraphs_are_long_enough(self):
|
||||
for n in range(1, 7):
|
||||
words = [len(p.split()) for p in self.paragraphs(n)]
|
||||
self.assertGreater(sum(words) / len(words), 40, "chapter %d" % n)
|
||||
|
||||
def test_no_line_ends_in_a_hyphen(self):
|
||||
for n in range(1, 7):
|
||||
self.assertFalse([p for p in self.paragraphs(n) if p.endswith("-")], "chapter %d" % n)
|
||||
left = len(re.findall(r"[A-Za-z]- [a-z]", "".join(self.text.values())))
|
||||
logged = len([k for k, _, m in self.notes if k == "hyphen-unresolved" and m.startswith("mid-line")])
|
||||
self.assertEqual(left, logged) # the only 'word- word' left are the ones logged as unresolved
|
||||
|
||||
def test_drop_caps_are_repaired(self):
|
||||
self.assertTrue(self.text[4].startswith("For many years he lectured"))
|
||||
self.assertTrue(self.text[5].startswith("When I am in this Studio"))
|
||||
for n in range(1, 7):
|
||||
for p in self.paragraphs(n):
|
||||
self.assertNotRegex(p[:12], r"^[A-Z] [A-Z]{2,}")
|
||||
|
||||
def test_no_running_headers_page_numbers_or_captions(self):
|
||||
for n in range(1, 7):
|
||||
self.assertNotRegex(self.text[n], r"THE MAN WHO|WE MEET THE|FIVE LAWS OF|VISION FROM THE|PAGE ?\d+:|Right: |LEFT:|\u2022")
|
||||
|
||||
def test_every_change_is_logged(self):
|
||||
kinds = {k for k, _, _ in self.notes}
|
||||
for k in ("dropcap", "smallcaps", "hyphen-joined", "header-removed", "caption-removed", "quote-fixed", "paragraph-joined"):
|
||||
self.assertIn(k, kinds)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -20,7 +20,7 @@ import voice
|
||||
CFG = settings.load()
|
||||
|
||||
|
||||
def test_piece(path, hz, seconds, level):
|
||||
def make_test_piece(path, hz, seconds, level):
|
||||
"""TEST signal: speech-like (harmonics, 4 Hz syllable rhythm, a little noise) at amplitude `level`."""
|
||||
expr = "%g*(0.6*sin(2*PI*%g*t)+0.3*sin(4*PI*%g*t)+0.1*sin(6*PI*%g*t))*(0.55+0.45*sin(2*PI*4*t))" % (level, hz, hz, hz)
|
||||
am.ffmpeg(["-f", "lavfi", "-i", "aevalsrc='%s':s=24000:d=%g" % (expr, seconds), "-ar", "24000", "-ac", "1", path])
|
||||
@@ -32,7 +32,7 @@ class AudioTests(unittest.TestCase):
|
||||
items = []
|
||||
for i, (hz, lvl) in enumerate([(120, 0.02), (150, 0.05), (130, 0.03)]):
|
||||
p = os.path.join(d, "t%d.wav" % i)
|
||||
test_piece(p, hz, 4, lvl)
|
||||
make_test_piece(p, hz, 4, lvl)
|
||||
items.append({"wav": p, "gap_after_s": 0.7})
|
||||
if i == 0:
|
||||
items.append({"cue": "chime", "gap_after_s": 0.25})
|
||||
@@ -46,7 +46,7 @@ class AudioTests(unittest.TestCase):
|
||||
def test_checker_fails_bad_files(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
raw = os.path.join(d, "raw.wav")
|
||||
test_piece(raw, 130, 4, 1.4) # hot test signal, clips at full scale
|
||||
make_test_piece(raw, 130, 4, 1.4) # hot test signal, clips at full scale
|
||||
bad = os.path.join(d, "bad.mp3") # stereo, 128k, 22.05 kHz, no room tone
|
||||
am.ffmpeg(["-i", raw, "-ac", "2", "-ar", "22050", "-c:a", "libmp3lame", "-b:a", "128k", bad])
|
||||
failed = {n for n, v, p in acx_check.judge(CFG, bad, acx_check.measure(CFG, bad)) if not p}
|
||||
@@ -70,7 +70,7 @@ class StubHandler(http.server.BaseHTTPRequestHandler):
|
||||
if self.path.endswith("/audio/speech") or self.path == "/tts":
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = os.path.join(d, "x.wav")
|
||||
test_piece(p, 130, 1, 0.1)
|
||||
make_test_piece(p, 130, 1, 0.1)
|
||||
data = open(p, "rb").read()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "audio/wav")
|
||||
|
||||
Reference in New Issue
Block a user