Fix paragraph breaks, drop caps, headers and captions for chapters 2-6 of the Clark book
Root cause: clean-up found paragraph breaks from the layout indent of pdftotext -layout
(columns, relative to the smallest indent in the whole chapter). Chapter 1 happened to
have a clean left margin; later pages are skewed and drift 1-2 columns, drop caps push
the first lines right by varying amounts, and some pages wrap text around a label, so
continuation lines were taken as paragraph starts (a blank line inside a sentence).
Fix (all deterministic, all logged to warnings.log):
- pdftext: read each line's position in points (pdftotext -bbox-layout) and attach it to
the layout line it belongs to (units() now carries 'geom'; merged/partial lines get none).
- cleanup: a paragraph starts when a line sits >= 4.5 pt right of the line before it, or
>= 2.5 pt lower than the page's normal line pitch. A line without a position or starting
in lower case never starts a paragraph. Text wrapped beside a drop cap is not a start.
Without positions the old indent rule is still used.
- cleanup: drop cap + small caps on ONE layout line ('F OR MANY YEARS') is repaired like
the two-line form (chapters 4 and 5).
- pageparts.header_zone: the running header is found even when text follows it without a
blank line, and the right-hand header (chapter title, page number sometimes misread as
't5' or 'a') is recognised from the chapter title.
- cleanup: all-capitals lines below the text are captions (removed); an all-capitals line
above the text is kept as a section heading in title case ('The Deferred Preface');
'Right: MARK TWIN' style captions removed; '5. JOY' labels set inside a sentence removed.
- cleanup: paragraph starting with a lone quote mark + lower-case word joins the paragraph
before it; paragraph-leading ' + capital becomes '"' (not elisions such as 'Tis); a bullet
in place of an opening quote becomes '"'; closing quote with no opening quote, and a '1'
that is probably the letter I, are logged only.
- cleanup: mid-line 'worth- while' joined only when certain, otherwise logged.
- tests: new _pipeline/tests/test_cleanup.py (36 tests, 27 fail on the previous code);
test_pipeline.py helper test_piece renamed make_test_piece so pytest stops collecting it.
Chapter 1 text is byte-identical to the approved pilot text.
Re-run for chapters 1-6: cleanup, figures, script, license. Text, script, warnings and
book.json for chapters 2-6 added.
This commit is contained in:
Audiobook Producer
+164
-11
@@ -75,9 +75,10 @@ def smallcaps_lead(text):
|
||||
def unit_lines(unit, title, heading, first_unit, notes):
|
||||
"""Body lines of one unit as dicts {text, indent, where}; removed parts are noted."""
|
||||
raw = [l.rstrip() for l in unit["text"].split("\n")]
|
||||
geom = unit.get("geom") or [None] * len(raw)
|
||||
where = "pdf p.%d%s" % (unit["pdf_page"], unit["half"] or "")
|
||||
drop = set()
|
||||
z = pageparts.header_zone(raw, title)
|
||||
z = pageparts.header_zone(raw, title, [heading["title"]] if heading else [])
|
||||
if z:
|
||||
drop.update(range(z[0], z[1]))
|
||||
what = "page number %d" % z[2] if z[2] is not None else "no page number"
|
||||
@@ -89,6 +90,9 @@ def unit_lines(unit, title, heading, first_unit, notes):
|
||||
if first_unit and heading:
|
||||
drop.update(range(0, heading["title_end"]))
|
||||
notes.append(("heading-removed", where, "chapter heading 'CHAPTER %d / %s' (spoken header is added in the script)" % (heading["number"], heading["title"])))
|
||||
# last line of running text: the lines after it (all capitals only) are picture captions, not headings
|
||||
body = [i for i, l in enumerate(raw) if i not in drop and any(c.islower() for c in l)]
|
||||
last_body = body[-1] if body else -1
|
||||
out = []
|
||||
for i, ln in enumerate(raw):
|
||||
if i in drop or not ln.strip():
|
||||
@@ -96,17 +100,87 @@ def unit_lines(unit, title, heading, first_unit, notes):
|
||||
if pageparts.CAPTION_RE.match(ln):
|
||||
notes.append(("caption-removed", where, "figure caption '%s' (listed in figures/skipped_figures.md)" % ln.strip()))
|
||||
continue
|
||||
out.append({"text": ln.strip(), "indent": len(ln) - len(ln.lstrip()), "where": where})
|
||||
if CAPTION_MIXED_RE.match(ln):
|
||||
notes.append(("caption-removed", where, "figure caption '%s' (marker not in capitals)" % ln.strip()))
|
||||
continue
|
||||
letters = [c for c in ln if c.isalpha()]
|
||||
if len(letters) >= 3 and not any(c.islower() for c in letters):
|
||||
if i > last_body:
|
||||
notes.append(("caption-removed", where, "all-capitals line '%s' below the text (picture caption or label)" % ln.strip()))
|
||||
continue
|
||||
spoken = bookio.title_case(" ".join(ln.split()))
|
||||
notes.append(("heading-kept", where, "all-capitals line above the text kept as a section heading: '%s' -> '%s'" % (ln.strip(), spoken)))
|
||||
ln = " " * (len(ln) - len(ln.lstrip())) + spoken
|
||||
g = geom[i] if i < len(geom) else None
|
||||
out.append({"text": ln.strip(), "indent": len(ln) - len(ln.lstrip()), "where": where,
|
||||
"unit": (unit["pdf_page"], unit["half"]), "x": g["x"] if g else None, "y": g["y"] if g else None})
|
||||
return out
|
||||
|
||||
|
||||
INDENT_PT = 4.5 # a paragraph's first line sits at least this many points right of the line before it
|
||||
GAP_PT = 2.5 # or this many points lower than the page's normal line spacing
|
||||
SAMELINE_CAP = re.compile(r"^([A-Z])\s{2,}([A-Z]{2,})\b")
|
||||
|
||||
|
||||
def unit_stats(lines):
|
||||
"""Per page unit: normal line pitch and the local left margin, read from line positions in points."""
|
||||
stats = {}
|
||||
for u in {l["unit"] for l in lines}:
|
||||
mine = [l for l in lines if l["unit"] == u and l["x"] is not None]
|
||||
gaps = sorted(b["y"] - a["y"] for a, b in zip(mine, mine[1:]) if 8 < b["y"] - a["y"] < 20)
|
||||
stats[u] = {"pitch": gaps[(len(gaps) - 1) // 2] if gaps else 13.0, "lines": mine}
|
||||
return stats
|
||||
|
||||
|
||||
def local_margin(stats, ln):
|
||||
"""Left margin near the first line of a unit: the leftmost of its next few lines (the page may be skewed)."""
|
||||
near = [l["x"] for l in stats[ln["unit"]]["lines"] if l is not ln and 0 < l["y"] - ln["y"] < 90]
|
||||
return min(near) if near else ln["x"]
|
||||
|
||||
|
||||
def starts_paragraph(prev, ln, stats):
|
||||
"""True if the line opens a paragraph. Evidence is geometric: indented relative to the line before it, or
|
||||
set lower than normal line spacing. A line with no position, or one that begins in lower case, never starts one."""
|
||||
if ln["x"] is None or ln["text"][:1].islower():
|
||||
return False
|
||||
if prev is not None and prev["unit"] == ln["unit"] and prev["x"] is not None:
|
||||
return ln["x"] - prev["x"] >= INDENT_PT or ln["y"] - prev["y"] >= stats[ln["unit"]]["pitch"] + GAP_PT
|
||||
return ln["x"] - local_margin(stats, ln) >= INDENT_PT
|
||||
|
||||
|
||||
def paragraphs_from(lines, inline, notes):
|
||||
"""Group lines into paragraphs by first-line indent, repair drop caps and hyphenation."""
|
||||
"""Group lines into paragraphs and repair drop caps. With line positions (the normal case) a paragraph break is
|
||||
an indent or a vertical gap; without them (older callers) it is the layout indent of the first line."""
|
||||
geometric = any(l.get("x") is not None for l in lines)
|
||||
stats = unit_stats(lines) if geometric else None
|
||||
base = min(l["indent"] for l in lines)
|
||||
i, paras, cur, cap_indent = 0, [], None, None
|
||||
i, paras, cur, cap_indent, wrap, prev = 0, [], None, None, False, None
|
||||
while i < len(lines):
|
||||
ln = dict(lines[i])
|
||||
is_cap = split_dropcap(lines, i)
|
||||
same = SAMELINE_CAP.match(ln["text"]) if not is_cap else None
|
||||
if geometric:
|
||||
if wrap and ln["x"] is not None and ln["x"] - local_margin(stats, ln) >= INDENT_PT and not same:
|
||||
is_start = False # still beside the drop cap
|
||||
else:
|
||||
wrap = False
|
||||
is_start = cur is None or starts_paragraph(prev, ln, stats)
|
||||
else:
|
||||
is_start = ln["indent"] - base >= 1 and not (cap_indent is not None and ln["indent"] == cap_indent)
|
||||
if same and (cur is None or is_start):
|
||||
letter, rest = same.group(1), ln["text"][same.end(1):].strip()
|
||||
first = same.group(2)
|
||||
if known(letter + first) or not (letter in "AI" and known(first)):
|
||||
merged, how = letter + rest, "joined"
|
||||
else:
|
||||
merged, how = letter + " " + rest, "kept as its own word"
|
||||
notes.append(("dropcap", ln["where"], "drop cap '%s' + '%s' on one line: %s" % (letter, rest[:12], how)))
|
||||
ln.update(text=merged, indent=base + 1, smallcaps=True)
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
wrap, prev = True, lines[i]
|
||||
i += 1
|
||||
continue
|
||||
if is_cap:
|
||||
letter, frag = ln["text"], lines[i + 1]
|
||||
first = re.match(r"[A-Z]+", frag["text"]).group(0)
|
||||
@@ -117,20 +191,21 @@ def paragraphs_from(lines, inline, notes):
|
||||
merged = letter + " " + frag["text"]
|
||||
how = "kept as its own word"
|
||||
notes.append(("dropcap", ln["where"], "drop cap '%s' + '%s': %s" % (letter, frag["text"][:12], how)))
|
||||
ln = {"text": merged, "indent": base + 1, "where": ln["where"], "smallcaps": True}
|
||||
ln = dict(frag, text=merged, indent=base + 1, where=ln["where"], smallcaps=True)
|
||||
cap_indent = frag["indent"]
|
||||
i += 2
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
wrap, prev = True, frag
|
||||
continue
|
||||
starts = ln["indent"] - base >= 1 and not (cap_indent is not None and ln["indent"] == cap_indent)
|
||||
if ln["indent"] != cap_indent:
|
||||
cap_indent = None
|
||||
if starts or cur is None:
|
||||
if is_start or cur is None:
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
else:
|
||||
cur.append(ln)
|
||||
prev = lines[i]
|
||||
i += 1
|
||||
return paras
|
||||
|
||||
@@ -151,11 +226,83 @@ def join_lines(para, inline, notes):
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
ELISION = {"Tis", "Twas", "Twere", "Twill", "Twould"}
|
||||
CAPTION_MIXED_RE = re.compile(r"^\s*(?:Left|Right|Above|Below|Opposite|Facing)\s*:\s*[A-Z][A-Z .,'\-]+\s*$")
|
||||
MIDLINE_HYPHEN_RE = re.compile(r"\b([A-Za-z]+)- ([a-z]+)\b")
|
||||
LABEL_RE = re.compile(r"\s\d\.\s+[A-Z]{3,}(?:\s+[A-Z]{3,})*(?=\s+[a-z])")
|
||||
DIGIT_ONE_RE = re.compile(r"(?<![\d.,$#/-])\b1 (?=[a-z]+\b)")
|
||||
|
||||
|
||||
def strip_inline_labels(text, where, notes):
|
||||
"""The book sets '5. JOY' style labels into the middle of a sentence (a typeset inset, not words of the sentence).
|
||||
Left in, they would be read aloud inside the sentence, so they are removed and logged."""
|
||||
def drop(m):
|
||||
notes.append(("inline-label-removed", where, "typeset label '%s' sat inside a sentence" % m.group(0).strip()))
|
||||
return ""
|
||||
return LABEL_RE.sub(drop, text)
|
||||
|
||||
|
||||
def fix_midline_hyphens(text, where, notes, inline):
|
||||
"""'worth- while' inside a line: the scan kept a line-end hyphen after the line was re-flowed.
|
||||
Joined only when the answer is certain (the hyphenated form occurs elsewhere in the book, or the joined
|
||||
word is a dictionary word). Otherwise logged and left as it is."""
|
||||
def fix(m):
|
||||
joined, reason, ok = join_hyphen(m.group(1), m.group(2), inline)
|
||||
certain = ok and (reason.startswith("hyphenated form") or reason.startswith("joined word"))
|
||||
notes.append(("hyphen-joined" if certain else "hyphen-unresolved", where,
|
||||
"mid-line '%s- %s' -> %s (%s)" % (m.group(1), m.group(2), "'%s'" % joined if certain else "not changed", reason)))
|
||||
return joined if certain else m.group(0)
|
||||
return MIDLINE_HYPHEN_RE.sub(fix, text)
|
||||
|
||||
|
||||
def merge_orphan_quotes(paras, chapter, notes):
|
||||
"""[(text, where)] -> same, where a paragraph that opens with a lone quote mark and a lower-case word is
|
||||
joined to the paragraph before it. No paragraph starts like that: the page broke the line after an opening
|
||||
quoted word, as in: They "milked' / " every operation. The stray ' closing the quoted word is made a '"'."""
|
||||
out = []
|
||||
for text, where in paras:
|
||||
if out and re.match(r'"\s+[a-z]', text):
|
||||
prev, prev_where = out[-1]
|
||||
fixed = prev
|
||||
if prev.count('"') % 2 == 1 and re.search(r"[A-Za-z]'$", prev):
|
||||
fixed = prev[:-1] + '"'
|
||||
out[-1] = (fixed + " " + text[1:].strip(), prev_where)
|
||||
notes.append(("paragraph-joined", where, "paragraph began with a lone quote mark and a lower-case word, "
|
||||
"so it continues the paragraph before it ('...%s' + '%s...')" % (prev[-14:], text[:14])))
|
||||
else:
|
||||
out.append((text, where))
|
||||
return out
|
||||
|
||||
|
||||
def fix_quotes(text, where, notes):
|
||||
"""A paragraph with an unclosed double quote that ends in punctuation + ' : the ' is a misread closing quote."""
|
||||
"""Quote marks the scan got wrong. Fixed only when the rule is certain; otherwise the paragraph is logged.
|
||||
- an unclosed double quote that ends in punctuation + ' : the ' is a misread closing quote.
|
||||
- a paragraph that opens with ' + capital letter (not an elision such as 'Tis): a misread opening
|
||||
double quote, as at the start of each paragraph of a long speech.
|
||||
- a closing double quote (after punctuation) with no opening quote before it in the paragraph is logged only."""
|
||||
if text.count('"') % 2 == 1 and re.search(r"[.?!]'$", text):
|
||||
notes.append(("quote-fixed", where, "closing quote misread as apostrophe at '...%s'" % text[-12:]))
|
||||
return text[:-1] + '"'
|
||||
text = text[:-1] + '"'
|
||||
if re.match(r"\u2022[A-Z]", text):
|
||||
notes.append(("quote-fixed", where, "opening quote misread as a bullet at '%s...'" % text[:16]))
|
||||
text = '"' + text[1:]
|
||||
m = re.match(r"'([A-Z][a-z]*)", text)
|
||||
if m and m.group(1) not in ELISION:
|
||||
notes.append(("quote-fixed", where, "opening quote misread as apostrophe at '%s...'" % text[:16]))
|
||||
text = '"' + text[1:]
|
||||
opened = False
|
||||
for q in re.finditer('"', text):
|
||||
before = text[q.start() - 1] if q.start() else ""
|
||||
if before in ("", " ", "(", "\u2014"):
|
||||
opened = True
|
||||
elif before == "'":
|
||||
opened = not opened
|
||||
elif opened:
|
||||
opened = False
|
||||
else:
|
||||
notes.append(("quote-unmatched", where, "closing quote with no opening quote before it: '...%s' (not changed)" % text[max(0, q.start() - 14):q.start() + 1]))
|
||||
for m in DIGIT_ONE_RE.finditer(text):
|
||||
notes.append(("digit-one", where, "'1' before a lowercase word, probably the letter I misread: '...%s' (not changed)" % text[max(0, m.start() - 8):m.end() + 8]))
|
||||
return text
|
||||
|
||||
|
||||
@@ -180,14 +327,20 @@ def run(book_dir, book, pdf, chapter_numbers):
|
||||
for ui in range(ch["first_unit"], ch["last_unit"] + 1):
|
||||
lines += unit_lines(units[ui], book["title"], heads[ch["number"]], ui == ch["first_unit"], notes)
|
||||
paras = paragraphs_from(lines, inline, notes)
|
||||
texts = []
|
||||
for n, p in enumerate(paras, start=1):
|
||||
joined = []
|
||||
for p in paras:
|
||||
t = join_lines(p, inline, notes)
|
||||
if p[0].get("smallcaps"):
|
||||
before, t = t, smallcaps_lead(t)
|
||||
if before != t:
|
||||
notes.append(("smallcaps", p[0]["where"], "'%s...' -> '%s...'" % (before[:14], t[:14])))
|
||||
joined.append((t, p[0]["where"]))
|
||||
joined = merge_orphan_quotes(joined, ch["number"], notes)
|
||||
texts = []
|
||||
for n, (t, _) in enumerate(joined, start=1):
|
||||
where = "ch%d para %d" % (ch["number"], n)
|
||||
t = strip_inline_labels(t, where, notes)
|
||||
t = fix_midline_hyphens(t, where, notes, inline)
|
||||
t = fix_quotes(t, where, notes)
|
||||
odd_words(t, where, notes)
|
||||
texts.append(t)
|
||||
|
||||
Reference in New Issue
Block a user