Add pipeline code
This commit is contained in:
Grok Bot
@@ -0,0 +1,200 @@
|
||||
"""Stage 2, clean-up: turn the text layer of a chapter into clean paragraphs.
|
||||
Every rule is deterministic. Every change and every word the dictionary does not know goes to warnings.log.
|
||||
Author words are never rewritten, only layout artifacts are removed or repaired."""
|
||||
import os
|
||||
import re
|
||||
|
||||
from spellchecker import SpellChecker
|
||||
|
||||
import bookio
|
||||
import pageparts
|
||||
import pdftext
|
||||
import structure
|
||||
|
||||
WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?")
|
||||
_dict = SpellChecker()
|
||||
|
||||
|
||||
def known(word):
|
||||
return bool(_dict.known([word.lower()]))
|
||||
|
||||
|
||||
def vocabulary(units):
|
||||
"""Words and hyphenated words that occur inline (not split by a line break) anywhere in the book."""
|
||||
inline = set()
|
||||
for u in units:
|
||||
for ln in u["text"].split("\n"):
|
||||
toks = re.findall(r"[A-Za-z]+(?:-[A-Za-z]+)*", ln)
|
||||
if ln.rstrip().endswith("-") and toks:
|
||||
toks = toks[:-1]
|
||||
inline.update(t.lower() for t in toks)
|
||||
return inline
|
||||
|
||||
|
||||
def parts_known(word, inline):
|
||||
return all(known(p) or p.lower() in inline for p in word.split("-") if p)
|
||||
|
||||
|
||||
def join_hyphen(left, frag, inline):
|
||||
"""Decide how to join 'left-' + 'frag'. Returns (joined_word, reason, resolved)."""
|
||||
joined, hyph = left + frag, left + "-" + frag
|
||||
if hyph.lower() in inline:
|
||||
return hyph, "hyphenated form occurs elsewhere in the book", True
|
||||
if parts_known(joined, inline) and (known(joined.split("-")[-1]) or joined.lower() in inline):
|
||||
return joined, "joined word is a known word", True
|
||||
if known(left.split("-")[-1]) and known(frag):
|
||||
return hyph, "both fragments are words and the joined form is unknown, hyphen kept", True
|
||||
return joined, "neither form is known, joined by default", False
|
||||
|
||||
|
||||
def split_dropcap(lines, i):
|
||||
"""A lone capital letter followed by a line that starts with the rest of the first word."""
|
||||
if re.fullmatch(r"[A-Z]", lines[i]["text"].strip()) and i + 1 < len(lines):
|
||||
nxt = lines[i + 1]["text"].strip()
|
||||
if re.match(r"[A-Z]{2,}\b", nxt) or re.match(r"[A-Z]{2,}$", nxt):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def smallcaps_lead(text):
|
||||
"""'ALL MY LIFE I have' -> 'All my life I have' (leading run of capitals after a drop cap)."""
|
||||
toks = text.split(" ")
|
||||
n = 0
|
||||
while n < len(toks) and toks[n].strip(".,;:!?\"'").isupper():
|
||||
n += 1
|
||||
out = []
|
||||
for k, t in enumerate(toks):
|
||||
if k < n and len(t.strip(".,;:!?\"'")) > 1:
|
||||
t = t.lower()
|
||||
if k == 0:
|
||||
t = t[:1].upper() + t[1:]
|
||||
out.append(t)
|
||||
return " ".join(out)
|
||||
|
||||
|
||||
def unit_lines(unit, title, heading, first_unit, notes):
|
||||
"""Body lines of one unit as dicts {text, indent, where}; removed parts are noted."""
|
||||
raw = [l.rstrip() for l in unit["text"].split("\n")]
|
||||
where = "pdf p.%d%s" % (unit["pdf_page"], unit["half"] or "")
|
||||
drop = set()
|
||||
z = pageparts.header_zone(raw, title)
|
||||
if z:
|
||||
drop.update(range(z[0], z[1]))
|
||||
what = "page number %d" % z[2] if z[2] is not None else "no page number"
|
||||
notes.append(("header-removed", where, "running header '%s' (%s)" % (" ".join(l.strip() for l in raw[z[0]:z[1]]), what)))
|
||||
f = pageparts.footer_number(raw)
|
||||
if f:
|
||||
drop.add(f[0])
|
||||
notes.append(("page-number-removed", where, "footer page number %d" % f[1]))
|
||||
if first_unit and heading:
|
||||
drop.update(range(0, heading["title_end"]))
|
||||
notes.append(("heading-removed", where, "chapter heading 'CHAPTER %d / %s' (spoken header is added in the script)" % (heading["number"], heading["title"])))
|
||||
out = []
|
||||
for i, ln in enumerate(raw):
|
||||
if i in drop or not ln.strip():
|
||||
continue
|
||||
if pageparts.CAPTION_RE.match(ln):
|
||||
notes.append(("caption-removed", where, "figure caption '%s' (listed in figures/skipped_figures.md)" % ln.strip()))
|
||||
continue
|
||||
out.append({"text": ln.strip(), "indent": len(ln) - len(ln.lstrip()), "where": where})
|
||||
return out
|
||||
|
||||
|
||||
def paragraphs_from(lines, inline, notes):
|
||||
"""Group lines into paragraphs by first-line indent, repair drop caps and hyphenation."""
|
||||
base = min(l["indent"] for l in lines)
|
||||
i, paras, cur, cap_indent = 0, [], None, None
|
||||
while i < len(lines):
|
||||
ln = dict(lines[i])
|
||||
is_cap = split_dropcap(lines, i)
|
||||
if is_cap:
|
||||
letter, frag = ln["text"], lines[i + 1]
|
||||
first = re.match(r"[A-Z]+", frag["text"]).group(0)
|
||||
if known(letter + first) or not (letter in "AI" and known(first)):
|
||||
merged = letter + frag["text"]
|
||||
how = "joined"
|
||||
else:
|
||||
merged = letter + " " + frag["text"]
|
||||
how = "kept as its own word"
|
||||
notes.append(("dropcap", ln["where"], "drop cap '%s' + '%s': %s" % (letter, frag["text"][:12], how)))
|
||||
ln = {"text": merged, "indent": base + 1, "where": ln["where"], "smallcaps": True}
|
||||
cap_indent = frag["indent"]
|
||||
i += 2
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
continue
|
||||
starts = ln["indent"] - base >= 1 and not (cap_indent is not None and ln["indent"] == cap_indent)
|
||||
if ln["indent"] != cap_indent:
|
||||
cap_indent = None
|
||||
if starts or cur is None:
|
||||
cur = [ln]
|
||||
paras.append(cur)
|
||||
else:
|
||||
cur.append(ln)
|
||||
i += 1
|
||||
return paras
|
||||
|
||||
|
||||
def join_lines(para, inline, notes):
|
||||
text = para[0]["text"]
|
||||
for ln in para[1:]:
|
||||
nxt = ln["text"]
|
||||
m = re.search(r"([A-Za-z]+(?:-[A-Za-z]+)*)-$", text)
|
||||
f = re.match(r"([a-z]+)(.*)$", nxt)
|
||||
if m and f:
|
||||
joined, reason, ok = join_hyphen(m.group(1), f.group(1), inline)
|
||||
notes.append(("hyphen-joined" if ok else "hyphen-unresolved", ln["where"],
|
||||
"'%s-' + '%s' -> '%s' (%s)" % (m.group(1), f.group(1), joined, reason)))
|
||||
text = text[:m.start()] + joined + f.group(2)
|
||||
else:
|
||||
text = text + " " + nxt
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def fix_quotes(text, where, notes):
|
||||
"""A paragraph with an unclosed double quote that ends in punctuation + ' : the ' is a misread closing quote."""
|
||||
if text.count('"') % 2 == 1 and re.search(r"[.?!]'$", text):
|
||||
notes.append(("quote-fixed", where, "closing quote misread as apostrophe at '...%s'" % text[-12:]))
|
||||
return text[:-1] + '"'
|
||||
return text
|
||||
|
||||
|
||||
def odd_words(text, where, notes):
|
||||
for w in WORD_RE.findall(text):
|
||||
core = re.sub(r"'s$", "", w)
|
||||
if len(core) > 1 and not known(core):
|
||||
notes.append(("odd-word", where, "'%s' is not in the dictionary%s" % (w, " (capitalised: probably a name)" if w[0].isupper() else "")))
|
||||
|
||||
|
||||
def run(book_dir, book, pdf, chapter_numbers):
|
||||
"""Writes text/chapter-NN.txt for the chosen chapters. Returns warnings."""
|
||||
units = pdftext.units(pdf)
|
||||
heads = {h["number"]: h for h in structure.find_headings(units)}
|
||||
inline = vocabulary(units)
|
||||
os.makedirs(os.path.join(book_dir, "text"), exist_ok=True)
|
||||
notes = []
|
||||
for ch in book["chapters"]:
|
||||
if ch["number"] not in chapter_numbers:
|
||||
continue
|
||||
lines = []
|
||||
for ui in range(ch["first_unit"], ch["last_unit"] + 1):
|
||||
lines += unit_lines(units[ui], book["title"], heads[ch["number"]], ui == ch["first_unit"], notes)
|
||||
paras = paragraphs_from(lines, inline, notes)
|
||||
texts = []
|
||||
for n, p in enumerate(paras, start=1):
|
||||
t = join_lines(p, inline, notes)
|
||||
if p[0].get("smallcaps"):
|
||||
before, t = t, smallcaps_lead(t)
|
||||
if before != t:
|
||||
notes.append(("smallcaps", p[0]["where"], "'%s...' -> '%s...'" % (before[:14], t[:14])))
|
||||
where = "ch%d para %d" % (ch["number"], n)
|
||||
t = fix_quotes(t, where, notes)
|
||||
odd_words(t, where, notes)
|
||||
texts.append(t)
|
||||
name = "chapter-%02d.txt" % ch["number"]
|
||||
with open(os.path.join(book_dir, "text", name), "w", encoding="utf-8") as f:
|
||||
f.write("\n\n".join(texts) + "\n")
|
||||
ch["text_file"] = "text/" + name
|
||||
ch["words"] = sum(len(t.split()) for t in texts)
|
||||
ch["paragraphs"] = len(texts)
|
||||
return notes
|
||||
Reference in New Issue
Block a user