Audiobook Producer
227 lines
11 KiB
Python
227 lines
11 KiB
Python
"""Stage 3, structure: find chapter boundaries from the text layer. Audible style file list is made by the script stage."""
|
|
import difflib
|
|
import re
|
|
|
|
import bookio
|
|
import pageparts
|
|
import pdftext
|
|
|
|
WORDS = "ONE TWO THREE FOUR FIVE SIX SEVEN EIGHT NINE TEN ELEVEN TWELVE THIRTEEN FOURTEEN FIFTEEN SIXTEEN SEVENTEEN EIGHTEEN NINETEEN TWENTY".split()
|
|
ROMAN = {"I": 1, "V": 5, "X": 10, "L": 50}
|
|
HEADING_RE = re.compile(r"^\s*([A-Z]{6,9}|CHAPTER)\b[\s.:]*(.*?)[\s.,:;]*$")
|
|
|
|
|
|
def chapter_number(token):
|
|
if token.isdigit():
|
|
return int(token)
|
|
if token in WORDS:
|
|
return WORDS.index(token) + 1
|
|
if set(token) <= set(ROMAN):
|
|
total = 0
|
|
for a, b in zip(token, token[1:] + " "):
|
|
total += -ROMAN[a] if b in ROMAN and ROMAN[b] > ROMAN[a] else ROMAN[a]
|
|
return total
|
|
return None
|
|
|
|
|
|
def is_chapter_word(word):
|
|
"""CHAPTER, or what OCR makes of it (CUAPTER, CIEAPTER, OUAPTER): upper-case letters, 6 to 9 of them, 70% like CHAPTER."""
|
|
return word == "CHAPTER" or (re.fullmatch(r"[A-Z]{6,9}", word) is not None and difflib.SequenceMatcher(None, word, "CHAPTER").ratio() >= 0.7
|
|
and word not in ("CHAPTERS", "CAPTAIN", "CHARTER", "CHAPLET"))
|
|
|
|
|
|
def heading_number(line, expected):
|
|
"""(number, guessed) for a 'CHAPTER <number>' line, else None. The number may be a roman numeral, digits or a word.
|
|
OCR often damages the numeral ('I I r', 'X V', 'X f I'): a short unreadable token after an upper-case CHAPTER is
|
|
taken to be the next chapter in sequence (guessed=True, and a warning is written by run())."""
|
|
m = HEADING_RE.match(line)
|
|
if not m:
|
|
return None
|
|
if not is_chapter_word(m.group(1)):
|
|
return None
|
|
rest = m.group(2).strip()
|
|
if re.fullmatch(r"[A-Z]+|\d+", rest):
|
|
n = chapter_number(rest)
|
|
if n is not None:
|
|
return n, False
|
|
collapsed = re.sub(r"\s+", "", rest)
|
|
if 0 < len(rest) <= 8 and re.fullmatch(r"[A-Za-z0-9|!.,]+", collapsed):
|
|
n = chapter_number(collapsed.upper().strip(".,"))
|
|
if n is not None and not (" " in rest and n != expected): # 'I I' is more likely III than II when III is next
|
|
return n, False
|
|
return expected, True
|
|
return None
|
|
|
|
|
|
def find_headings(units):
|
|
"""'CHAPTER ONE' then the title line(s) up to a blank line, within the first 12 lines of a unit.
|
|
A heading whose number does not rise (a table-of-contents entry that came before the real chapter) is replaced by the
|
|
later one. With no heading at all the whole body is one chapter ('synthetic')."""
|
|
found = []
|
|
for ui, u in enumerate(units):
|
|
lines = u["text"].split("\n")
|
|
shown = 0
|
|
for li, ln in enumerate(lines):
|
|
if ln.strip():
|
|
shown += 1
|
|
if shown > 12:
|
|
break
|
|
got = heading_number(ln, (found[-1]["number"] + 1) if found else 1)
|
|
if got is None:
|
|
continue
|
|
n, guessed = got
|
|
k = li + 1
|
|
while k < len(lines) and not lines[k].strip():
|
|
k += 1
|
|
title = []
|
|
while k < len(lines) and lines[k].strip():
|
|
title.append(lines[k].strip())
|
|
k += 1
|
|
if len(" ".join(title)) > 100: # no blank line after the title: keep the leading upper-case lines only
|
|
keep = 1
|
|
while keep < len(title) and keep < 3 and pageparts._mostly_upper(title[keep]):
|
|
keep += 1
|
|
k = k - (len(title) - keep)
|
|
title = title[:keep]
|
|
h = {"unit": ui, "line": li, "title_end": k, "number": n, "title": bookio.title_case(" ".join(title))}
|
|
if guessed:
|
|
h["guessed"] = True
|
|
while found and found[-1]["number"] >= n: # the earlier one was a contents entry or a misread
|
|
found.pop()
|
|
h["replaced_earlier"] = True
|
|
found.append(h)
|
|
break
|
|
if not found:
|
|
sizes = [len(re.sub(r"\s", "", u["text"])) for u in units]
|
|
filled = sorted(n for n in sizes if n >= 20)
|
|
floor = max(40, 0.4 * filled[len(filled) // 2]) if filled else 0 # a page with 40% of the median text or more is body
|
|
for ui, n in enumerate(sizes):
|
|
if filled and n >= floor:
|
|
return [{"unit": ui, "line": -1, "title_end": 0, "number": 1, "title": "The Complete Text", "synthetic": True}]
|
|
return found
|
|
|
|
|
|
def toc_chapters(units, before_unit):
|
|
"""Chapter lines of the table of contents, read from the units before the first chapter: [(roman, title, page)]."""
|
|
lines = []
|
|
for u in units[:before_unit]:
|
|
lines += u["text"].split("\n")
|
|
out = []
|
|
for i, ln in enumerate(lines):
|
|
m = re.match(r"^\s*([IVX]+)\.\s+(.+?)\s*\.{3,}\s*(\d+)?\s*$", ln)
|
|
if not m:
|
|
continue
|
|
page = m.group(3)
|
|
if page is None and i + 1 < len(lines) and re.match(r"^\s*\d+\s*$", lines[i + 1]):
|
|
page = lines[i + 1].strip()
|
|
out.append((chapter_number(m.group(1)), bookio.title_case(m.group(2)), int(page) if page else None))
|
|
return out
|
|
|
|
|
|
def unknown_words(text):
|
|
"""How many words of a title the spelling dictionary does not know (used to choose between two readings of one title)."""
|
|
import cleanup
|
|
return sum(1 for w in re.findall(r"[A-Za-z]+", text) if len(w) > 2 and not cleanup.known(w))
|
|
|
|
|
|
def toc_chapter_lines(units, before_unit):
|
|
"""Contents in the form 'CHAPTER V.' on one line and 'Title ........ 20' on the next: {number: (title, page)}."""
|
|
lines = []
|
|
for u in units[:before_unit]:
|
|
lines += [ln for ln in u["text"].split("\n") if ln.strip()]
|
|
out, last = {}, 0
|
|
for i, ln in enumerate(lines[:-1]):
|
|
got = heading_number(ln, last + 1)
|
|
if got is None or not re.search(r"\.{3,}", lines[i + 1]):
|
|
continue
|
|
title = re.split(r"\s*\.{3,}", lines[i + 1].strip())[0].strip(" .,")
|
|
if len(re.sub(r"[^A-Za-z]", "", title)) >= 4 and got[0] not in out:
|
|
out[got[0]] = (bookio.title_case(re.sub(r"\s+", " ", title)), None)
|
|
last = got[0]
|
|
return out
|
|
|
|
|
|
def book_pages(units, title):
|
|
"""Printed page number of each unit (None if unknown). Printed numbers are read from headers/footers;
|
|
a unit without one takes its neighbor's number plus or minus its distance."""
|
|
printed = []
|
|
for u in units:
|
|
lines = u["text"].split("\n")
|
|
z = pageparts.header_zone(lines, title)
|
|
num = z[2] if z else None
|
|
if num is None:
|
|
f = pageparts.footer_number(lines)
|
|
num = f[1] if f else None
|
|
printed.append(num)
|
|
anchors = [i for i, n in enumerate(printed) if n is not None]
|
|
pages = []
|
|
for i in range(len(units)):
|
|
if printed[i] is not None:
|
|
pages.append(printed[i])
|
|
continue
|
|
near = min(anchors, key=lambda a: (abs(a - i), a > i)) if anchors else None
|
|
guess = printed[near] + (i - near) if near is not None else None
|
|
pages.append(guess if guess and guess >= 1 else None)
|
|
return pages, printed
|
|
|
|
|
|
def run(book_dir, book, pdf):
|
|
"""Fills book['chapters']. Returns warnings."""
|
|
units = pdftext.units(pdf)
|
|
heads = find_headings(units)
|
|
warnings = []
|
|
if not heads:
|
|
raise SystemExit("no chapter headings found and no page with a body of text to treat as a single chapter")
|
|
for h in heads:
|
|
if h.get("synthetic"):
|
|
warnings.append(("no-chapter-headings", "book", "no 'CHAPTER <number>' headings found; the text from pdf page %d on is treated as one chapter" % units[h["unit"]]["pdf_page"]))
|
|
if h.get("guessed"):
|
|
warnings.append(("chapter-number-guessed", "chapter %d" % h["number"], "the number after CHAPTER on pdf page %d was unreadable (OCR); taken as the next in sequence" % units[h["unit"]]["pdf_page"]))
|
|
if h.get("replaced_earlier"):
|
|
warnings.append(("heading-replaced", "chapter %d" % h["number"], "an earlier CHAPTER line with the same or a higher number (a contents entry or misread) was dropped in favour of the heading on pdf page %d" % units[h["unit"]]["pdf_page"]))
|
|
pages, printed = book_pages(units, book["title"])
|
|
toc = {n: (t, p) for n, t, p in toc_chapters(units, heads[0]["unit"])}
|
|
for n, tp in toc_chapter_lines(units, heads[0]["unit"]).items():
|
|
toc.setdefault(n, tp)
|
|
old = {c["number"]: c for c in book.get("chapters", [])}
|
|
chapters = []
|
|
for k, h in enumerate(heads):
|
|
if k + 1 < len(heads):
|
|
nh = heads[k + 1]
|
|
last_unit = nh["unit"] - 1
|
|
lines = units[nh["unit"]]["text"].split("\n")
|
|
z = pageparts.header_zone(lines, book["title"])
|
|
stray = [l for i, l in enumerate(lines[:nh["line"]]) if l.strip() and not (z and z[0] <= i < z[1])]
|
|
if stray:
|
|
warnings.append(("chapter-starts-mid-page", "chapter %d" % nh["number"],
|
|
"text sits above the heading on its page; it is not assigned to the previous chapter"))
|
|
else:
|
|
last_unit = len(units) - 1
|
|
ch = {
|
|
"number": h["number"],
|
|
"title": h["title"],
|
|
"first_unit": h["unit"],
|
|
"last_unit": last_unit,
|
|
"pdf_pages": [units[h["unit"]]["pdf_page"], units[last_unit]["pdf_page"]],
|
|
"book_pages": [pages[h["unit"]], pages[last_unit]],
|
|
"switches": old.get(h["number"], {}).get("switches", {}),
|
|
}
|
|
if h["number"] in toc:
|
|
t_title, t_page = toc[h["number"]]
|
|
ch["toc_page"] = t_page
|
|
if t_page is not None and t_page != ch["book_pages"][0]:
|
|
warnings.append(("toc-mismatch", "chapter %d" % h["number"],
|
|
"table of contents says page %s, structure found page %s" % (t_page, ch["book_pages"][0])))
|
|
sim = pageparts.similar(t_title, h["title"])
|
|
if 0.5 <= sim < 1 and t_page is None and not h.get("synthetic") and unknown_words(t_title) < unknown_words(h["title"]):
|
|
warnings.append(("title-from-contents", "chapter %d" % h["number"], "heading read '%s' (OCR damage?); the contents page says '%s' with fewer unknown words, which is used" % (h["title"], t_title)))
|
|
ch["title"] = t_title
|
|
elif sim < 0.9:
|
|
warnings.append(("toc-mismatch", "chapter %d" % h["number"],
|
|
"table of contents title '%s' differs from heading '%s'" % (t_title, h["title"])))
|
|
chapters.append(ch)
|
|
warnings.append(("last-chapter-end", "chapter %d" % chapters[-1]["number"],
|
|
"no later heading, so the chapter is taken to the end of the book (back matter, if any, is included)"))
|
|
book["chapters"] = chapters
|
|
return warnings
|