Add pipeline code

_pipeline/pageparts.py

@@ -0,0 +1,65 @@
+"""Recognize the non-text parts of a page: running header, page number, figure caption."""
+import re
+from difflib import SequenceMatcher
+
+CAPTION_RE = re.compile(r"^\s*(LEFT|RIGHT|ABOVE|BELOW|OPPOSITE|FACING)\s*:\s*([A-Z][A-Z .,'\-]+?)\s*$")
+_NUM_LEAD = re.compile(r"^\s*(\d{1,3})\s{2,}(\S.*)$")
+_NUM_TRAIL = re.compile(r"^(.*\S)\s{2,}(\d{1,3})\s*$")
+_NUM_ONLY = re.compile(r"^\s*(\d{1,3})\s*$")
+
+
+def norm(text):
+    return re.sub(r"[^a-z]", "", text.lower())
+
+
+def similar(a, b):
+    return SequenceMatcher(None, norm(a), norm(b)).ratio()
+
+
+def header_zone(lines, title):
+    """The running header at the top of a unit.
+    Returns (first_line, last_line_exclusive, page_number or None) or None.
+    A header is the first block of lines (up to a blank line) that either carries a page number
+    next to uppercase text, or reads as the book title."""
+    i = 0
+    while i < len(lines) and not lines[i].strip():
+        i += 1
+    j = i
+    while j < len(lines) and lines[j].strip():
+        j += 1
+    zone = lines[i:j]
+    if not zone:
+        return None
+    number, rest = None, []
+    for ln in zone:
+        m = _NUM_LEAD.match(ln) or None
+        if m and number is None:
+            number, rest = int(m.group(1)), rest + [m.group(2)]
+            continue
+        m = _NUM_TRAIL.match(ln)
+        if m and number is None:
+            number, rest = int(m.group(2)), rest + [m.group(1)]
+            continue
+        m = _NUM_ONLY.match(ln)
+        if m and number is None:
+            number = int(m.group(1))
+            continue
+        rest.append(ln)
+    text = " ".join(rest)
+    letters = [c for c in text if c.isalpha()]
+    mostly_upper = (not letters) or sum(c.isupper() for c in letters) / len(letters) > 0.7
+    if number is not None and mostly_upper and len(zone) <= 3:
+        return i, j, number
+    if title and similar(text, title) >= 0.9:
+        return i, j, None
+    return None
+
+
+def footer_number(lines):
+    """A lone page number as the last non-empty line. Returns (line_index, number) or None."""
+    k = len(lines) - 1
+    while k >= 0 and not lines[k].strip():
+        k -= 1
+    if k >= 0 and _NUM_ONLY.match(lines[k]):
+        return k, int(lines[k])
+    return None