Audiobook Producer
296 lines
12 KiB
Python
296 lines
12 KiB
Python
"""One command for a whole book: PDF in, finished book folder out.
|
|
|
|
python3 _pipeline/cli.py process <pdf> [--dry-run] [--force] [--title T --author A] [--compare BOOKS_ROOT [--compare-strict]] [--result-file F]
|
|
|
|
Stages, in order, for ALL chapters the structure stage finds:
|
|
ingest (OCR only if pages lack a text layer) -> structure -> cleanup -> figures -> script -> license
|
|
-> [voice -> assemble -> ACX check] (skipped with --dry-run, which instead proves the voice server is reachable)
|
|
|
|
Rules:
|
|
* Any failed step or failed ACX check stops the run with a clear message and exit code 1.
|
|
* Warnings are logged to warnings.log in the book folder. There is no approval gate.
|
|
* Idempotent: an audio file that already exists is skipped, unless its script or the voice settings changed
|
|
since it was built (book.json "audio_built") or --force is given.
|
|
* Output goes where the settings say: books_dir (env AUDIOBOOK_BOOKS_DIR), scratch work_dir (env AUDIOBOOK_WORK_DIR),
|
|
voice endpoint (env AUDIOBOOK_VOICE_ENDPOINT).
|
|
"""
|
|
import difflib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
|
|
import acx_check
|
|
import assemble
|
|
import bookio
|
|
import ingest
|
|
import settings
|
|
import voice
|
|
|
|
|
|
class Failed(Exception):
|
|
"""A pipeline step failed. The message is what the operator reads."""
|
|
|
|
|
|
class Tee(io.TextIOBase):
|
|
def __init__(self, *streams):
|
|
self.streams = streams
|
|
|
|
def write(self, s):
|
|
for st in self.streams:
|
|
st.write(s)
|
|
return len(s)
|
|
|
|
def flush(self):
|
|
for st in self.streams:
|
|
st.flush()
|
|
|
|
|
|
def sidecar(pdf):
|
|
"""Optional <pdf name>.json next to the PDF: {"title": "...", "author": "..."}."""
|
|
path = os.path.splitext(pdf)[0] + ".json"
|
|
if os.path.exists(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
|
|
def guess_from_filename(pdf):
|
|
"""'The Title - by Glenn Clark (Extra).pdf' or 'Title by Glenn Clark.pdf' -> (title, author), else (None, None)."""
|
|
stem = os.path.splitext(os.path.basename(pdf))[0]
|
|
m = re.match(r"^(?P<title>.+?)[\s\-–—]+by\s+(?P<author>[^()]+?)\s*(\(.*\))?\s*$", stem, re.I)
|
|
if m:
|
|
return m.group("title").strip(" -–—"), m.group("author").strip()
|
|
return None, None
|
|
|
|
|
|
def voice_fingerprint(cfg, script_path):
|
|
h = hashlib.sha256()
|
|
with open(script_path, "rb") as f:
|
|
h.update(f.read())
|
|
v = cfg["voice"]
|
|
h.update(json.dumps([v["api"], v["voice"], v.get("extra")], sort_keys=True).encode())
|
|
h.update(json.dumps(cfg["audio"], sort_keys=True).encode())
|
|
return h.hexdigest()[:16]
|
|
|
|
|
|
def read_if(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def book_json_for_compare(path):
|
|
book = json.loads(read_if(path))
|
|
book.get("license", {}).pop("checked", None)
|
|
return json.dumps(book, indent=1, sort_keys=True)
|
|
|
|
|
|
def compare_books(new_dir, old_dir):
|
|
"""Text-stage output of new_dir against an existing book folder. Returns a list of difference descriptions."""
|
|
diffs = []
|
|
pairs = [("book.json", book_json_for_compare), ("warnings.log", read_if), ("figures/skipped_figures.md", read_if),
|
|
("LICENSE_STATUS.md", lambda p: re.sub(r"Date checked: .*", "", read_if(p)))]
|
|
for sub in ("text", "script"):
|
|
names = set(os.listdir(os.path.join(old_dir, sub))) | set(os.listdir(os.path.join(new_dir, sub)))
|
|
pairs += [(os.path.join(sub, n), read_if) for n in sorted(names)]
|
|
for rel, reader in pairs:
|
|
a, b = os.path.join(old_dir, rel), os.path.join(new_dir, rel)
|
|
if not os.path.exists(a) or not os.path.exists(b):
|
|
diffs.append("%s: missing in %s" % (rel, "new output" if os.path.exists(a) else "existing book"))
|
|
continue
|
|
ta, tb = reader(a), reader(b)
|
|
if ta != tb:
|
|
d = list(difflib.unified_diff(ta.splitlines(), tb.splitlines(), "existing/" + rel, "new/" + rel, lineterm="", n=0))
|
|
diffs.append("%s differs (%d diff lines):\n %s" % (rel, len(d), "\n ".join(d[:12])))
|
|
return diffs
|
|
|
|
|
|
def step(name):
|
|
print("\n=== %s ===" % name, flush=True)
|
|
|
|
|
|
def resolve_meta(pdf, title=None, author=None):
|
|
"""(title, author, note). Order: explicit arguments, the .json next to the PDF, the PDF itself, the file name.
|
|
Raises Failed, telling the operator what to add, if none of them works."""
|
|
meta = sidecar(pdf)
|
|
title, author = title or meta.get("title"), author or meta.get("author")
|
|
try:
|
|
title, author = ingest.detect(pdf, title, author)
|
|
except Exception as e:
|
|
raise Failed("could not read the PDF: %s" % e)
|
|
note = None
|
|
if not (title and author):
|
|
t, a = guess_from_filename(pdf)
|
|
title, author = title or t, author or a
|
|
note = "title/author not found in the PDF; taking them from the file name: '%s' by '%s'" % (title, author)
|
|
if not (title and author):
|
|
raise Failed("ingest: could not find title and author in the PDF or its file name. Put a file with the same name and "
|
|
'.json next to the PDF: {"title": "...", "author": "..."}')
|
|
return title, author, note
|
|
|
|
|
|
def check_ocr(cfg, book_dir):
|
|
"""Pages with no text layer must have been OCRed. Fails the run if they were not (setting ocr.required)."""
|
|
book = bookio.read_book(book_dir)
|
|
missing = book["pages_without_text_layer"]
|
|
if not missing:
|
|
print("OCR: not needed, every page has a text layer")
|
|
return
|
|
ocr_pdf = os.path.join(settings.resolve(cfg, "work_dir"), os.path.basename(book_dir), "ocr.pdf")
|
|
pages = ",".join(map(str, missing))
|
|
if not os.path.exists(ocr_pdf):
|
|
msg = "pdf pages %s have no text layer and OCR did not produce a text copy (is ocrmypdf installed? see warnings.log)" % pages
|
|
if cfg["ocr"].get("required", True):
|
|
raise Failed("ingest: " + msg)
|
|
print("WARNING:", msg)
|
|
return
|
|
import pdftext
|
|
counts = pdftext.chars_per_page(ocr_pdf)
|
|
got = {p: counts[p - 1] for p in missing}
|
|
print("OCR applied to pdf pages %s; characters found after OCR: %s" % (pages, got))
|
|
empty = [p for p, n in got.items() if n < cfg["ocr"]["min_chars_per_page"]]
|
|
if empty:
|
|
bookio.write_warnings(book_dir, "process-ocr", [("ocr-no-text", "pdf pages " + ",".join(map(str, empty)),
|
|
"OCR found no text (blank page or picture only)")])
|
|
|
|
|
|
def run_text_stages(cfg, pdf, title, author):
|
|
"""Ingest through license for every chapter. Returns (book_dir, book)."""
|
|
import cli
|
|
title, author, note = resolve_meta(pdf, title, author)
|
|
step("ingest")
|
|
if note:
|
|
print(note)
|
|
try:
|
|
book_dir = ingest.run(cfg, pdf, title, author)
|
|
except SystemExit as e:
|
|
raise Failed("ingest: %s" % e)
|
|
if note:
|
|
bookio.write_warnings(book_dir, "ingest-metadata", [("metadata-from-filename", os.path.basename(pdf), "title '%s', author '%s'" % (title, author))])
|
|
print("ingest done ->", book_dir)
|
|
check_ocr(cfg, book_dir)
|
|
for name in ("structure",):
|
|
step(name)
|
|
try:
|
|
cli.run_stage(name, cfg, book_dir)
|
|
except SystemExit as e:
|
|
raise Failed("%s: %s" % (name, e))
|
|
book = bookio.read_book(book_dir)
|
|
numbers = [c["number"] for c in book["chapters"]]
|
|
if numbers != list(range(1, len(numbers) + 1)):
|
|
bookio.write_warnings(book_dir, "process", [("chapter-numbers", "structure", "chapter numbers found: %s (expected 1..%d in order)" % (numbers, len(numbers)))])
|
|
print("chapters found:", numbers)
|
|
for name in ("cleanup", "figures", "script", "license"):
|
|
step(name)
|
|
try:
|
|
cli.run_stage(name, cfg, book_dir, numbers)
|
|
except SystemExit as e:
|
|
raise Failed("%s: %s" % (name, e))
|
|
return book_dir, bookio.read_book(book_dir)
|
|
|
|
|
|
def run_audio_stages(cfg, book_dir, book, force):
|
|
out_dir = settings.audio_dir(cfg, book_dir)
|
|
built = book.setdefault("audio_built", {})
|
|
todo = []
|
|
for name in book["audio_files"]:
|
|
fp = voice_fingerprint(cfg, os.path.join(book_dir, "script", name + ".json"))
|
|
have = os.path.exists(os.path.join(out_dir, name + ".mp3"))
|
|
if have and not force and built.get(name, fp) == fp:
|
|
print("skip %s: audio exists and is current" % name)
|
|
else:
|
|
todo.append((name, fp))
|
|
if todo:
|
|
step("voice (%d of %d files)" % (len(todo), len(book["audio_files"])))
|
|
h = voice.ensure_loaded(cfg)
|
|
print("voice server ok:", h)
|
|
names = [n for n, _ in todo]
|
|
for n, fp in todo: # finished pieces from an interrupted run are reused only if script and voice settings are the same
|
|
wd = voice.work_dir_for(cfg, book_dir, n)
|
|
stamp = os.path.join(wd, "fingerprint")
|
|
if os.path.isdir(wd) and (force or not os.path.exists(stamp) or read_if(stamp) != fp):
|
|
shutil.rmtree(wd)
|
|
os.makedirs(wd, exist_ok=True)
|
|
with open(stamp, "w") as f:
|
|
f.write(fp)
|
|
voice.run(cfg, book_dir, names=names)
|
|
step("assemble")
|
|
for n in names:
|
|
assemble.run(cfg, book_dir, only=n)
|
|
for n, fp in todo:
|
|
built[n] = fp
|
|
bookio.write_book(book_dir, book)
|
|
if cfg["voice"].get("unload_when_done"):
|
|
voice.unload(cfg)
|
|
print("voice model unloaded to free the GPU")
|
|
else:
|
|
print("nothing to render")
|
|
step("ACX check")
|
|
rc = acx_check.run(cfg, [out_dir])
|
|
if rc != 0:
|
|
raise Failed("ACX check FAILED (see the table above); nothing should be committed")
|
|
|
|
|
|
def process(cfg, pdf, dry_run=False, force=False, title=None, author=None, compare=None, result_file=None, compare_strict=False):
|
|
"""Returns 0 on success, 1 on failure (message printed)."""
|
|
pdf = os.path.abspath(pdf)
|
|
if not os.path.isfile(pdf):
|
|
print("PROCESS FAILED: no such file: %s" % pdf)
|
|
return 1
|
|
log = io.StringIO()
|
|
real_out = sys.stdout
|
|
sys.stdout = Tee(real_out, log)
|
|
book_dir = None
|
|
try:
|
|
print("process %s%s" % (os.path.basename(pdf), " [DRY RUN: text stages only]" if dry_run else ""))
|
|
book_dir, book = run_text_stages(cfg, pdf, title, author)
|
|
if compare:
|
|
old = os.path.join(compare, os.path.relpath(book_dir, settings.resolve(cfg, "books_dir")))
|
|
step("compare with the existing book folder %s" % old)
|
|
if not os.path.isdir(old):
|
|
print("no existing folder to compare with (this is a new book)")
|
|
else:
|
|
diffs = compare_books(book_dir, old)
|
|
if diffs and compare_strict:
|
|
raise Failed("output differs from the existing book folder:\n" + "\n".join(diffs))
|
|
if diffs:
|
|
print("DIFFERENCES from the existing book folder (report only; add --compare-strict to fail on them):\n" + "\n".join(diffs))
|
|
else:
|
|
print("identical to the existing text, script, figures list, warnings, book.json and license file")
|
|
if dry_run:
|
|
step("voice server reachability (no audio rendered)")
|
|
try:
|
|
h = voice.health(cfg)
|
|
except voice.VoiceError as e:
|
|
raise Failed(str(e))
|
|
print("voice server reachable:", h)
|
|
else:
|
|
try:
|
|
run_audio_stages(cfg, book_dir, book, force)
|
|
except voice.VoiceError as e:
|
|
raise Failed("voice: %s" % e)
|
|
print("\nPROCESS OK: %s" % book_dir)
|
|
rc = 0
|
|
except Failed as e:
|
|
print("\nPROCESS FAILED: %s" % e)
|
|
rc = 1
|
|
except Exception as e: # any other error is also a loud failure, with the type named
|
|
print("\nPROCESS FAILED: unexpected %s: %s" % (type(e).__name__, e))
|
|
import traceback
|
|
traceback.print_exc(file=sys.stdout)
|
|
rc = 1
|
|
finally:
|
|
sys.stdout = real_out
|
|
if rc == 0 and book_dir and not dry_run:
|
|
out_dir = settings.audio_dir(cfg, book_dir)
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
with open(os.path.join(out_dir, "run.log"), "w", encoding="utf-8") as f:
|
|
f.write(log.getvalue())
|
|
if result_file:
|
|
with open(result_file, "w", encoding="utf-8") as f:
|
|
json.dump({"ok": rc == 0, "book_dir": book_dir, "pdf": pdf}, f)
|
|
return rc
|