Add pipeline code

2026-10-17 15:27:14 -05:00
parent 3cd5a4c658
commit 5f0e2a9768
21 changed files with 1612 additions and 0 deletions
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""One command line for the whole pipeline.
+
+  python3 _pipeline/cli.py run <pdf> --chapters 1      ingest, structure, cleanup, figures, script, license
+  python3 _pipeline/cli.py ingest|structure|cleanup|figures|script|license ...
+  python3 _pipeline/cli.py voice <book_dir> [--only NAME]   script segments -> wav pieces (needs the voice endpoint)
+  python3 _pipeline/cli.py assemble <book_dir>              wav pieces -> MP3 per script file in audio/
+  python3 _pipeline/cli.py check <mp3 or folder> ...        ACX checker, pass/fail table
+"""
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import bookio
+import cleanup
+import figures
+import ingest
+import license as license_stage
+import script
+import settings
+import structure
+
+
+def chapters_arg(text):
+    return [int(x) for x in text.split(",")]
+
+
+def stage(name, fn, book_dir):
+    warnings = fn()
+    bookio.write_warnings(book_dir, name, warnings)
+    print("%-10s done, %d warnings" % (name, len(warnings)))
+
+
+def run_stage(name, cfg, book_dir, chapters=None):
+    book = bookio.read_book(book_dir)
+    pdf = ingest.work_pdf(cfg, book_dir, book)
+    if name == "structure":
+        stage(name, lambda: structure.run(book_dir, book, pdf), book_dir)
+        for c in book["chapters"]:
+            print("  chapter %d  %s  (pdf pp. %d-%d, book pp. %s-%s)" % (
+                c["number"], c["title"], c["pdf_pages"][0], c["pdf_pages"][1], c["book_pages"][0], c["book_pages"][1]))
+    elif name == "cleanup":
+        stage(name, lambda: cleanup.run(book_dir, book, pdf, chapters), book_dir)
+    elif name == "figures":
+        stage(name, lambda: figures.run(book_dir, book, pdf, chapters), book_dir)
+    elif name == "script":
+        stage(name, lambda: script.run(cfg, book_dir, book, chapters), book_dir)
+    elif name == "license":
+        stage(name, lambda: license_stage.run(cfg, book_dir, book, pdf) or [], book_dir)
+    bookio.write_book(book_dir, book)
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--settings", help="path to settings.yaml (default: _pipeline/settings.yaml)")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("run")
+    p.add_argument("pdf")
+    p.add_argument("--chapters", required=True, type=chapters_arg, help="chapter numbers, e.g. 1 or 1,2")
+    p.add_argument("--title")
+    p.add_argument("--author")
+    p = sub.add_parser("ingest")
+    p.add_argument("pdf")
+    p.add_argument("--title")
+    p.add_argument("--author")
+    sub.add_parser("structure").add_argument("book_dir")
+    for name in ("cleanup", "figures", "script"):
+        p = sub.add_parser(name)
+        p.add_argument("book_dir")
+        p.add_argument("--chapters", required=True, type=chapters_arg)
+    sub.add_parser("license").add_argument("book_dir")
+    p = sub.add_parser("voice")
+    p.add_argument("book_dir")
+    p.add_argument("--only", help="one script name, e.g. chapter-01")
+    p = sub.add_parser("assemble")
+    p.add_argument("book_dir")
+    p.add_argument("--only")
+    p = sub.add_parser("check")
+    p.add_argument("paths", nargs="+")
+    a = ap.parse_args(argv)
+    cfg = settings.load(a.settings)
+
+    if a.cmd in ("run", "ingest"):
+        book_dir = ingest.run(cfg, os.path.abspath(a.pdf), a.title, a.author)
+        print("ingest     done ->", book_dir)
+        if a.cmd == "run":
+            for name in ("structure", "cleanup", "figures", "script", "license"):
+                run_stage(name, cfg, book_dir, a.chapters)
+    elif a.cmd in ("structure", "cleanup", "figures", "script", "license"):
+        run_stage(a.cmd, cfg, os.path.abspath(a.book_dir), getattr(a, "chapters", None))
+    elif a.cmd == "voice":
+        import voice
+        voice.run(cfg, os.path.abspath(a.book_dir), a.only)
+    elif a.cmd == "assemble":
+        import assemble
+        assemble.run(cfg, os.path.abspath(a.book_dir), a.only)
+    elif a.cmd == "check":
+        import acx_check
+        sys.exit(acx_check.run(cfg, a.paths))
+
+
+if __name__ == "__main__":
+    main()