#!/usr/bin/env python3 """One command line for the whole pipeline. python3 _pipeline/cli.py process [--dry-run] [--force] EVERYTHING for a whole book, all chapters; exit 1 on any failure python3 _pipeline/cli.py run --chapters 1 ingest, structure, cleanup, figures, script, license python3 _pipeline/cli.py ingest|structure|cleanup|figures|script|license ... python3 _pipeline/cli.py voice [--only NAME] script segments -> wav pieces (needs the voice endpoint) python3 _pipeline/cli.py assemble wav pieces -> MP3 per script file in audio/ python3 _pipeline/cli.py check ... ACX checker, pass/fail table """ import argparse import json import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import bookio import cleanup import figures import ingest import license as license_stage import script import settings import structure def chapters_arg(text): return [int(x) for x in text.split(",")] def stage(name, fn, book_dir): warnings = fn() bookio.write_warnings(book_dir, name, warnings) print("%-10s done, %d warnings" % (name, len(warnings))) def run_stage(name, cfg, book_dir, chapters=None): book = bookio.read_book(book_dir) pdf = ingest.work_pdf(cfg, book_dir, book) if name == "structure": stage(name, lambda: structure.run(book_dir, book, pdf), book_dir) for c in book["chapters"]: print(" chapter %d %s (pdf pp. %d-%d, book pp. %s-%s)" % ( c["number"], c["title"], c["pdf_pages"][0], c["pdf_pages"][1], c["book_pages"][0], c["book_pages"][1])) elif name == "cleanup": stage(name, lambda: cleanup.run(book_dir, book, pdf, chapters), book_dir) elif name == "figures": stage(name, lambda: figures.run(book_dir, book, pdf, chapters), book_dir) elif name == "script": stage(name, lambda: script.run(cfg, book_dir, book, chapters), book_dir) elif name == "license": stage(name, lambda: license_stage.run(cfg, book_dir, book, pdf) or [], book_dir) bookio.write_book(book_dir, book) def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--settings", help="path to settings.yaml (default: _pipeline/settings.yaml)") sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("run") p.add_argument("pdf") p.add_argument("--chapters", required=True, type=chapters_arg, help="chapter numbers, e.g. 1 or 1,2") p.add_argument("--title") p.add_argument("--author") p = sub.add_parser("process", help="whole book, all stages, all chapters") p.add_argument("pdf") p.add_argument("--dry-run", action="store_true", help="text stages only; checks the voice server is reachable; renders nothing") p.add_argument("--force", action="store_true", help="re-render audio even if it exists") p.add_argument("--title") p.add_argument("--author") p.add_argument("--compare", help="a books folder (like the repo books/); the text-stage output is compared with the same / folder in it") p.add_argument("--compare-strict", action="store_true", help="with --compare: fail when the output differs (default: report only)") p.add_argument("--result-file", help="write {ok, book_dir, pdf} JSON here") p.add_argument("--source-url", help="where the PDF was downloaded from (recorded in book.json)") p.add_argument("--name-hint", action="append", default=[], help="another file name to read a title from (repeatable)") p = sub.add_parser("bookpath", help="print the book folder (relative to the repo root) this PDF will go to") p.add_argument("pdf") p.add_argument("--name-hint", action="append", default=[]) p = sub.add_parser("fetch", help="download the PDF named by a .url/.txt file into a scratch folder; writes /fetch.json") p.add_argument("urlfile") p.add_argument("--out", required=True) p = sub.add_parser("ingest") p.add_argument("pdf") p.add_argument("--title") p.add_argument("--author") sub.add_parser("structure").add_argument("book_dir") for name in ("cleanup", "figures", "script"): p = sub.add_parser(name) p.add_argument("book_dir") p.add_argument("--chapters", required=True, type=chapters_arg) sub.add_parser("license").add_argument("book_dir") p = sub.add_parser("voice") p.add_argument("book_dir") p.add_argument("--only", help="one script name, e.g. chapter-01") p = sub.add_parser("assemble") p.add_argument("book_dir") p.add_argument("--only") p = sub.add_parser("check") p.add_argument("paths", nargs="+") a = ap.parse_args(argv) cfg = settings.load(a.settings) if a.cmd == "bookpath": import process try: m = process.resolve_meta(cfg, os.path.abspath(a.pdf), name_hints=a.name_hint) except process.Failed as e: sys.exit(str(e)) print("\n".join(m["log"]), file=sys.stderr) print(os.path.relpath(ingest.book_folder(cfg, m["author"], m["title"]), settings.REPO_ROOT)) return if a.cmd == "fetch": import fetch try: res = fetch.fetch_from_file(cfg, a.urlfile, a.out) with open(os.path.join(a.out, "fetch.json"), "w", encoding="utf-8") as f: json.dump(res, f) except fetch.FetchError as e: sys.exit("FETCH FAILED: %s" % e) return if a.cmd == "process": import process sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict, a.source_url, a.name_hint)) if a.cmd in ("run", "ingest"): book_dir = ingest.run(cfg, os.path.abspath(a.pdf), a.title, a.author) print("ingest done ->", book_dir) if a.cmd == "run": for name in ("structure", "cleanup", "figures", "script", "license"): run_stage(name, cfg, book_dir, a.chapters) elif a.cmd in ("structure", "cleanup", "figures", "script", "license"): run_stage(a.cmd, cfg, os.path.abspath(a.book_dir), getattr(a, "chapters", None)) elif a.cmd == "voice": import voice voice.run(cfg, os.path.abspath(a.book_dir), a.only) elif a.cmd == "assemble": import assemble assemble.run(cfg, os.path.abspath(a.book_dir), a.only) elif a.cmd == "check": import acx_check sys.exit(acx_check.run(cfg, a.paths)) if __name__ == "__main__": main()