diff --git a/.gitea/workflows/audiobook.yml b/.gitea/workflows/audiobook.yml index 0e43735..c3bdc8b 100644 --- a/.gitea/workflows/audiobook.yml +++ b/.gitea/workflows/audiobook.yml @@ -22,7 +22,7 @@ permissions: concurrency: group: audiobook - cancel-in-progress: true # TEMPORARY: clearing a hung run (reverted right after) + cancel-in-progress: false # one book at a time; a second drop waits jobs: build: @@ -42,7 +42,13 @@ jobs: timeout-minutes: 15 # a stalled fetch fails the run instead of hanging it for hours uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 100 # enough history to rebase onto a main that moved during a long render + filter: blob:none # do not download every book's audio and PDFs on every run + sparse-checkout: | + /_pipeline/ + /_incoming/ + /.gitea/ + sparse-checkout-cone-mode: false - name: Install tools (fresh container every run) run: bash _pipeline/ci/install.sh - name: Unit tests (dry runs only) diff --git a/_pipeline/ci/run.sh b/_pipeline/ci/run.sh index 34968b2..274ebf8 100755 --- a/_pipeline/ci/run.sh +++ b/_pipeline/ci/run.sh @@ -12,8 +12,13 @@ BRANCH="${BRANCH:-main}" git config user.name "audiobook-bot" git config user.email "audiobook-bot@noreply.invalid" +# The checkout is sparse (code and _incoming only). Books are pulled in only when needed. +sparse_add() { printf '%s\n' "$@" | git sparse-checkout add --stdin; } +esc() { printf '%s' "$1" | sed -e 's/[][*?\\#!]/\\&/g'; } + pdfs=() if [ -n "${PDF_INPUT:-}" ]; then + sparse_add "/$(esc "$PDF_INPUT")" [ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; } pdfs=("$PDF_INPUT") else @@ -30,7 +35,7 @@ push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and for i in 1 2 3 4 5; do if git push origin "HEAD:refs/heads/$BRANCH"; then return 0; fi echo "push attempt $i failed; fetching and rebasing" - git pull --rebase origin "$BRANCH" || return 1 + git fetch -q --depth=100 origin "$BRANCH" && git rebase FETCH_HEAD || { git rebase --abort 2>/dev/null; return 1; } done return 1 } @@ -39,6 +44,17 @@ failed=0 for pdf in "${pdfs[@]}"; do echo; echo "################ $pdf" result="$(mktemp)" + rel="$(python3 _pipeline/cli.py bookpath "$pdf")" || { echo "FAILED: $rel"; failed=1; continue; } + echo "book folder: $rel" + e="$(esc "$rel")" + if git cat-file -e "HEAD:$rel" 2>/dev/null; then + if [ "$DRY_RUN" = "true" ]; then # dry run compares text only: skip audio + sparse_add "/$e/book.json" "/$e/warnings.log" "/$e/LICENSE_STATUS.md" "/$e/text/" "/$e/script/" "/$e/figures/" + else # a real run needs the existing audio so it can skip it + sparse_add "/$e/" + fi + echo "this book already exists in the repo; existing files fetched" + fi if [ "$DRY_RUN" = "true" ]; then export AUDIOBOOK_BOOKS_DIR="/tmp/audiobook-dry/books"; rm -rf /tmp/audiobook-dry mkdir -p /tmp/audiobook-dry diff --git a/_pipeline/cli.py b/_pipeline/cli.py index ff1f98c..6299da7 100644 --- a/_pipeline/cli.py +++ b/_pipeline/cli.py @@ -71,6 +71,8 @@ def main(argv=None): p.add_argument("--compare", help="a books folder (like the repo books/); the text-stage output is compared with the same / folder in it") p.add_argument("--compare-strict", action="store_true", help="with --compare: fail when the output differs (default: report only)") p.add_argument("--result-file", help="write {ok, book_dir, pdf} JSON here") + p = sub.add_parser("bookpath", help="print the book folder (relative to the repo root) this PDF will go to") + p.add_argument("pdf") p = sub.add_parser("ingest") p.add_argument("pdf") p.add_argument("--title") @@ -92,6 +94,14 @@ def main(argv=None): a = ap.parse_args(argv) cfg = settings.load(a.settings) + if a.cmd == "bookpath": + import process + try: + title, author, _ = process.resolve_meta(os.path.abspath(a.pdf)) + except process.Failed as e: + sys.exit(str(e)) + print(os.path.relpath(ingest.book_folder(cfg, author, title), settings.REPO_ROOT)) + return if a.cmd == "process": import process sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict)) diff --git a/_pipeline/ingest.py b/_pipeline/ingest.py index aafee05..3f11f23 100644 --- a/_pipeline/ingest.py +++ b/_pipeline/ingest.py @@ -40,14 +40,23 @@ def author_slug(author): return bookio.slug(" ".join([parts[-1]] + parts[:-1])) if parts else "unknown" +def book_folder(cfg, author, title): + return os.path.join(settings.resolve(cfg, "books_dir"), author_slug(author), bookio.slug(title)) + + +def detect(pdf, title=None, author=None): + """Title and author from the PDF itself (title: PDF metadata; author: a 'By ' line on the first pages).""" + author = author or find_author([u["text"] for u in pdftext.units(pdf, first_pages=6)]) + title = title or find_title(pdf) + return title, author + + def run(cfg, pdf, title=None, author=None): """Returns the book folder path.""" - pdf_units = pdftext.units(pdf) - author = author or find_author([u["text"] for u in pdf_units[:6]]) - title = title or find_title(pdf) + title, author = detect(pdf, title, author) if not author or not title: raise SystemExit("could not find title/author in the PDF; pass --title and --author") - book_dir = os.path.join(settings.resolve(cfg, "books_dir"), author_slug(author), bookio.slug(title)) + book_dir = book_folder(cfg, author, title) os.makedirs(book_dir, exist_ok=True) copy = os.path.join(book_dir, os.path.basename(pdf)) if os.path.exists(copy) and sha256(copy) != sha256(pdf): diff --git a/_pipeline/pdftext.py b/_pipeline/pdftext.py index da1c375..c67b8b3 100644 --- a/_pipeline/pdftext.py +++ b/_pipeline/pdftext.py @@ -85,11 +85,12 @@ def align(raw_lines, boxes, window=8): return result -def units(pdf): +def units(pdf, first_pages=None): """Reading units in order: [{'pdf_page', 'half', 'text', 'geom'}]. Text keeps the layout (indentation, hyphens). 'geom' is parallel to text.split('\\n'): the position of each line on the page (see align).""" result = [] - for p, (w, h) in enumerate(page_geometry(pdf), start=1): + geo = page_geometry(pdf) + for p, (w, h) in enumerate(geo[:first_pages] if first_pages else geo, start=1): boxes = page_lines(pdf, p) for half, x, width in halves(w, h): text = _run(["pdftotext", "-layout", "-f", str(p), "-l", str(p), diff --git a/_pipeline/process.py b/_pipeline/process.py index c6d0ee7..8887bd4 100644 --- a/_pipeline/process.py +++ b/_pipeline/process.py @@ -112,6 +112,26 @@ def step(name): print("\n=== %s ===" % name, flush=True) +def resolve_meta(pdf, title=None, author=None): + """(title, author, note). Order: explicit arguments, the .json next to the PDF, the PDF itself, the file name. + Raises Failed, telling the operator what to add, if none of them works.""" + meta = sidecar(pdf) + title, author = title or meta.get("title"), author or meta.get("author") + try: + title, author = ingest.detect(pdf, title, author) + except Exception as e: + raise Failed("could not read the PDF: %s" % e) + note = None + if not (title and author): + t, a = guess_from_filename(pdf) + title, author = title or t, author or a + note = "title/author not found in the PDF; taking them from the file name: '%s' by '%s'" % (title, author) + if not (title and author): + raise Failed("ingest: could not find title and author in the PDF or its file name. Put a file with the same name and " + '.json next to the PDF: {"title": "...", "author": "..."}') + return title, author, note + + def check_ocr(cfg, book_dir): """Pages with no text layer must have been OCRed. Fails the run if they were not (setting ocr.required).""" book = bookio.read_book(book_dir) @@ -140,19 +160,16 @@ def check_ocr(cfg, book_dir): def run_text_stages(cfg, pdf, title, author): """Ingest through license for every chapter. Returns (book_dir, book).""" import cli - meta = sidecar(pdf) - title, author = title or meta.get("title"), author or meta.get("author") + title, author, note = resolve_meta(pdf, title, author) step("ingest") + if note: + print(note) try: book_dir = ingest.run(cfg, pdf, title, author) except SystemExit as e: - t, a = guess_from_filename(pdf) - if not (t and a): - raise Failed("ingest: %s. Put a file with the same name and .json next to the PDF: " - '{"title": "...", "author": "..."}' % e) - print("ingest: title/author not found in the PDF; taking them from the file name: '%s' by '%s'" % (t, a)) - book_dir = ingest.run(cfg, pdf, title or t, author or a) - bookio.write_warnings(book_dir, "ingest-metadata", [("metadata-from-filename", os.path.basename(pdf), "title '%s', author '%s'" % (t, a))]) + raise Failed("ingest: %s" % e) + if note: + bookio.write_warnings(book_dir, "ingest-metadata", [("metadata-from-filename", os.path.basename(pdf), "title '%s', author '%s'" % (title, author))]) print("ingest done ->", book_dir) check_ocr(cfg, book_dir) for name in ("structure",):