Workflow: sparse partial checkout (code and _incoming only), book lookup before processing, safe rebase retry; revert cancel-in-progress

.gitea/workflows/audiobook.yml

@@ -22,7 +22,7 @@ permissions:
 
 concurrency:
   group: audiobook
-  cancel-in-progress: true            # TEMPORARY: clearing a hung run (reverted right after)
+  cancel-in-progress: false           # one book at a time; a second drop waits
 
 jobs:
   build:
@@ -42,7 +42,13 @@ jobs:
         timeout-minutes: 15           # a stalled fetch fails the run instead of hanging it for hours
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          fetch-depth: 100                # enough history to rebase onto a main that moved during a long render
+          filter: blob:none               # do not download every book's audio and PDFs on every run
+          sparse-checkout: |
+            /_pipeline/
+            /_incoming/
+            /.gitea/
+          sparse-checkout-cone-mode: false
       - name: Install tools (fresh container every run)
         run: bash _pipeline/ci/install.sh
       - name: Unit tests (dry runs only)

_pipeline/ci/run.sh

@@ -12,8 +12,13 @@ BRANCH="${BRANCH:-main}"
 git config user.name "audiobook-bot"
 git config user.email "audiobook-bot@noreply.invalid"
 
+# The checkout is sparse (code and _incoming only). Books are pulled in only when needed.
+sparse_add() { printf '%s\n' "$@" | git sparse-checkout add --stdin; }
+esc() { printf '%s' "$1" | sed -e 's/[][*?\\#!]/\\&/g'; }
+
 pdfs=()
 if [ -n "${PDF_INPUT:-}" ]; then
+  sparse_add "/$(esc "$PDF_INPUT")"
   [ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; }
   pdfs=("$PDF_INPUT")
 else
@@ -30,7 +35,7 @@ push_with_retry() {   # push HEAD to the branch; if the branch moved, rebase and
   for i in 1 2 3 4 5; do
     if git push origin "HEAD:refs/heads/$BRANCH"; then return 0; fi
     echo "push attempt $i failed; fetching and rebasing"
-    git pull --rebase origin "$BRANCH" || return 1
+    git fetch -q --depth=100 origin "$BRANCH" && git rebase FETCH_HEAD || { git rebase --abort 2>/dev/null; return 1; }
   done
   return 1
 }
@@ -39,6 +44,17 @@ failed=0
 for pdf in "${pdfs[@]}"; do
   echo; echo "################ $pdf"
   result="$(mktemp)"
+  rel="$(python3 _pipeline/cli.py bookpath "$pdf")" || { echo "FAILED: $rel"; failed=1; continue; }
+  echo "book folder: $rel"
+  e="$(esc "$rel")"
+  if git cat-file -e "HEAD:$rel" 2>/dev/null; then
+    if [ "$DRY_RUN" = "true" ]; then   # dry run compares text only: skip audio
+      sparse_add "/$e/book.json" "/$e/warnings.log" "/$e/LICENSE_STATUS.md" "/$e/text/" "/$e/script/" "/$e/figures/"
+    else                               # a real run needs the existing audio so it can skip it
+      sparse_add "/$e/"
+    fi
+    echo "this book already exists in the repo; existing files fetched"
+  fi
   if [ "$DRY_RUN" = "true" ]; then
     export AUDIOBOOK_BOOKS_DIR="/tmp/audiobook-dry/books"; rm -rf /tmp/audiobook-dry
     mkdir -p /tmp/audiobook-dry

_pipeline/cli.py

@@ -71,6 +71,8 @@ def main(argv=None):
     p.add_argument("--compare", help="a books folder (like the repo books/); the text-stage output is compared with the same <author>/<book> folder in it")
     p.add_argument("--compare-strict", action="store_true", help="with --compare: fail when the output differs (default: report only)")
     p.add_argument("--result-file", help="write {ok, book_dir, pdf} JSON here")
+    p = sub.add_parser("bookpath", help="print the book folder (relative to the repo root) this PDF will go to")
+    p.add_argument("pdf")
     p = sub.add_parser("ingest")
     p.add_argument("pdf")
     p.add_argument("--title")
@@ -92,6 +94,14 @@ def main(argv=None):
     a = ap.parse_args(argv)
     cfg = settings.load(a.settings)
 
+    if a.cmd == "bookpath":
+        import process
+        try:
+            title, author, _ = process.resolve_meta(os.path.abspath(a.pdf))
+        except process.Failed as e:
+            sys.exit(str(e))
+        print(os.path.relpath(ingest.book_folder(cfg, author, title), settings.REPO_ROOT))
+        return
     if a.cmd == "process":
         import process
         sys.exit(process.process(cfg, a.pdf, a.dry_run, a.force, a.title, a.author, a.compare, a.result_file, a.compare_strict))

_pipeline/ingest.py

@@ -40,14 +40,23 @@ def author_slug(author):
     return bookio.slug(" ".join([parts[-1]] + parts[:-1])) if parts else "unknown"
 
 
+def book_folder(cfg, author, title):
+    return os.path.join(settings.resolve(cfg, "books_dir"), author_slug(author), bookio.slug(title))
+
+
+def detect(pdf, title=None, author=None):
+    """Title and author from the PDF itself (title: PDF metadata; author: a 'By <Name>' line on the first pages)."""
+    author = author or find_author([u["text"] for u in pdftext.units(pdf, first_pages=6)])
+    title = title or find_title(pdf)
+    return title, author
+
+
 def run(cfg, pdf, title=None, author=None):
     """Returns the book folder path."""
-    pdf_units = pdftext.units(pdf)
+    title, author = detect(pdf, title, author)
-    author = author or find_author([u["text"] for u in pdf_units[:6]])
-    title = title or find_title(pdf)
     if not author or not title:
         raise SystemExit("could not find title/author in the PDF; pass --title and --author")
-    book_dir = os.path.join(settings.resolve(cfg, "books_dir"), author_slug(author), bookio.slug(title))
+    book_dir = book_folder(cfg, author, title)
     os.makedirs(book_dir, exist_ok=True)
     copy = os.path.join(book_dir, os.path.basename(pdf))
     if os.path.exists(copy) and sha256(copy) != sha256(pdf):

_pipeline/pdftext.py

@@ -85,11 +85,12 @@ def align(raw_lines, boxes, window=8):
     return result
 
 
-def units(pdf):
+def units(pdf, first_pages=None):
     """Reading units in order: [{'pdf_page', 'half', 'text', 'geom'}]. Text keeps the layout (indentation, hyphens).
     'geom' is parallel to text.split('\\n'): the position of each line on the page (see align)."""
     result = []
-    for p, (w, h) in enumerate(page_geometry(pdf), start=1):
+    geo = page_geometry(pdf)
+    for p, (w, h) in enumerate(geo[:first_pages] if first_pages else geo, start=1):
         boxes = page_lines(pdf, p)
         for half, x, width in halves(w, h):
             text = _run(["pdftotext", "-layout", "-f", str(p), "-l", str(p),

_pipeline/process.py

@@ -112,6 +112,26 @@ def step(name):
     print("\n=== %s ===" % name, flush=True)
 
 
+def resolve_meta(pdf, title=None, author=None):
+    """(title, author, note). Order: explicit arguments, the .json next to the PDF, the PDF itself, the file name.
+    Raises Failed, telling the operator what to add, if none of them works."""
+    meta = sidecar(pdf)
+    title, author = title or meta.get("title"), author or meta.get("author")
+    try:
+        title, author = ingest.detect(pdf, title, author)
+    except Exception as e:
+        raise Failed("could not read the PDF: %s" % e)
+    note = None
+    if not (title and author):
+        t, a = guess_from_filename(pdf)
+        title, author = title or t, author or a
+        note = "title/author not found in the PDF; taking them from the file name: '%s' by '%s'" % (title, author)
+    if not (title and author):
+        raise Failed("ingest: could not find title and author in the PDF or its file name. Put a file with the same name and "
+                     '.json next to the PDF: {"title": "...", "author": "..."}')
+    return title, author, note
+
+
 def check_ocr(cfg, book_dir):
     """Pages with no text layer must have been OCRed. Fails the run if they were not (setting ocr.required)."""
     book = bookio.read_book(book_dir)
@@ -140,19 +160,16 @@ def check_ocr(cfg, book_dir):
 def run_text_stages(cfg, pdf, title, author):
     """Ingest through license for every chapter. Returns (book_dir, book)."""
     import cli
-    meta = sidecar(pdf)
+    title, author, note = resolve_meta(pdf, title, author)
-    title, author = title or meta.get("title"), author or meta.get("author")
     step("ingest")
+    if note:
+        print(note)
     try:
         book_dir = ingest.run(cfg, pdf, title, author)
     except SystemExit as e:
-        t, a = guess_from_filename(pdf)
+        raise Failed("ingest: %s" % e)
-        if not (t and a):
+    if note:
-            raise Failed("ingest: %s. Put a file with the same name and .json next to the PDF: "
+        bookio.write_warnings(book_dir, "ingest-metadata", [("metadata-from-filename", os.path.basename(pdf), "title '%s', author '%s'" % (title, author))])
-                         '{"title": "...", "author": "..."}' % e)
-        print("ingest: title/author not found in the PDF; taking them from the file name: '%s' by '%s'" % (t, a))
-        book_dir = ingest.run(cfg, pdf, title or t, author or a)
-        bookio.write_warnings(book_dir, "ingest-metadata", [("metadata-from-filename", os.path.basename(pdf), "title '%s', author '%s'" % (t, a))])
     print("ingest     done ->", book_dir)
     check_ocr(cfg, book_dir)
     for name in ("structure",):