Add Gitea Actions workflow: _incoming/*.pdf drop-folder pipeline, with dry-run dispatch

.gitea/workflows/audiobook.yml

@@ -0,0 +1,59 @@
+# Drop a PDF in _incoming/ and commit it. This job turns it into a finished audiobook folder and commits that back.
+# No bot is involved: every step is a deterministic script in _pipeline/.
+name: audiobook
+
+on:
+  push:
+    branches: [main, 'test/**']       # test/** lets the pipeline be proven on a branch without touching main
+    paths: ['_incoming/*.pdf']        # only a PDF landing in _incoming/ starts a run
+  workflow_dispatch:
+    inputs:
+      pdf:
+        description: 'PDF path inside the repo (empty = every PDF waiting in _incoming/)'
+        required: false
+        default: 'books/clark-glenn/the-man-who-tapped-the-secrets-of-the-universe/The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf'
+      dry_run:
+        description: 'true = text stages only, voice reachability check, push-permission test, commit nothing'
+        type: boolean
+        default: true
+
+permissions:
+  contents: write
+
+concurrency:
+  group: audiobook
+  cancel-in-progress: false           # one book at a time; a second drop waits
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 720              # long books render for hours
+    # The commit-back commit carries [skip ci] and removes the PDF from _incoming/, so it cannot start another run.
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+    env:
+      AUDIOBOOK_VOICE_ENDPOINT: ${{ vars.AUDIOBOOK_VOICE_ENDPOINT || 'http://192.168.1.202:8090/tts' }}
+      AUDIOBOOK_WORK_DIR: /tmp/audiobook-work
+      AUDIOBOOK_REQUIRE_OCR: '1'
+      BRANCH: ${{ github.ref_name }}
+      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'false' }}
+      PDF_INPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.pdf || '' }}
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install tools (fresh container every run)
+        run: bash _pipeline/ci/install.sh
+      - name: Unit tests (dry runs only)
+        if: ${{ env.DRY_RUN == 'true' }}
+        run: |
+          .venv/bin/pip install -q --disable-pip-version-check reportlab pillow
+          .venv/bin/python -W ignore -m unittest discover -s _pipeline/tests 2>&1 | grep -v -E '^\s+(data|read) =' | tail -40
+          test "${PIPESTATUS[0]}" -eq 0
+      - name: Voice server reachable from this job
+        run: |
+          echo "voice endpoint: $AUDIOBOOK_VOICE_ENDPOINT"
+          base="${AUDIOBOOK_VOICE_ENDPOINT%/tts}"
+          curl -sS --max-time 20 "$base/api/model-info" && echo
+      - name: Process
+        run: bash _pipeline/ci/run.sh

_pipeline/ci/install.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Installs everything the job needs in a FRESH container (node:20-bookworm). Safe to run twice.
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq --no-install-recommends ffmpeg poppler-utils tesseract-ocr tesseract-ocr-eng ocrmypdf ghostscript \
+  python3 python3-venv python3-pip git curl ca-certificates
+python3 -m venv .venv
+.venv/bin/pip install -q --disable-pip-version-check -r _pipeline/requirements.txt
+echo "--- tool versions"
+python3 --version; ffmpeg -version | head -1; tesseract --version 2>&1 | head -1; ocrmypdf --version; pdftotext -v 2>&1 | head -1

_pipeline/ci/run.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# The whole job, in one place. Inputs (environment):
+#   DRY_RUN      true|false   (true: text stages only, voice reachability, push-permission test, commits nothing to the branch)
+#   PDF_INPUT    optional path relative to the repo root (manual runs). Empty = every PDF waiting in _incoming/
+#   BRANCH       branch to commit back to
+#   AUDIOBOOK_*  setting overrides (see _pipeline/settings.py)
+set -uo pipefail
+cd "${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}"
+export PATH="$PWD/.venv/bin:$PATH"
+DRY_RUN="${DRY_RUN:-false}"
+BRANCH="${BRANCH:-main}"
+git config user.name "audiobook-bot"
+git config user.email "audiobook-bot@noreply.invalid"
+
+pdfs=()
+if [ -n "${PDF_INPUT:-}" ]; then
+  [ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; }
+  pdfs=("$PDF_INPUT")
+else
+  while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f -iname '*.pdf' | sort)
+fi
+if [ "${#pdfs[@]}" -eq 0 ]; then
+  echo "Nothing to do: no PDF in _incoming/."
+  exit 0
+fi
+echo "PDFs to process: ${#pdfs[@]}"; printf '  %s\n' "${pdfs[@]}"
+
+push_with_retry() {   # push HEAD to the branch; if the branch moved, rebase and try again
+  local i
+  for i in 1 2 3 4 5; do
+    if git push origin "HEAD:refs/heads/$BRANCH"; then return 0; fi
+    echo "push attempt $i failed; fetching and rebasing"
+    git pull --rebase origin "$BRANCH" || return 1
+  done
+  return 1
+}
+
+failed=0
+for pdf in "${pdfs[@]}"; do
+  echo; echo "################ $pdf"
+  result="$(mktemp)"
+  if [ "$DRY_RUN" = "true" ]; then
+    export AUDIOBOOK_BOOKS_DIR="/tmp/audiobook-dry/books"; rm -rf /tmp/audiobook-dry
+    mkdir -p /tmp/audiobook-dry
+    args=(--dry-run --compare "$PWD/books")
+  else
+    unset AUDIOBOOK_BOOKS_DIR
+    args=()
+  fi
+  python3 _pipeline/cli.py process "$pdf" "${args[@]}" --result-file "$result"
+  rc=$?
+  if [ $rc -ne 0 ]; then
+    echo "FAILED: process exited $rc for $pdf. Nothing was committed. The PDF stays where it is."
+    git checkout -q -- . 2>/dev/null; git clean -qfd -- books 2>/dev/null
+    failed=1; continue
+  fi
+  [ "$DRY_RUN" = "true" ] && continue
+
+  bookdir="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['book_dir'])" "$result")"
+  case "$pdf" in
+    _incoming/*)
+      git rm -q -- "$pdf"
+      side="${pdf%.[pP][dD][fF]}.json"; [ -f "$side" ] && git rm -q -- "$side"
+      ;;
+  esac
+  git add -A -- "$bookdir"
+  if git diff --cached --quiet; then echo "No changes to commit for $pdf"; continue; fi
+  rel="${bookdir#"$PWD"/}"
+  git commit -q -m "Audiobook: $rel (from $pdf, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]"
+  git --no-pager show --stat --format='commit %H' HEAD | head -40
+  if push_with_retry; then echo "PUSHED to $BRANCH: $rel"; else echo "FAILED: could not push to $BRANCH"; failed=1; fi
+done
+
+if [ "$DRY_RUN" = "true" ]; then
+  echo; echo "################ push-permission test (throwaway branch, deleted afterwards)"
+  tb="dryrun/run-${GITHUB_RUN_ID:-$$}"
+  git checkout -q -b "$tb"
+  git commit -q --allow-empty -m "dry-run push test [skip ci]"
+  if git push origin "refs/heads/$tb:refs/heads/$tb"; then
+    echo "push of $tb: OK"
+    if git push origin --delete "$tb"; then echo "delete of $tb: OK"; else echo "FAILED: could not delete $tb"; failed=1; fi
+  else
+    echo "FAILED: the job token cannot push. Check Settings > Actions is enabled for the repo and the token has write access."
+    failed=1
+  fi
+fi
+
+[ $failed -eq 0 ] && echo "JOB OK" || echo "JOB FAILED"
+exit $failed