From 53d21ea2e59bcf3dfe0e27eea5a699f2d3733c5f Mon Sep 17 00:00:00 2001 From: Audiobook Producer Date: Sun, 18 Oct 2026 01:34:58 -0500 Subject: [PATCH] Add Gitea Actions workflow: _incoming/*.pdf drop-folder pipeline, with dry-run dispatch --- .gitea/workflows/audiobook.yml | 59 ++++++++++++++++++++++ _pipeline/ci/install.sh | 11 +++++ _pipeline/ci/run.sh | 89 ++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 .gitea/workflows/audiobook.yml create mode 100755 _pipeline/ci/install.sh create mode 100755 _pipeline/ci/run.sh diff --git a/.gitea/workflows/audiobook.yml b/.gitea/workflows/audiobook.yml new file mode 100644 index 0000000..6db3cf0 --- /dev/null +++ b/.gitea/workflows/audiobook.yml @@ -0,0 +1,59 @@ +# Drop a PDF in _incoming/ and commit it. This job turns it into a finished audiobook folder and commits that back. +# No bot is involved: every step is a deterministic script in _pipeline/. +name: audiobook + +on: + push: + branches: [main, 'test/**'] # test/** lets the pipeline be proven on a branch without touching main + paths: ['_incoming/*.pdf'] # only a PDF landing in _incoming/ starts a run + workflow_dispatch: + inputs: + pdf: + description: 'PDF path inside the repo (empty = every PDF waiting in _incoming/)' + required: false + default: 'books/clark-glenn/the-man-who-tapped-the-secrets-of-the-universe/The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf' + dry_run: + description: 'true = text stages only, voice reachability check, push-permission test, commit nothing' + type: boolean + default: true + +permissions: + contents: write + +concurrency: + group: audiobook + cancel-in-progress: false # one book at a time; a second drop waits + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 720 # long books render for hours + # The commit-back commit carries [skip ci] and removes the PDF from _incoming/, so it cannot start another run. + if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} + env: + AUDIOBOOK_VOICE_ENDPOINT: ${{ vars.AUDIOBOOK_VOICE_ENDPOINT || 'http://192.168.1.202:8090/tts' }} + AUDIOBOOK_WORK_DIR: /tmp/audiobook-work + AUDIOBOOK_REQUIRE_OCR: '1' + BRANCH: ${{ github.ref_name }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'false' }} + PDF_INPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.pdf || '' }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install tools (fresh container every run) + run: bash _pipeline/ci/install.sh + - name: Unit tests (dry runs only) + if: ${{ env.DRY_RUN == 'true' }} + run: | + .venv/bin/pip install -q --disable-pip-version-check reportlab pillow + .venv/bin/python -W ignore -m unittest discover -s _pipeline/tests 2>&1 | grep -v -E '^\s+(data|read) =' | tail -40 + test "${PIPESTATUS[0]}" -eq 0 + - name: Voice server reachable from this job + run: | + echo "voice endpoint: $AUDIOBOOK_VOICE_ENDPOINT" + base="${AUDIOBOOK_VOICE_ENDPOINT%/tts}" + curl -sS --max-time 20 "$base/api/model-info" && echo + - name: Process + run: bash _pipeline/ci/run.sh diff --git a/_pipeline/ci/install.sh b/_pipeline/ci/install.sh new file mode 100755 index 0000000..ddab8ba --- /dev/null +++ b/_pipeline/ci/install.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Installs everything the job needs in a FRESH container (node:20-bookworm). Safe to run twice. +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq --no-install-recommends ffmpeg poppler-utils tesseract-ocr tesseract-ocr-eng ocrmypdf ghostscript \ + python3 python3-venv python3-pip git curl ca-certificates +python3 -m venv .venv +.venv/bin/pip install -q --disable-pip-version-check -r _pipeline/requirements.txt +echo "--- tool versions" +python3 --version; ffmpeg -version | head -1; tesseract --version 2>&1 | head -1; ocrmypdf --version; pdftotext -v 2>&1 | head -1 diff --git a/_pipeline/ci/run.sh b/_pipeline/ci/run.sh new file mode 100755 index 0000000..34968b2 --- /dev/null +++ b/_pipeline/ci/run.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# The whole job, in one place. Inputs (environment): +# DRY_RUN true|false (true: text stages only, voice reachability, push-permission test, commits nothing to the branch) +# PDF_INPUT optional path relative to the repo root (manual runs). Empty = every PDF waiting in _incoming/ +# BRANCH branch to commit back to +# AUDIOBOOK_* setting overrides (see _pipeline/settings.py) +set -uo pipefail +cd "${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +export PATH="$PWD/.venv/bin:$PATH" +DRY_RUN="${DRY_RUN:-false}" +BRANCH="${BRANCH:-main}" +git config user.name "audiobook-bot" +git config user.email "audiobook-bot@noreply.invalid" + +pdfs=() +if [ -n "${PDF_INPUT:-}" ]; then + [ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; } + pdfs=("$PDF_INPUT") +else + while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f -iname '*.pdf' | sort) +fi +if [ "${#pdfs[@]}" -eq 0 ]; then + echo "Nothing to do: no PDF in _incoming/." + exit 0 +fi +echo "PDFs to process: ${#pdfs[@]}"; printf ' %s\n' "${pdfs[@]}" + +push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and try again + local i + for i in 1 2 3 4 5; do + if git push origin "HEAD:refs/heads/$BRANCH"; then return 0; fi + echo "push attempt $i failed; fetching and rebasing" + git pull --rebase origin "$BRANCH" || return 1 + done + return 1 +} + +failed=0 +for pdf in "${pdfs[@]}"; do + echo; echo "################ $pdf" + result="$(mktemp)" + if [ "$DRY_RUN" = "true" ]; then + export AUDIOBOOK_BOOKS_DIR="/tmp/audiobook-dry/books"; rm -rf /tmp/audiobook-dry + mkdir -p /tmp/audiobook-dry + args=(--dry-run --compare "$PWD/books") + else + unset AUDIOBOOK_BOOKS_DIR + args=() + fi + python3 _pipeline/cli.py process "$pdf" "${args[@]}" --result-file "$result" + rc=$? + if [ $rc -ne 0 ]; then + echo "FAILED: process exited $rc for $pdf. Nothing was committed. The PDF stays where it is." + git checkout -q -- . 2>/dev/null; git clean -qfd -- books 2>/dev/null + failed=1; continue + fi + [ "$DRY_RUN" = "true" ] && continue + + bookdir="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['book_dir'])" "$result")" + case "$pdf" in + _incoming/*) + git rm -q -- "$pdf" + side="${pdf%.[pP][dD][fF]}.json"; [ -f "$side" ] && git rm -q -- "$side" + ;; + esac + git add -A -- "$bookdir" + if git diff --cached --quiet; then echo "No changes to commit for $pdf"; continue; fi + rel="${bookdir#"$PWD"/}" + git commit -q -m "Audiobook: $rel (from $pdf, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]" + git --no-pager show --stat --format='commit %H' HEAD | head -40 + if push_with_retry; then echo "PUSHED to $BRANCH: $rel"; else echo "FAILED: could not push to $BRANCH"; failed=1; fi +done + +if [ "$DRY_RUN" = "true" ]; then + echo; echo "################ push-permission test (throwaway branch, deleted afterwards)" + tb="dryrun/run-${GITHUB_RUN_ID:-$$}" + git checkout -q -b "$tb" + git commit -q --allow-empty -m "dry-run push test [skip ci]" + if git push origin "refs/heads/$tb:refs/heads/$tb"; then + echo "push of $tb: OK" + if git push origin --delete "$tb"; then echo "delete of $tb: OK"; else echo "FAILED: could not delete $tb"; failed=1; fi + else + echo "FAILED: the job token cannot push. Check Settings > Actions is enabled for the repo and the token has write access." + failed=1 + fi +fi + +[ $failed -eq 0 ] && echo "JOB OK" || echo "JOB FAILED" +exit $failed