Audiobook Producer
118 lines
5.2 KiB
Bash
Executable File
118 lines
5.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The whole job, in one place. Inputs (environment):
|
|
# DRY_RUN true|false (true: text stages only, voice reachability, push-permission test, commits nothing to the branch)
|
|
# PDF_INPUT optional path relative to the repo root (manual runs). Empty = every PDF waiting in _incoming/
|
|
# BRANCH branch to commit back to
|
|
# AUDIOBOOK_* setting overrides (see _pipeline/settings.py)
|
|
set -uo pipefail
|
|
cd "${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}"
|
|
export PATH="$PWD/.venv/bin:$PATH"
|
|
DRY_RUN="${DRY_RUN:-false}"
|
|
BRANCH="${BRANCH:-main}"
|
|
git config user.name "audiobook-bot"
|
|
git config user.email "audiobook-bot@noreply.invalid"
|
|
|
|
# The checkout is sparse (code and _incoming only). Books are pulled in only when needed.
|
|
sparse_add() { printf '%s\n' "$@" | git sparse-checkout add --stdin; }
|
|
esc() { printf '%s' "$1" | sed -e 's/[][*?\\#!]/\\&/g'; }
|
|
|
|
pdfs=()
|
|
if [ -n "${PDF_INPUT:-}" ]; then
|
|
sparse_add "/$(esc "$PDF_INPUT")"
|
|
[ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; }
|
|
pdfs=("$PDF_INPUT")
|
|
else
|
|
while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f -iname '*.pdf' | sort)
|
|
fi
|
|
CLARK_PDF="books/clark-glenn/the-man-who-tapped-the-secrets-of-the-universe/The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf"
|
|
if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then
|
|
echo "Dry run with nothing in _incoming/: using the Clark book as a regression sample."
|
|
sparse_add "/$(esc "$CLARK_PDF")"
|
|
pdfs=("$CLARK_PDF")
|
|
fi
|
|
if [ "${#pdfs[@]}" -eq 0 ]; then
|
|
echo "Nothing to do: no PDF in _incoming/."
|
|
exit 0
|
|
fi
|
|
echo "PDFs to process: ${#pdfs[@]}"; printf ' %s\n' "${pdfs[@]}"
|
|
|
|
push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and try again
|
|
local i
|
|
for i in 1 2 3 4 5; do
|
|
if git push origin "HEAD:refs/heads/$BRANCH"; then return 0; fi
|
|
echo "push attempt $i failed; fetching and rebasing"
|
|
git fetch -q --depth=100 origin "$BRANCH" && git rebase FETCH_HEAD || { git rebase --abort 2>/dev/null; return 1; }
|
|
done
|
|
return 1
|
|
}
|
|
|
|
failed=0
|
|
for pdf in "${pdfs[@]}"; do
|
|
echo; echo "################ $pdf"
|
|
result="$(mktemp)"
|
|
rel="$(python3 _pipeline/cli.py bookpath "$pdf")" || { echo "FAILED: $rel"; failed=1; continue; }
|
|
echo "book folder: $rel"
|
|
e="$(esc "$rel")"
|
|
if git cat-file -e "HEAD:$rel" 2>/dev/null; then
|
|
echo "this book already exists in the repo"
|
|
if [ "$DRY_RUN" = "true" ]; then # dry run compares text only: skip audio
|
|
sparse_add "/$e/book.json" "/$e/warnings.log" "/$e/LICENSE_STATUS.md" "/$e/text/" "/$e/script/" "/$e/figures/"
|
|
else # a real run needs the existing audio so it can skip it
|
|
sparse_add "/$e/"
|
|
fi
|
|
elif [ "$DRY_RUN" != "true" ]; then
|
|
sparse_add "/$e/" # new book: its files must be inside the sparse checkout or git add ignores them
|
|
fi
|
|
if [ "$DRY_RUN" = "true" ]; then
|
|
export AUDIOBOOK_BOOKS_DIR="/tmp/audiobook-dry/books"; rm -rf /tmp/audiobook-dry
|
|
mkdir -p /tmp/audiobook-dry
|
|
args=(--dry-run --compare "$PWD/books")
|
|
else
|
|
unset AUDIOBOOK_BOOKS_DIR
|
|
args=()
|
|
fi
|
|
python3 _pipeline/cli.py process "$pdf" "${args[@]}" --result-file "$result"
|
|
rc=$?
|
|
if [ $rc -ne 0 ]; then
|
|
echo "FAILED: process exited $rc for $pdf. Nothing was committed. The PDF stays where it is."
|
|
git checkout -q -- . 2>/dev/null; git clean -qfd -- books 2>/dev/null
|
|
failed=1; continue
|
|
fi
|
|
[ "$DRY_RUN" = "true" ] && continue
|
|
|
|
bookdir="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['book_dir'])" "$result")"
|
|
rel="${bookdir#"$PWD"/}"
|
|
git add -A -- "$bookdir" || { echo "FAILED: git add failed for $rel. Nothing was committed."; failed=1; continue; }
|
|
# Safety net: the book must really be staged, or the PDF must not be removed from _incoming/.
|
|
if ! git ls-files --error-unmatch -- "$rel/book.json" >/dev/null 2>&1; then
|
|
echo "FAILED: $rel/book.json is not tracked after git add. Nothing was committed."; failed=1; continue
|
|
fi
|
|
case "$pdf" in
|
|
_incoming/*)
|
|
git rm -q -- "$pdf"
|
|
side="${pdf%.[pP][dD][fF]}.json"; [ -f "$side" ] && git rm -q -- "$side"
|
|
;;
|
|
esac
|
|
if git diff --cached --quiet; then echo "No changes to commit for $pdf"; continue; fi
|
|
git commit -q -m "Audiobook: $rel (from $pdf, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]"
|
|
git --no-pager show --stat --format='commit %H' HEAD | head -40
|
|
if push_with_retry; then echo "PUSHED to $BRANCH: $rel"; else echo "FAILED: could not push to $BRANCH"; failed=1; fi
|
|
done
|
|
|
|
if [ "$DRY_RUN" = "true" ]; then
|
|
echo; echo "################ push-permission test (throwaway branch, deleted afterwards)"
|
|
tb="dryrun/run-${GITHUB_RUN_ID:-$$}"
|
|
git checkout -q -b "$tb"
|
|
git commit -q --allow-empty -m "dry-run push test [skip ci]"
|
|
if git push origin "refs/heads/$tb:refs/heads/$tb"; then
|
|
echo "push of $tb: OK"
|
|
if git push origin --delete "$tb"; then echo "delete of $tb: OK"; else echo "FAILED: could not delete $tb"; failed=1; fi
|
|
else
|
|
echo "FAILED: the job token cannot push. Check Settings > Actions is enabled for the repo and the token has write access."
|
|
failed=1
|
|
fi
|
|
fi
|
|
|
|
[ $failed -eq 0 ] && echo "JOB OK" || echo "JOB FAILED"
|
|
exit $failed
|