Ty_Tech/old-books
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop a link: .url/.txt in _incoming/ is downloaded, verified as a PDF, stored as the book's source PDF; workflow also triggers on link files

Browse Source
This commit is contained in:
Audiobook Producer
2026-10-18 13:25:01 -05:00
parent b72c0464ef
commit 9287ea8e16
6 changed files with 406 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_pipeline/ci/run.sh
+31 -15
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env bash
# The whole job, in one place. Inputs (environment):
# DRY_RUN true|false (true: text stages only, voice reachability, push-permission test, commits nothing to the branch)
# PDF_INPUT optional path relative to the repo root (manual runs). Empty = every PDF waiting in _incoming/
# PDF_INPUT optional path relative to the repo root (manual runs): a PDF or a .url/.txt link file.
# Empty = every PDF, .url and .txt file waiting in _incoming/
# BRANCH branch to commit back to
# AUDIOBOOK_* setting overrides (see _pipeline/settings.py)
set -uo pipefail
@@ -22,7 +23,7 @@ if [ -n "${PDF_INPUT:-}" ]; then
[ -f "$PDF_INPUT" ] || { echo "FAILED: no such file in the repo: $PDF_INPUT"; exit 1; }
pdfs=("$PDF_INPUT")
else
while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f -iname '*.pdf' | sort)
while IFS= read -r f; do pdfs+=("$f"); done < <(find _incoming -maxdepth 1 -type f \( -iname '*.pdf' -o -iname '*.url' -o -iname '*.txt' \) | sort)
fi
CLARK_PDF="books/clark-glenn/the-man-who-tapped-the-secrets-of-the-universe/The Man Who Tapped the Secrets of the Universe - by Glenn Clark (Walter Russell).pdf"
if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then
@@ -31,10 +32,10 @@ if [ "${#pdfs[@]}" -eq 0 ] && [ "$DRY_RUN" = "true" ]; then
pdfs=("$CLARK_PDF")
fi
if [ "${#pdfs[@]}" -eq 0 ]; then
echo "Nothing to do: no PDF in _incoming/."
echo "Nothing to do: no PDF or link file (.url/.txt) in _incoming/."
exit 0
fi
echo "PDFs to process: ${#pdfs[@]}"; printf ' %s\n' "${pdfs[@]}"
echo "Items to process: ${#pdfs[@]}"; printf ' %s\n' "${pdfs[@]}"
push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and try again
local i
@@ -47,10 +48,25 @@ push_with_retry() { # push HEAD to the branch; if the branch moved, rebase and
}
failed=0
for pdf in "${pdfs[@]}"; do
echo; echo "################ $pdf"
n=0
for item in "${pdfs[@]}"; do
n=$((n+1))
pdf="$item"; extra=(); hints=()
echo; echo "################ $item"
result="$(mktemp)"
rel="$(python3 _pipeline/cli.py bookpath "$pdf")" || { echo "FAILED: $rel"; failed=1; continue; }
case "${item,,}" in
*.url|*.txt) # a link to a PDF: download it into the scratch folder, check it is a real PDF, then treat it like a dropped PDF
fdir="/tmp/audiobook-fetch/$n"; rm -rf "$fdir"; mkdir -p "$fdir"
if ! python3 _pipeline/cli.py fetch "$item" --out "$fdir"; then
echo "FAILED: could not get the PDF for $item (reason above). Nothing was committed. The link file stays where it is."
failed=1; continue
fi
pdf="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['pdf'])" "$fdir/fetch.json")"
surl="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['source_url'])" "$fdir/fetch.json")"
hints=(--name-hint "$(basename "$item")"); extra=(--source-url "$surl" "${hints[@]}")
;;
esac
rel="$(python3 _pipeline/cli.py bookpath "$pdf" "${hints[@]}")" || { echo "FAILED: $rel"; failed=1; continue; }
echo "book folder: $rel"
e="$(esc "$rel")"
if git cat-file -e "HEAD:$rel" 2>/dev/null; then
@@ -71,10 +87,10 @@ for pdf in "${pdfs[@]}"; do
unset AUDIOBOOK_BOOKS_DIR
args=()
fi
python3 _pipeline/cli.py process "$pdf" "${args[@]}" --result-file "$result"
python3 _pipeline/cli.py process "$pdf" "${args[@]}" "${extra[@]}" --result-file "$result"
rc=$?
if [ $rc -ne 0 ]; then
echo "FAILED: process exited $rc for $pdf. Nothing was committed. The PDF stays where it is."
echo "FAILED: process exited $rc for $item. Nothing was committed. The file stays where it is in _incoming/."
git checkout -q -- . 2>/dev/null; git clean -qfd -- books 2>/dev/null
failed=1; continue
fi
@@ -87,14 +103,14 @@ for pdf in "${pdfs[@]}"; do
if ! git ls-files --error-unmatch -- "$rel/book.json" >/dev/null 2>&1; then
echo "FAILED: $rel/book.json is not tracked after git add. Nothing was committed."; failed=1; continue
fi
case "$pdf" in
_incoming/*)
git rm -q -- "$pdf"
side="${pdf%.[pP][dD][fF]}.json"; [ -f "$side" ] && git rm -q -- "$side"
case "$item" in
_incoming/*) # the dropped PDF or link file (and its optional .json) leaves _incoming/ in the same commit
git rm -q -- "$item"
side="${item%.*}.json"; [ -f "$side" ] && git rm -q -- "$side"
;;
esac
if git diff --cached --quiet; then echo "No changes to commit for $pdf"; continue; fi
git commit -q -m "Audiobook: $rel (from $pdf, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]"
if git diff --cached --quiet; then echo "No changes to commit for $item"; continue; fi
git commit -q -m "Audiobook: $rel (from $item, Actions run ${GITHUB_RUN_NUMBER:-?}) [skip ci]"
git --no-pager show --stat --format='commit %H' HEAD | head -40
if push_with_retry; then echo "PUSHED to $BRANCH: $rel"; else echo "FAILED: could not push to $BRANCH"; failed=1; fi
done