Metadata never fails a drop: deterministic title/author detection (metadata, first-page text, OCR of scanned first pages, file name), sources recorded in book.json; decrypt scratch copy before OCR
This commit is contained in:
Audiobook Producer
@@ -0,0 +1,26 @@
|
||||
"""Small PDF builder for the metadata and fetch tests (needs reportlab, which the CI unit-test step installs)."""
|
||||
from reportlab.lib.pagesizes import letter
|
||||
from reportlab.pdfgen import canvas
|
||||
|
||||
|
||||
def make_pdf(path, pages, title=None, author=None):
|
||||
"""pages: list of lists of text lines; an empty list = a page with only a picture-like box (no text layer)."""
|
||||
c = canvas.Canvas(path, pagesize=letter)
|
||||
if title:
|
||||
c.setTitle(title)
|
||||
if author:
|
||||
c.setAuthor(author)
|
||||
for lines in pages:
|
||||
y = 700
|
||||
if not lines:
|
||||
c.rect(100, 100, 300, 400, fill=1)
|
||||
for ln in lines:
|
||||
if ln == "":
|
||||
y -= 30
|
||||
continue
|
||||
c.setFont("Helvetica", 18)
|
||||
c.drawString(100, y, ln)
|
||||
y -= 24
|
||||
c.showPage()
|
||||
c.save()
|
||||
return path
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Title and author detection. Run: python3 -m unittest discover -s _pipeline/tests -v (from the repo root)"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
import bookio
|
||||
import ingest
|
||||
import meta
|
||||
import settings
|
||||
|
||||
try:
|
||||
from helpers import make_pdf
|
||||
except ImportError:
|
||||
make_pdf = None
|
||||
|
||||
TITLE_PAGE = ["THE HINDU-YOGI", "", "Science of Breath", "", "A Complete Manual of the Oriental Breathing Philosophy", "", "By YOGI RAMACHARAKA", "", "THE YOGI PUBLICATION SOCIETY"]
|
||||
|
||||
|
||||
@unittest.skipIf(make_pdf is None, "reportlab not installed")
|
||||
class MetaTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.d = self.tmp.name
|
||||
self.cfg = settings.load()
|
||||
self.cfg["work_dir"] = os.path.join(self.d, "work")
|
||||
self.cfg["books_dir"] = os.path.join(self.d, "books")
|
||||
meta.OCR_FUNC = None
|
||||
|
||||
def tearDown(self):
|
||||
meta.OCR_FUNC = None
|
||||
self.tmp.cleanup()
|
||||
|
||||
def pdf(self, name, pages, **kw):
|
||||
return make_pdf(os.path.join(self.d, name), pages, **kw)
|
||||
|
||||
def test_pdf_metadata_title_and_byline_author(self):
|
||||
p = self.pdf("x.pdf", [["Some Title", "", "By Jane Q. Doe"], ["text"]], title="a quiet book")
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["title"], m["author"]), ("A Quiet Book", "Jane Q. Doe"))
|
||||
self.assertEqual(m["title_source"], "pdf-metadata")
|
||||
self.assertTrue(m["author_source"].startswith("first-page-text (by-line"))
|
||||
self.assertEqual(m["warnings"], [])
|
||||
|
||||
def test_junk_metadata_author_is_ignored_when_page_names_the_author(self):
|
||||
p = self.pdf("x.pdf", [["THE BOOK", "By Glenn Clark"]], title="THE BOOK", author="Bluepen")
|
||||
self.assertEqual(meta.resolve(self.cfg, p)["author"], "Glenn Clark")
|
||||
|
||||
def test_metadata_author_used_when_page_has_none(self):
|
||||
p = self.pdf("x.pdf", [["Just a page of prose without any credits at all."]], title="Real Title", author="Mary Shelley")
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["author"], m["author_source"]), ("Mary Shelley", "pdf-metadata"))
|
||||
|
||||
def test_page_title_and_uppercase_byline_without_metadata(self):
|
||||
p = self.pdf("scan0001.pdf", [TITLE_PAGE, ["body"]])
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual(m["author"], "Yogi Ramacharaka")
|
||||
self.assertTrue(m["title_source"].startswith("first-page-text"))
|
||||
self.assertEqual(m["title"], "The Hindu-yogi") # first block without a file-name hint
|
||||
|
||||
def test_file_name_hint_picks_the_title_line_on_the_page(self):
|
||||
p = self.pdf("2015.47927.Science-Of-Breath.pdf", [TITLE_PAGE])
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["title"], m["author"]), ("Science of Breath", "Yogi Ramacharaka"))
|
||||
|
||||
def test_name_line_under_title_without_by(self):
|
||||
p = self.pdf("x.pdf", [["A Study of Rivers", "Jane Doe", "", "Oxford Press 1921"]])
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["title"], m["author"]), ("A Study of Rivers", "Jane Doe"))
|
||||
self.assertTrue(m["author_source"].startswith("first-page-text (name-line"))
|
||||
|
||||
def test_scanned_first_pages_are_read_by_ocr(self):
|
||||
p = self.pdf("2015.47927.Science-Of-Breath.pdf", [[], [], [], [], []])
|
||||
calls = []
|
||||
|
||||
def fake(pdf, first, last, lang):
|
||||
calls.append((first, last))
|
||||
return [(5, "\n".join(TITLE_PAGE))] + [(i, "") for i in range(first, 5)]
|
||||
meta.OCR_FUNC = fake
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual(calls, [(1, 5)])
|
||||
self.assertEqual((m["title"], m["author"]), ("Science of Breath", "Yogi Ramacharaka"))
|
||||
self.assertIn("first-page-ocr", m["author_source"])
|
||||
self.assertIn("first-page-ocr", m["title_source"])
|
||||
meta.resolve(self.cfg, p)
|
||||
self.assertEqual(len(calls), 1, "the detection from the PDF is cached")
|
||||
|
||||
def test_nothing_found_uses_file_name_and_unknown_author_with_warning(self):
|
||||
p = self.pdf("2015.47927.Science-Of-Breath.pdf", [[], []])
|
||||
m = meta.resolve(self.cfg, p) # no tesseract here and no OCR_FUNC: pages stay unreadable
|
||||
self.assertEqual((m["title"], m["author"]), ("Science Of Breath", "Unknown"))
|
||||
self.assertEqual(m["author_source"], "default")
|
||||
self.assertTrue(m["title_source"].startswith("file-name"))
|
||||
kinds = [w[0] for w in m["warnings"]]
|
||||
self.assertIn("metadata-author", kinds)
|
||||
self.assertIn("metadata-title", kinds)
|
||||
self.assertEqual(ingest.author_slug(m["author"]), "unknown")
|
||||
|
||||
def test_title_from_filename_pattern_with_author(self):
|
||||
p = self.pdf("The Man Who Tapped - by Glenn Clark (Walter Russell).pdf", [[]])
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["title"], m["author"]), ("The Man Who Tapped", "Glenn Clark"))
|
||||
self.assertEqual(m["author_source"], "file-name (The Man Who Tapped - by Glenn Clark (Walter Russell).pdf)")
|
||||
|
||||
def test_title_found_author_missing_gives_unknown_and_warning(self):
|
||||
p = self.pdf("x.pdf", [["body text only"]], title="Only A Title")
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["title"], m["author"]), ("Only a Title", "Unknown"))
|
||||
self.assertIn("metadata-author", [w[0] for w in m["warnings"]])
|
||||
|
||||
def test_sidecar_and_arguments_win(self):
|
||||
p = self.pdf("x.pdf", [["T", "By Someone Else"]], title="Other")
|
||||
m = meta.resolve(self.cfg, p, sidecar={"title": "Side Title", "author": "Side Author"})
|
||||
self.assertEqual((m["title"], m["author"], m["title_source"], m["author_source"]), ("Side Title", "Side Author", "sidecar-json", "sidecar-json"))
|
||||
m = meta.resolve(self.cfg, p, title="Arg Title", sidecar={"title": "Side Title", "author": "Side Author"})
|
||||
self.assertEqual((m["title"], m["title_source"], m["author"]), ("Arg Title", "argument", "Side Author"))
|
||||
|
||||
def test_url_file_name_hint_is_used_when_the_pdf_name_is_only_an_id(self):
|
||||
p = self.pdf("2401.01234.pdf", [[]])
|
||||
m = meta.resolve(self.cfg, p, name_hints=["Walden Pond Notes.url"])
|
||||
self.assertEqual(m["title"], "Walden Pond Notes")
|
||||
self.assertIn("Walden Pond Notes.url", m["title_source"])
|
||||
|
||||
def test_never_raises_on_a_file_that_is_not_a_pdf(self):
|
||||
p = os.path.join(self.d, "My Odd_Scan 7.pdf")
|
||||
open(p, "w").write("this is not a pdf")
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual(m["author"], "Unknown")
|
||||
self.assertTrue(m["title"])
|
||||
|
||||
def test_ingest_records_sources_and_warnings_in_the_book_folder(self):
|
||||
p = self.pdf("2015.47927.Science-Of-Breath.pdf", [[], []])
|
||||
self.cfg["ocr"]["enabled"] = False
|
||||
d = ingest.run(self.cfg, p, source_url="https://example.org/a.pdf")
|
||||
book = bookio.read_book(d)
|
||||
self.assertEqual(d, os.path.join(self.cfg["books_dir"], "unknown", "science-of-breath"))
|
||||
self.assertEqual(book["author"], "Unknown")
|
||||
self.assertEqual(book["metadata_sources"]["author"], "default")
|
||||
self.assertTrue(book["metadata_sources"]["title"].startswith("file-name"))
|
||||
self.assertEqual(book["source_url"], "https://example.org/a.pdf")
|
||||
log = open(os.path.join(d, "warnings.log")).read()
|
||||
self.assertIn("ingest-metadata | metadata-author |", log)
|
||||
|
||||
def test_clark_style_page_gives_the_same_answer_as_before(self):
|
||||
p = self.pdf("c.pdf", [["THE MAN WHO TAPPED THE SECRETS OF THE UNIVERSE"], ["By Glenn Clark"]], title="THE MAN WHO TAPPED THE SECRETS OF THE UNIVERSE", author="Bluepen")
|
||||
m = meta.resolve(self.cfg, p)
|
||||
self.assertEqual((m["title"], m["author"]), ("The Man Who Tapped the Secrets of the Universe", "Glenn Clark"))
|
||||
|
||||
|
||||
class PureTests(unittest.TestCase):
|
||||
def test_cleaning_file_names(self):
|
||||
self.assertEqual(meta.clean_filename_title("2015.47927.Science-Of-Breath"), "Science Of Breath")
|
||||
self.assertEqual(meta.clean_filename_title("my_book_final"), "my book")
|
||||
self.assertEqual(meta.filename_candidates(["2401.01234.pdf", "Deep Work.url"])[0], "Deep Work")
|
||||
|
||||
def test_author_rules(self):
|
||||
self.assertEqual(meta.first_author("YOGI RAMACHARAKA"), "Yogi Ramacharaka")
|
||||
self.assertEqual(meta.first_author("Glenn Clark and Walter Russell"), "Glenn Clark")
|
||||
self.assertIsNone(meta.first_author("the sea and the sky"))
|
||||
self.assertIsNone(meta.first_author("The Yogi Publication Society"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user