"""Title and author detection. Run: python3 -m unittest discover -s _pipeline/tests -v (from the repo root)""" import json import os import sys import tempfile import unittest HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.join(HERE, "..")) sys.path.insert(0, HERE) import bookio import ingest import meta import settings try: from helpers import make_pdf except ImportError: make_pdf = None TITLE_PAGE = ["THE HINDU-YOGI", "", "Science of Breath", "", "A Complete Manual of the Oriental Breathing Philosophy", "", "By YOGI RAMACHARAKA", "", "THE YOGI PUBLICATION SOCIETY"] @unittest.skipIf(make_pdf is None, "reportlab not installed") class MetaTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.d = self.tmp.name self.cfg = settings.load() self.cfg["work_dir"] = os.path.join(self.d, "work") self.cfg["books_dir"] = os.path.join(self.d, "books") meta.OCR_FUNC = None def tearDown(self): meta.OCR_FUNC = None self.tmp.cleanup() def pdf(self, name, pages, **kw): return make_pdf(os.path.join(self.d, name), pages, **kw) def test_pdf_metadata_title_and_byline_author(self): p = self.pdf("x.pdf", [["Some Title", "", "By Jane Q. Doe"], ["text"]], title="a quiet book") m = meta.resolve(self.cfg, p) self.assertEqual((m["title"], m["author"]), ("A Quiet Book", "Jane Q. Doe")) self.assertEqual(m["title_source"], "pdf-metadata") self.assertTrue(m["author_source"].startswith("first-page-text (by-line")) self.assertEqual(m["warnings"], []) def test_junk_metadata_author_is_ignored_when_page_names_the_author(self): p = self.pdf("x.pdf", [["THE BOOK", "By Glenn Clark"]], title="THE BOOK", author="Bluepen") self.assertEqual(meta.resolve(self.cfg, p)["author"], "Glenn Clark") def test_metadata_author_used_when_page_has_none(self): p = self.pdf("x.pdf", [["Just a page of prose without any credits at all."]], title="Real Title", author="Mary Shelley") m = meta.resolve(self.cfg, p) self.assertEqual((m["author"], m["author_source"]), ("Mary Shelley", "pdf-metadata")) def test_page_title_and_uppercase_byline_without_metadata(self): p = self.pdf("scan0001.pdf", [TITLE_PAGE, ["body"]]) m = meta.resolve(self.cfg, p) self.assertEqual(m["author"], "Yogi Ramacharaka") self.assertTrue(m["title_source"].startswith("first-page-text")) self.assertEqual(m["title"], "The Hindu-yogi") # first block without a file-name hint def test_file_name_hint_picks_the_title_line_on_the_page(self): p = self.pdf("2015.47927.Science-Of-Breath.pdf", [TITLE_PAGE]) m = meta.resolve(self.cfg, p) self.assertEqual((m["title"], m["author"]), ("Science of Breath", "Yogi Ramacharaka")) def test_name_line_under_title_without_by(self): p = self.pdf("x.pdf", [["A Study of Rivers", "Jane Doe", "", "Oxford Press 1921"]]) m = meta.resolve(self.cfg, p) self.assertEqual((m["title"], m["author"]), ("A Study of Rivers", "Jane Doe")) self.assertTrue(m["author_source"].startswith("first-page-text (name-line")) def test_scanned_first_pages_are_read_by_ocr(self): p = self.pdf("2015.47927.Science-Of-Breath.pdf", [[], [], [], [], []]) calls = [] def fake(pdf, first, last, lang): calls.append((first, last)) return [(5, "\n".join(TITLE_PAGE))] + [(i, "") for i in range(first, 5)] meta.OCR_FUNC = fake m = meta.resolve(self.cfg, p) self.assertEqual(calls, [(1, 5)]) self.assertEqual((m["title"], m["author"]), ("Science of Breath", "Yogi Ramacharaka")) self.assertIn("first-page-ocr", m["author_source"]) self.assertIn("first-page-ocr", m["title_source"]) meta.resolve(self.cfg, p) self.assertEqual(len(calls), 1, "the detection from the PDF is cached") def test_nothing_found_uses_file_name_and_unknown_author_with_warning(self): p = self.pdf("2015.47927.Science-Of-Breath.pdf", [[], []]) m = meta.resolve(self.cfg, p) # no tesseract here and no OCR_FUNC: pages stay unreadable self.assertEqual((m["title"], m["author"]), ("Science Of Breath", "Unknown")) self.assertEqual(m["author_source"], "default") self.assertTrue(m["title_source"].startswith("file-name")) kinds = [w[0] for w in m["warnings"]] self.assertIn("metadata-author", kinds) self.assertIn("metadata-title", kinds) self.assertEqual(ingest.author_slug(m["author"]), "unknown") def test_title_from_filename_pattern_with_author(self): p = self.pdf("The Man Who Tapped - by Glenn Clark (Walter Russell).pdf", [[]]) m = meta.resolve(self.cfg, p) self.assertEqual((m["title"], m["author"]), ("The Man Who Tapped", "Glenn Clark")) self.assertEqual(m["author_source"], "file-name (The Man Who Tapped - by Glenn Clark (Walter Russell).pdf)") def test_title_found_author_missing_gives_unknown_and_warning(self): p = self.pdf("x.pdf", [["body text only"]], title="Only A Title") m = meta.resolve(self.cfg, p) self.assertEqual((m["title"], m["author"]), ("Only a Title", "Unknown")) self.assertIn("metadata-author", [w[0] for w in m["warnings"]]) def test_sidecar_and_arguments_win(self): p = self.pdf("x.pdf", [["T", "By Someone Else"]], title="Other") m = meta.resolve(self.cfg, p, sidecar={"title": "Side Title", "author": "Side Author"}) self.assertEqual((m["title"], m["author"], m["title_source"], m["author_source"]), ("Side Title", "Side Author", "sidecar-json", "sidecar-json")) m = meta.resolve(self.cfg, p, title="Arg Title", sidecar={"title": "Side Title", "author": "Side Author"}) self.assertEqual((m["title"], m["title_source"], m["author"]), ("Arg Title", "argument", "Side Author")) def test_url_file_name_hint_is_used_when_the_pdf_name_is_only_an_id(self): p = self.pdf("2401.01234.pdf", [[]]) m = meta.resolve(self.cfg, p, name_hints=["Walden Pond Notes.url"]) self.assertEqual(m["title"], "Walden Pond Notes") self.assertIn("Walden Pond Notes.url", m["title_source"]) def test_never_raises_on_a_file_that_is_not_a_pdf(self): p = os.path.join(self.d, "My Odd_Scan 7.pdf") open(p, "w").write("this is not a pdf") m = meta.resolve(self.cfg, p) self.assertEqual(m["author"], "Unknown") self.assertTrue(m["title"]) def test_ingest_records_sources_and_warnings_in_the_book_folder(self): p = self.pdf("2015.47927.Science-Of-Breath.pdf", [[], []]) self.cfg["ocr"]["enabled"] = False d = ingest.run(self.cfg, p, source_url="https://example.org/a.pdf") book = bookio.read_book(d) self.assertEqual(d, os.path.join(self.cfg["books_dir"], "unknown", "science-of-breath")) self.assertEqual(book["author"], "Unknown") self.assertEqual(book["metadata_sources"]["author"], "default") self.assertTrue(book["metadata_sources"]["title"].startswith("file-name")) self.assertEqual(book["source_url"], "https://example.org/a.pdf") log = open(os.path.join(d, "warnings.log")).read() self.assertIn("ingest-metadata | metadata-author |", log) def test_clark_style_page_gives_the_same_answer_as_before(self): p = self.pdf("c.pdf", [["THE MAN WHO TAPPED THE SECRETS OF THE UNIVERSE"], ["By Glenn Clark"]], title="THE MAN WHO TAPPED THE SECRETS OF THE UNIVERSE", author="Bluepen") m = meta.resolve(self.cfg, p) self.assertEqual((m["title"], m["author"]), ("The Man Who Tapped the Secrets of the Universe", "Glenn Clark")) class PureTests(unittest.TestCase): def test_cleaning_file_names(self): self.assertEqual(meta.clean_filename_title("2015.47927.Science-Of-Breath"), "Science Of Breath") self.assertEqual(meta.clean_filename_title("my_book_final"), "my book") self.assertEqual(meta.filename_candidates(["2401.01234.pdf", "Deep Work.url"])[0], "Deep Work") def test_library_style_metadata_author(self): self.assertEqual(meta.info_author_name("Lincoln, Abraham, 1809-1865"), "Abraham Lincoln") self.assertEqual(meta.info_author_name("Pope, Alexander, 1688-1744"), "Alexander Pope") self.assertEqual(meta.info_author_name("Jane Doe"), "Jane Doe") self.assertEqual(meta.info_author_name("Doe, Jane Q."), "Jane Q. Doe") self.assertEqual(meta.info_author_name("Acme Press, Inc."), "Acme Press, Inc.") def test_author_rules(self): self.assertEqual(meta.first_author("YOGI RAMACHARAKA"), "Yogi Ramacharaka") self.assertEqual(meta.first_author("Glenn Clark and Walter Russell"), "Glenn Clark") self.assertIsNone(meta.first_author("the sea and the sky")) self.assertIsNone(meta.first_author("The Yogi Publication Society")) if __name__ == "__main__": unittest.main()