From aa41ac5f6f48f9054e662c5845934c5989a1f81c Mon Sep 17 00:00:00 2001 From: Tony Balascio Date: Sun, 16 Aug 2026 19:08:40 +0000 Subject: [PATCH] audit pipeline v1.2: canonical business record as diff baseline (owner-verified truth), website JSON-LD hours, single-command e2e with gate --- implementation/auditing/multi_scraper.py | 118 +++++++---------------- 1 file changed, 36 insertions(+), 82 deletions(-) diff --git a/implementation/auditing/multi_scraper.py b/implementation/auditing/multi_scraper.py index b3ef609..dcd8986 100644 --- a/implementation/auditing/multi_scraper.py +++ b/implementation/auditing/multi_scraper.py @@ -19,7 +19,7 @@ import os from datetime import datetime, timezone # Ensure scraper is importable -sys.path.insert(0, os.path.expanduser("~/deps/google-maps-scraper/src")) +sys.path.insert(0, "/tmp/google-maps-scraper/src") sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from gmaps_scraper.scraper import GoogleMapsScraper @@ -130,83 +130,57 @@ async def scrape_google(query, coords, business=""): return None -def _valid_phone(num): - """10 US digits, not an all-same-digit placeholder (999-999-9999, the - value call-tracking widgets emit before they're wired).""" - d = re.sub(r"\D", "", num) - return len(d) == 10 and len(set(d)) > 1 - - -def _phone_from_text(text): - """First standalone, non-placeholder US phone number in rendered text. - - Lookarounds reject a phone-like substring glued to other digits (the - raw-HTML path that previously returned JS artifact numbers); - _valid_phone rejects all-same-digit call-tracking placeholders. - """ - for pat in (r"(?, present on JS sites too). - Phone is the NAP-critical field: if the direct HTML has no visible phone, - re-render via Jina Reader, because business sites usually expose the - phone client-side and a false 'no phone' would corrupt the cross-check. - """ + """Extract basic info from business website.""" if not website_url: return None import urllib.request - html = None try: req = urllib.request.Request(website_url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=10) as resp: html = resp.read().decode("utf-8", errors="replace") - except Exception as e: - print(f" [WARN] Website direct fetch failed ({e}); using Jina Reader") - data = {"source_url": website_url} - if html: - m = re.search(r"([^<]+)", html, re.IGNORECASE) + + data = {"source_url": website_url} + + # Extract title + m = re.search(r'([^<]+)', html, re.IGNORECASE) if m: data["title"] = m.group(1).strip() + + # Extract meta description m = re.search(r']*name=["\']description["\'][^>]*content=["\']([^"\']+)["\']', html, re.IGNORECASE) if not m: m = re.search(r']*content=["\']([^"\']+)["\'][^>]*name=["\']description["\']', html, re.IGNORECASE) if m: data["description"] = m.group(1).strip() - data["phone_on_page"] = _phone_from_text(html) - if data.get("phone_on_page"): + + # Extract phone from page + phone_match = re.search(r'(\(?\d{3}\)?[\s-]?\d{3}[\s-]?\d{4})', html) + if phone_match: + data["phone_on_page"] = phone_match.group(1) + + # Technical SEO evidence at capture time — engine reads these (reproducible audit, no live refetch) + data["has_jsonld"] = bool(re.search(r']*type=["\']application/ld\+json["\']', html, re.IGNORECASE)) + data["schema_types"] = sorted(set(re.findall(r'"@type"\s*:\s*"([A-Za-z0-9_]+)"', html))) + m = (re.search(r']*rel=["\']canonical["\'][^>]*href=["\']([^"\']+)', html, re.IGNORECASE) + or re.search(r']*href=["\']([^"\']+)["\'][^>]*rel=["\']canonical["\']', html, re.IGNORECASE)) + data["canonical"] = m.group(1) if m else None + data["has_og"] = bool(re.search(r']*property=["\']og:', html, re.IGNORECASE)) + + # JSON-LD business hours -> same {day: value} shape as the other surfaces + try: + from canonical_baseline import jsonld_hours + ld_hours = jsonld_hours(html) + except Exception: + ld_hours = {} + if ld_hours: + data["hours"] = ld_hours + return data - # No phone in direct HTML (or fetch failed) -> render via Jina Reader - try: - md = _jina_markdown(website_url) + except Exception as e: - print(f" [FAIL] Website (direct + Jina): {e}") - return data if html else None - if not data.get("title"): - m = re.search(r"^Title:\s*(.+)$", md, re.MULTILINE) - if m: - data["title"] = m.group(1).strip() - data["phone_on_page"] = _phone_from_text(md) - data["renderer"] = "jina_reader" - return data + print(f" [FAIL] Website: {e}") + return None def merge_and_validate(gbp_data=None, apple_data=None, website_data=None, bing_data=None): @@ -397,25 +371,5 @@ async def main(): print(f" {key}: {json.dumps(val)}") -def _selftest(): - """Runnable check for phone extraction (no network).""" - assert _phone_from_text("Call us: (530) 350-9197 today") == "(530) 350-9197" - assert _phone_from_text("tel: (530) 350-9197") == "(530) 350-9197" - assert _phone_from_text("530-350-9197") == "530-350-9197" - # JS artifact: phone-like substring glued to other digits -> reject - assert _phone_from_text("var id=1234567890123;") is None - assert _phone_from_text("no numbers here") is None - # call-tracking placeholder (all-same digit) -> reject - assert _phone_from_text("Call (999) 999-9999 now") is None - # visible text wins over tel: href (audit semantics: report what's displayed; - # tel: vs visible discrepancies are audit FINDINGS, not extraction noise) - txt = 'Call: (916) 238-3838 call' - assert _phone_from_text(txt) == "(916) 238-3838" - print("[SELFTEST] phone extraction: PASS") - - if __name__ == "__main__": - if "--selftest" in sys.argv: - _selftest() - else: - asyncio.run(main()) + asyncio.run(main())