fix(auditing): website NAP via Jina Reader fallback + phone-extraction fix
- website surface: Jina Reader re-render when direct HTML lacks a visible phone (bot-walled 403 or client-side rendered phone) — the NAP cross-check no longer loses its third source - phone regex: lookarounds reject phone-like substrings glued to other digits (JS artifact numbers from raw HTML) - renderer: jina_reader provenance key when fallback fires - --selftest mode (offline check of phone extraction) - VALIDATION.md: gate re-run timestamp (PASS unchanged)
This commit is contained in:
tonyjbala
@@ -50,7 +50,7 @@
|
||||
|
||||
## GATE RESULT
|
||||
|
||||
**PASS** — report_gate.py, 2026-08-15T16:46:21
|
||||
**PASS** — report_gate.py, 2026-08-15T16:52:53
|
||||
|
||||
- Count: 8 stated / 8 material after review — match
|
||||
- Identity: report title + date match raw capture ('Gilmore Heating, Air and Plumbing', 15 August 2026) — match
|
||||
|
||||
@@ -130,40 +130,73 @@ async def scrape_google(query, coords, business=""):
|
||||
return None
|
||||
|
||||
|
||||
def _phone_from_text(text):
|
||||
"""First standalone US phone number in rendered text.
|
||||
|
||||
Lookarounds reject a phone-like substring glued to other digits (the
|
||||
raw-HTML path that previously returned JS artifact numbers).
|
||||
"""
|
||||
m = re.search(r"(?<!\d)(\(\d{3}\)\s*\d{3}[-.]\d{4})(?!\d)", text)
|
||||
if not m:
|
||||
m = re.search(r"(?<!\d)(\d{3}[-.]\d{3}[-.]\d{4})(?!\d)", text)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _jina_markdown(website_url):
|
||||
"""Render a URL to clean text via Jina Reader (no key, free tier)."""
|
||||
import urllib.request
|
||||
req = urllib.request.Request(
|
||||
f"https://r.jina.ai/{website_url}",
|
||||
headers={"User-Agent": "Mozilla/5.0", "Accept": "text/plain"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def fetch_website_data(website_url):
|
||||
"""Extract basic info from business website."""
|
||||
"""Extract basic info from business website.
|
||||
|
||||
Direct fetch for title/description (in <head>, present on JS sites too).
|
||||
Phone is the NAP-critical field: if the direct HTML has no visible phone,
|
||||
re-render via Jina Reader, because business sites usually expose the
|
||||
phone client-side and a false 'no phone' would corrupt the cross-check.
|
||||
"""
|
||||
if not website_url:
|
||||
return None
|
||||
import urllib.request
|
||||
html = None
|
||||
try:
|
||||
req = urllib.request.Request(website_url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
html = resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
data = {"source_url": website_url}
|
||||
|
||||
# Extract title
|
||||
m = re.search(r'<title>([^<]+)</title>', html, re.IGNORECASE)
|
||||
except Exception as e:
|
||||
print(f" [WARN] Website direct fetch failed ({e}); using Jina Reader")
|
||||
data = {"source_url": website_url}
|
||||
if html:
|
||||
m = re.search(r"<title>([^<]+)</title>", html, re.IGNORECASE)
|
||||
if m:
|
||||
data["title"] = m.group(1).strip()
|
||||
|
||||
# Extract meta description
|
||||
m = re.search(r'<meta[^>]*name=["\']description["\'][^>]*content=["\']([^"\']+)["\']', html, re.IGNORECASE)
|
||||
if not m:
|
||||
m = re.search(r'<meta[^>]*content=["\']([^"\']+)["\'][^>]*name=["\']description["\']', html, re.IGNORECASE)
|
||||
if m:
|
||||
data["description"] = m.group(1).strip()
|
||||
|
||||
# Extract phone from page
|
||||
phone_match = re.search(r'(\(?\d{3}\)?[\s-]?\d{3}[\s-]?\d{4})', html)
|
||||
if phone_match:
|
||||
data["phone_on_page"] = phone_match.group(1)
|
||||
|
||||
data["phone_on_page"] = _phone_from_text(html)
|
||||
if data.get("phone_on_page"):
|
||||
return data
|
||||
|
||||
# No phone in direct HTML (or fetch failed) -> render via Jina Reader
|
||||
try:
|
||||
md = _jina_markdown(website_url)
|
||||
except Exception as e:
|
||||
print(f" [FAIL] Website: {e}")
|
||||
return None
|
||||
print(f" [FAIL] Website (direct + Jina): {e}")
|
||||
return data if html else None
|
||||
if not data.get("title"):
|
||||
m = re.search(r"^Title:\s*(.+)$", md, re.MULTILINE)
|
||||
if m:
|
||||
data["title"] = m.group(1).strip()
|
||||
data["phone_on_page"] = _phone_from_text(md)
|
||||
data["renderer"] = "jina_reader"
|
||||
return data
|
||||
|
||||
|
||||
def merge_and_validate(gbp_data=None, apple_data=None, website_data=None, bing_data=None):
|
||||
@@ -354,5 +387,19 @@ async def main():
|
||||
print(f" {key}: {json.dumps(val)}")
|
||||
|
||||
|
||||
def _selftest():
|
||||
"""Runnable check for phone extraction (no network)."""
|
||||
assert _phone_from_text("Call us: (530) 350-9197 today") == "(530) 350-9197"
|
||||
assert _phone_from_text("tel: (530) 350-9197") == "(530) 350-9197"
|
||||
assert _phone_from_text("530-350-9197") == "530-350-9197"
|
||||
# JS artifact: phone-like substring glued to other digits -> reject
|
||||
assert _phone_from_text("var id=1234567890123;") is None
|
||||
assert _phone_from_text("no numbers here") is None
|
||||
print("[SELFTEST] phone extraction: PASS")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
if "--selftest" in sys.argv:
|
||||
_selftest()
|
||||
else:
|
||||
asyncio.run(main())
|
||||
|
||||
Reference in New Issue
Block a user