Tony_tech/veripath
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multi_scraper: fingerprint website build stack + SSR signal (DR-003)

Browse Source
This commit is contained in:
Tony Balascio
2026-08-17 16:46:29 +00:00
parent 5341f3f965
commit 708684bb74
1 changed files with 84 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
implementation/auditing/multi_scraper.py
+83 -3
View File
@@ -22,9 +22,6 @@ from datetime import datetime, timezone
sys.path.insert(0, "/tmp/google-maps-scraper/src") sys.path.insert(0, "/tmp/google-maps-scraper/src")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from gmaps_scraper.scraper import GoogleMapsScraper
from apple_scraper import scrape_apple
from bing_provider import scrape_bing
BASE_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -130,6 +127,60 @@ async def scrape_google(query, coords, business=""):
return None return None
STACK_SIGNATURES = {
# AI app builders (client-built sites)
"lovable": r"lovable\.dev|/~flock\.js",
"v0": r"v0\.dev|__v0",
"bolt": r"stackblitz\.dev|bolt\b",
# Frameworks
"nextjs": r"__next|/_next/",
"nuxt": r"__nuxt",
"gatsby": r"gatsby",
"vite": r"/@vite/|vite/client|/_assets/",
"react": r"react",
"angular": r"angular",
"vue": r"vue",
"svelte": r"svelte",
# Page builders / CMS / ecommerce
"wordpress": r"wp-content|wp-includes|wordpress",
"squarespace": r"squarespace",
"wix": r"wix\.com|wixstatic",
"webflow": r"webflow",
"shopify": r"shopify|cdn\.shopify",
"bigcommerce": r"bigcommerce",
# Hosts (weak signal only)
"vercel": r"vercel|_next/static",
"netlify": r"netlify",
"cloudflare_pages": r"pages\.dev|cf-cache-status",
"github_pages": r"ghs\.|github\.io",
}
WEBSITE_STACK_ORDER = ["lovable", "v0", "bolt", "nextjs", "nuxt", "gatsby", "vite", "react",
"angular", "vue", "svelte", "wordpress", "squarespace", "wix",
"webflow", "shopify", "bigcommerce", "vercel", "netlify",
"cloudflare_pages", "github_pages"]
def fingerprint_website(html):
"""Fingerprint the website build stack + SSR signal.
ponytail: regex fingerprints on raw HTML — good enough for a known-site
audit; false-negatives land in 'unknown', never wrong with confidence.
"""
low = html.lower()
detected = [k for k in WEBSITE_STACK_ORDER if re.search(STACK_SIGNATURES[k], low)]
prose = re.sub(r"<script.*?</script>|<style.*?</style>|<[^>]+>", " ", html, flags=re.S)
word_count = len(prose.split())
return {
"detected": detected,
"primary": detected[0] if detected else "unknown",
# SSR signal: word count a non-JS crawler (plain HTTP fetch) can read.
# <50 = content rendered client-side; AI crawlers that skip JS see ~nothing.
"html_word_count": word_count,
"ssr_signal": "weak" if word_count < 50 else "present",
}
def fetch_website_data(website_url): def fetch_website_data(website_url):
"""Extract basic info from business website.""" """Extract basic info from business website."""
if not website_url: if not website_url:
@@ -167,6 +218,9 @@ def fetch_website_data(website_url):
data["canonical"] = m.group(1) if m else None data["canonical"] = m.group(1) if m else None
data["has_og"] = bool(re.search(r'<meta[^>]*property=["\']og:', html, re.IGNORECASE)) data["has_og"] = bool(re.search(r'<meta[^>]*property=["\']og:', html, re.IGNORECASE))
# Build stack + SSR signal (who built it, and what a non-JS crawler reads)
data["stack"] = fingerprint_website(html)
# JSON-LD business hours -> same {day: value} shape as the other surfaces # JSON-LD business hours -> same {day: value} shape as the other surfaces
try: try:
from canonical_baseline import jsonld_hours from canonical_baseline import jsonld_hours
@@ -289,6 +343,9 @@ def merge_and_validate(gbp_data=None, apple_data=None, website_data=None, bing_d
async def main(): async def main():
from gmaps_scraper.scraper import GoogleMapsScraper
from apple_scraper import scrape_apple
from bing_provider import scrape_bing
if len(sys.argv) < 3: if len(sys.argv) < 3:
print("Usage: multi_scraper.py <business_name> <city, state>") print("Usage: multi_scraper.py <business_name> <city, state>")
sys.exit(1) sys.exit(1)
@@ -371,5 +428,28 @@ async def main():
print(f" {key}: {json.dumps(val)}") print(f" {key}: {json.dumps(val)}")
def _selftest():
lovable_html = """
<html><head><title>T</title>
<script type="application/ld+json">{"@type":"BeautySalon"}</script>
<script src="/assets/index-DfsFDpW1.js"></script>
<script src="/~flock.js"></script>
<link href="/assets/index-BW0u6Jxt.css" rel="stylesheet"></head>
<body><noscript>required</noscript></body></html>
"""
f = fingerprint_website(lovable_html)
assert f["detected"][0] == "lovable", f
assert f["primary"] == "lovable"
assert f["html_word_count"] < 50 and f["ssr_signal"] == "weak", f
full_html = "<html><body>" + ("<p>word </p>" * 80) + "</body></html>"
f2 = fingerprint_website(full_html)
assert f2["ssr_signal"] == "present" and f2["html_word_count"] == 80, f2
assert f2["primary"] == "unknown"
print("SELF-TEST OK: lovable detected, SSR weak (13-word page) / present (80-word page)")
if __name__ == "__main__": if __name__ == "__main__":
if len(sys.argv) > 1 and sys.argv[1] == "--selftest":
_selftest()
else:
asyncio.run(main()) asyncio.run(main())