multi_scraper: fingerprint website build stack + SSR signal (DR-003)

2026-08-17 16:46:29 +00:00
parent 5341f3f965
commit 708684bb74
1 changed files with 84 additions and 4 deletions
@@ -22,9 +22,6 @@ from datetime import datetime, timezone
 sys.path.insert(0, "/tmp/google-maps-scraper/src")
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

-from gmaps_scraper.scraper import GoogleMapsScraper
-from apple_scraper import scrape_apple
-from bing_provider import scrape_bing

 BASE_DIR = os.path.dirname(os.path.abspath(__file__))

@@ -130,6 +127,60 @@ async def scrape_google(query, coords, business=""):
    return None


+STACK_SIGNATURES = {
+    # AI app builders (client-built sites)
+    "lovable": r"lovable\.dev|/~flock\.js",
+    "v0": r"v0\.dev|__v0",
+    "bolt": r"stackblitz\.dev|bolt\b",
+    # Frameworks
+    "nextjs": r"__next|/_next/",
+    "nuxt": r"__nuxt",
+    "gatsby": r"gatsby",
+    "vite": r"/@vite/|vite/client|/_assets/",
+    "react": r"react",
+    "angular": r"angular",
+    "vue": r"vue",
+    "svelte": r"svelte",
+    # Page builders / CMS / ecommerce
+    "wordpress": r"wp-content|wp-includes|wordpress",
+    "squarespace": r"squarespace",
+    "wix": r"wix\.com|wixstatic",
+    "webflow": r"webflow",
+    "shopify": r"shopify|cdn\.shopify",
+    "bigcommerce": r"bigcommerce",
+    # Hosts (weak signal only)
+    "vercel": r"vercel|_next/static",
+    "netlify": r"netlify",
+    "cloudflare_pages": r"pages\.dev|cf-cache-status",
+    "github_pages": r"ghs\.|github\.io",
+}
+
+WEBSITE_STACK_ORDER = ["lovable", "v0", "bolt", "nextjs", "nuxt", "gatsby", "vite", "react",
+                       "angular", "vue", "svelte", "wordpress", "squarespace", "wix",
+                       "webflow", "shopify", "bigcommerce", "vercel", "netlify",
+                       "cloudflare_pages", "github_pages"]
+
+
+def fingerprint_website(html):
+    """Fingerprint the website build stack + SSR signal.
+
+    ponytail: regex fingerprints on raw HTML — good enough for a known-site
+    audit; false-negatives land in 'unknown', never wrong with confidence.
+    """
+    low = html.lower()
+    detected = [k for k in WEBSITE_STACK_ORDER if re.search(STACK_SIGNATURES[k], low)]
+    prose = re.sub(r"<script.*?</script>|<style.*?</style>|<[^>]+>", " ", html, flags=re.S)
+    word_count = len(prose.split())
+    return {
+        "detected": detected,
+        "primary": detected[0] if detected else "unknown",
+        # SSR signal: word count a non-JS crawler (plain HTTP fetch) can read.
+        # <50 = content rendered client-side; AI crawlers that skip JS see ~nothing.
+        "html_word_count": word_count,
+        "ssr_signal": "weak" if word_count < 50 else "present",
+    }
+
+
 def fetch_website_data(website_url):
    """Extract basic info from business website."""
    if not website_url:
@@ -167,6 +218,9 @@ def fetch_website_data(website_url):
        data["canonical"] = m.group(1) if m else None
        data["has_og"] = bool(re.search(r'<meta[^>]*property=["\']og:', html, re.IGNORECASE))

+        # Build stack + SSR signal (who built it, and what a non-JS crawler reads)
+        data["stack"] = fingerprint_website(html)
+
        # JSON-LD business hours -> same {day: value} shape as the other surfaces
        try:
            from canonical_baseline import jsonld_hours
@@ -289,6 +343,9 @@ def merge_and_validate(gbp_data=None, apple_data=None, website_data=None, bing_d


 async def main():
+    from gmaps_scraper.scraper import GoogleMapsScraper
+    from apple_scraper import scrape_apple
+    from bing_provider import scrape_bing
    if len(sys.argv) < 3:
        print("Usage: multi_scraper.py <business_name> <city, state>")
        sys.exit(1)
@@ -371,5 +428,28 @@ async def main():
            print(f"  {key}: {json.dumps(val)}")


+def _selftest():
+    lovable_html = """
+    <html><head><title>T</title>
+    <script type="application/ld+json">{"@type":"BeautySalon"}</script>
+    <script src="/assets/index-DfsFDpW1.js"></script>
+    <script src="/~flock.js"></script>
+    <link href="/assets/index-BW0u6Jxt.css" rel="stylesheet"></head>
+    <body><noscript>required</noscript></body></html>
+    """
+    f = fingerprint_website(lovable_html)
+    assert f["detected"][0] == "lovable", f
+    assert f["primary"] == "lovable"
+    assert f["html_word_count"] < 50 and f["ssr_signal"] == "weak", f
+    full_html = "<html><body>" + ("<p>word </p>" * 80) + "</body></html>"
+    f2 = fingerprint_website(full_html)
+    assert f2["ssr_signal"] == "present" and f2["html_word_count"] == 80, f2
+    assert f2["primary"] == "unknown"
+    print("SELF-TEST OK: lovable detected, SSR weak (13-word page) / present (80-word page)")
+
+
 if __name__ == "__main__":
-    asyncio.run(main())
+    if len(sys.argv) > 1 and sys.argv[1] == "--selftest":
+        _selftest()
+    else:
+        asyncio.run(main())