#!/usr/bin/env python3
"""EU vantage-point measurement for the converter privacy study.

Runs on a Frankfurt VSI. Loads each service's homepage in a FRESH browser
context, records every third-party request, and reports whether a consent gate
appeared.

🔴 It deliberately DOES NOT accept consent. The pre-consent state is what an
EEA visitor actually gets by default, and is the honest number to compare
against the India run, where no gate appears at all.

Output: eu-results.json
"""
import asyncio, json, re, sys
from urllib.parse import urlparse
from playwright.async_api import async_playwright

SERVICES = [
    ("CloudConvert",   "https://cloudconvert.com/"),
    ("Zamzar",         "https://www.zamzar.com/"),
    ("FreeConvert",    "https://www.freeconvert.com/"),
    ("Smallpdf",       "https://smallpdf.com/"),
    ("iLovePDF",       "https://www.ilovepdf.com/"),
    ("Convertio",      "https://convertio.co/"),
    ("Online-Convert", "https://www.online-convert.com/"),
    ("AnyConv",        "https://anyconv.com/"),
    ("Aconvert",       "https://www.aconvert.com/"),
    ("Sejda",          "https://www.sejda.com/"),
    ("PDF24",          "https://tools.pdf24.org/en/"),
]

AD = re.compile(
    r"googlesyndication|doubleclick|googleadservices|googletagservices|adtrafficquality|"
    r"fundingchoices|google-analytics|googletagmanager|analytics\.google|adnxs|criteo|"
    r"taboola|outbrain|pubmatic|rubiconproject|openx|adroll|quantserve|scorecardresearch|"
    r"facebook\.|hotjar|clarity\.ms|amplitude|segment\.|mixpanel|adsrvr|bing\.|linkedin|"
    r"tiktok|amazon-adsystem|media\.net|id5-sync|hadronid|crwdcntrl|33across|deepintent|"
    r"stackadapt|smartadserver|casalemedia|contextweb|3lift|seedtag|indexww|kueezrtb|"
    r"presage|yieldmo|lijit|360yield|dotomi|fastclick|trustedstack|minutemedia|ascendeum|"
    r"hbwrapper|pub\.network|primis|confiant|sovrn|wurfl|2mdn|flashtalking", re.I)

# Consent management platforms and the generic wording they use.
CMP_HOST = re.compile(r"clickio|onetrust|cookielaw|sourcepoint|quantcast|didomi|sirdata|"
                      r"usercentrics|cookiebot|iubenda|trustarc|consentmanager|fundingchoices", re.I)
CMP_TEXT = re.compile(r"consent|cookie|privacy|legitimate interest|manage options|"
                      r"accept all|reject all|partners|vendors|einwillig|zustimm", re.I)

def reg(host: str) -> str:
    parts = host.split(".")
    return host if len(parts) <= 2 else ".".join(parts[-2:])

async def measure(pw, name, url):
    browser = await pw.chromium.launch(args=["--no-sandbox"])
    ctx = await browser.new_context(
        locale="de-DE", timezone_id="Europe/Berlin",
        viewport={"width": 1440, "height": 900},
        user_agent=("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
                    "(KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36"))
    page = await ctx.new_page()
    hosts = {}
    def on_req(r):
        try:
            h = urlparse(r.url).hostname
        except Exception:
            return
        if h:
            hosts[h] = hosts.get(h, 0) + 1
    page.on("request", on_req)

    err = None
    try:
        await page.goto(url, wait_until="load", timeout=60000)
        await page.wait_for_timeout(9000)   # let late ad/consent tags fire
    except Exception as e:
        err = str(e)[:120]

    site = urlparse(url).hostname.replace("www.", "")
    third = {h: n for h, n in hosts.items()
             if h != site and not h.endswith("." + site) and reg(h) != reg(site)}
    orgs = sorted({reg(h) for h in third})

    # Consent gate detection: a CMP host was contacted, or visible consent wording.
    cmp_hosts = [h for h in third if CMP_HOST.search(h)]
    body = ""
    try:
        body = (await page.inner_text("body"))[:6000]
    except Exception:
        pass
    text_hit = bool(CMP_TEXT.search(body)) and bool(
        re.search(r"accept|agree|reject|manage|zustimmen|ablehnen", body, re.I))

    # Try to pull a declared partner count out of the dialog text.
    partners = None
    m = re.search(r"(\d{2,4})\s*(?:partner|vendor|third part|anbieter)", body, re.I)
    if m:
        partners = int(m.group(1))

    cookies = await ctx.cookies()
    await browser.close()
    return {
        "service": name, "url": url, "error": err,
        "requests_total": sum(hosts.values()),
        "tp_hosts": len(third), "tp_orgs": len(orgs),
        "adtech_orgs": len([o for o in orgs if AD.search(o)]),
        "orgs": orgs,
        "consent_gate": bool(cmp_hosts) or text_hit,
        "cmp_hosts": sorted(set(cmp_hosts)),
        "declared_partners": partners,
        "cookies_before_consent": len(cookies),
        "third_party_cookies": len([c for c in cookies if reg(c["domain"].lstrip(".")) != reg(site)]),
    }

async def main():
    out = []
    async with async_playwright() as pw:
        for name, url in SERVICES:
            print(f"  measuring {name} ...", flush=True)
            try:
                r = await measure(pw, name, url)
            except Exception as e:
                r = {"service": name, "url": url, "error": str(e)[:160]}
            out.append(r)
            print(f"    hosts={r.get('tp_hosts')} orgs={r.get('tp_orgs')} "
                  f"gate={r.get('consent_gate')} cookies={r.get('cookies_before_consent')}", flush=True)
    with open("eu-results.json", "w") as f:
        json.dump({"vantage": "DE / Frankfurt", "consent_accepted": False, "results": out}, f, indent=2)
    print("wrote eu-results.json")

asyncio.run(main())