#!/usr/bin/env python3
"""Build the publishable dataset for the metadata + third-party study.

Every value here was measured on 2026-08-15 from a single location (India),
one pass per service, default options, using a real Chrome browser driven via
the Chrome DevTools Protocol. Raw evidence lives alongside this file.

Emits dataset.json and dataset.csv. These are the files that ship with the
study so any reader can check the numbers.
"""
import json, csv, os

MEASURED = "2026-08-15"
VANTAGE = "IN"          # single location; stated as a limitation, not hidden
ROUTE_IMG = "jpg->png"
ROUTE_PDF = "jpg->pdf"

# metadata_returned: did the output still contain GPS / body serial / owner name
# metadata_location: "direct" | "embedded" (inside the PDF's image stream) | null
# tp_hosts: distinct third-party hostnames contacted on the DOWNLOAD page
# tp_orgs: those hostnames collapsed to registrable domains
# declared_partners: number the operator's own consent dialog names, if any
ROWS = [
    dict(service="CloudConvert", slug="cloudconvert", route=ROUTE_IMG,
         metadata_returned=True, metadata_location="direct",
         output="8-bit RGB", output_bytes=208000,
         tp_hosts=0, tp_orgs=0, adtech_orgs=0, declared_partners=None,
         session_recorders=[], retention_claim=None, retention_shown=None,
         notes="Zero third parties on homepage and download page. First-party Plausible analytics."),
    dict(service="AnyConv", slug="anyconv", route=ROUTE_IMG,
         metadata_returned=True, metadata_location="direct",
         output="8-bit RGB", output_bytes=204409,
         tp_hosts=73, tp_orgs=46, adtech_orgs=37, declared_partners=None,
         session_recorders=[], retention_claim="1 hour", retention_shown=None,
         notes="Largest measured third-party surface: 483 requests, 46 organisations, incl. identity-sync services."),
    dict(service="Convertio", slug="convertio", route=ROUTE_IMG,
         metadata_returned=True, metadata_location="direct",
         output="8-bit RGB", output_bytes=204708,
         tp_hosts=11, tp_orgs=8, adtech_orgs=5, declared_partners=193,
         session_recorders=[], retention_claim="we do not store or backup your files",
         retention_shown="Files will be stored for 24 hours",
         notes="ToS and product contradict on retention. Consent dialog (Clickio) declares 193 partners against 11 measured hosts."),
    dict(service="Aconvert", slug="aconvert", route=ROUTE_IMG,
         metadata_returned=True, metadata_location="direct",
         output="8-bit RGB", output_bytes=204683,
         tp_hosts=8, tp_orgs=6, adtech_orgs=4, declared_partners=None,
         session_recorders=[], retention_claim="two hours", retention_shown="1-2 hours",
         notes="No terms of service exists; /terms.html serves a 404 body. Three different retention numbers across its own pages."),
    dict(service="Online-Convert", slug="online-convert", route=ROUTE_IMG,
         metadata_returned=True, metadata_location="direct",
         output="8-bit palette", output_bytes=70973,
         tp_hosts=14, tp_orgs=7, adtech_orgs=5, declared_partners=None,
         session_recorders=[], retention_claim="after the storage period expires (unspecified)",
         retention_shown=None,
         notes="Palettises to 71 KB yet keeps all metadata; the control that disproves 'compression strips metadata'."),
    dict(service="FreeConvert", slug="freeconvert", route=ROUTE_IMG,
         metadata_returned=False, metadata_location=None,
         output="8-bit palette", output_bytes=73113,
         tp_hosts=73, tp_orgs=45, adtech_orgs=None, declared_partners=None,
         session_recorders=["Microsoft Clarity"], retention_claim="8 hours",
         retention_shown="deleted after 8 hours",
         notes="Strips all metadata. Only service whose retention claim matches its product exactly."),
    dict(service="Zamzar", slug="zamzar", route=ROUTE_IMG,
         metadata_returned=False, metadata_location=None,
         output="8-bit RGB", output_bytes=198147,
         tp_hosts=7, tp_orgs=5, adtech_orgs=3, declared_partners=None,
         session_recorders=[], retention_claim=None, retention_shown="Files are stored for 24 hours",
         notes="Strips all metadata at full RGB quality; proves stripping is deliberate, not a compression artefact."),
    dict(service="PDF24", slug="pdf24", route=ROUTE_PDF,
         metadata_returned=True, metadata_location="embedded",
         output="PDF 1.6", output_bytes=138820,
         tp_hosts=11, tp_orgs=6, adtech_orgs=4, declared_partners=None,
         session_recorders=[], retention_claim="one hour", retention_shown=None,
         notes="Output reads clean to exiftool but contains the full payload inside the embedded image. Ships an immediate-erase button and self-hosted Matomo."),
    dict(service="Smallpdf", slug="smallpdf", route=ROUTE_PDF,
         metadata_returned=False, metadata_location=None,
         output="PDF 1.4", output_bytes=137829,
         tp_hosts=16, tp_orgs=12, adtech_orgs=7, declared_partners=None,
         session_recorders=["Microsoft Clarity", "Hotjar"], retention_claim=None, retention_shown=None,
         notes="Cleanest deliberate strip: embedded image passed through at original size with EXIF removed. Runs two session recorders after upload."),
    dict(service="iLovePDF", slug="ilovepdf", route=ROUTE_PDF,
         metadata_returned=False, metadata_location=None,
         output="PDF 1.7 (resampled 1240x930)", output_bytes=25956,
         tp_hosts=10, tp_orgs=7, adtech_orgs=5, declared_partners=None,
         session_recorders=[], retention_claim="two (2) hours", retention_shown=None,
         notes="Metadata absent, but the image is resampled, so loss may be incidental rather than deliberate. Motive not established. Silently downscales."),
    dict(service="Sejda", slug="sejda", route=ROUTE_PDF,
         metadata_returned=False, metadata_location=None,
         output="PDF 1.5", output_bytes=71963,
         tp_hosts=0, tp_orgs=0, adtech_orgs=0, declared_partners=None,
         session_recorders=[], retention_claim=None, retention_shown=None,
         notes="Only service good on both axes: strips metadata and loads zero third parties."),
]

CONTROL = [
    dict(conversion="jpg->png (local)", metadata_returned=True, metadata_location="direct",
         note="Library default preserves everything. Stripping is therefore the deliberate act."),
    dict(conversion="jpg->pdf (local)", metadata_returned=True, metadata_location="embedded",
         note="Full EXIF survives inside the embedded image; exiftool on the PDF shows nothing."),
    dict(conversion="heic->jpg (local)", metadata_returned=True, metadata_location="direct",
         note="Same as jpg->png."),
]

out = {
    "study": "Metadata and third-party exposure across online file converters",
    "measured_date": MEASURED,
    "vantage_point": VANTAGE,
    "method": {
        "specimen": "Generated 1600x1200 JPEG with a complete EXIF payload: GPS at a public landmark (Eiffel Tower), camera body serial, owner name, make/model/lens, timestamp, and a unique marker string.",
        "options": "Default only. No metadata toggles were touched.",
        "passes": 1,
        "browser": "Chrome via DevTools Protocol, one isolated context per service",
        "third_party_stage": "Measured on the download page after a real conversion, not the homepage.",
    },
    "limitations": [
        "Single geographic vantage point (India). An EEA visitor meets a consent gate first and will see different third-party behaviour.",
        "One pass, one file, one route per service. No claim about consistency over time or across formats.",
        "Default options only; some services may offer a metadata toggle that was not used.",
        "Motive is never observable. We report what came back, not why.",
        "Measured host counts understate data sharing where a consent management platform brokers it (see Convertio: 11 hosts, 193 declared partners).",
        "FreeConvert's third-party figures were captured before the resource-timing buffer was raised; host count is reliable, request volume is understated.",
    ],
    "disclosure": "This study was produced by Morphjet, which sells a local file converter and therefore competes with every service measured. Method, specimen, and raw outputs are published so the results can be checked independently.",
    "control": CONTROL,
    "results": ROWS,
    "summary": {
        "services_measured": len(ROWS),
        "returned_metadata": sum(1 for r in ROWS if r["metadata_returned"]),
        "removed_metadata": sum(1 for r in ROWS if not r["metadata_returned"]),
        "zero_third_parties": sum(1 for r in ROWS if r["tp_hosts"] == 0),
        "with_session_recorders": sum(1 for r in ROWS if r["session_recorders"]),
    },
}

here = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(here, "dataset.json"), "w") as f:
    json.dump(out, f, indent=2)

cols = ["service", "slug", "route", "metadata_returned", "metadata_location", "output",
        "output_bytes", "tp_hosts", "tp_orgs", "adtech_orgs", "declared_partners",
        "session_recorders", "retention_claim", "retention_shown", "notes"]
with open(os.path.join(here, "dataset.csv"), "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=cols)
    w.writeheader()
    for r in ROWS:
        row = dict(r)
        row["session_recorders"] = "; ".join(row["session_recorders"])
        w.writerow(row)

s = out["summary"]
print(f"services: {s['services_measured']}")
print(f"  returned metadata : {s['returned_metadata']}")
print(f"  removed metadata  : {s['removed_metadata']}")
print(f"  zero third parties: {s['zero_third_parties']}")
print(f"  session recorders : {s['with_session_recorders']}")
print("wrote dataset.json + dataset.csv")
