"""phx_fetch.py — fetch live public data for corpus post IDs (no X API).

Uses the public syndication endpoint (same one that powers embedded tweets).
Polite: single-threaded, delay between requests, resumable JSONL output.

Usage:
  python phx_fetch.py --ids ids.txt --out fetched.jsonl --delay 0.7
"""
from __future__ import annotations

import argparse
import json
import math
import os
import time
import urllib.request
import urllib.error

p = argparse.ArgumentParser()
p.add_argument("--ids", required=True, help="file with one post id per line")
p.add_argument("--out", required=True)
p.add_argument("--delay", type=float, default=0.7)
p.add_argument("--max", type=int, default=100000)
args = p.parse_args()

DIGS = "0123456789abcdefghijklmnopqrstuvwxyz"


def tok36(tid: str) -> str:
    x = int(tid) / 1e15 * math.pi
    i = int(x)
    f = x - i
    s = ""
    while i:
        s = DIGS[i % 36] + s
        i //= 36
    s += "."
    for _ in range(12):
        f *= 36
        d = int(f)
        s += DIGS[d]
        f -= d
    return s.replace("0", "").replace(".", "")


done = set()
if os.path.exists(args.out):
    with open(args.out, encoding="utf-8") as f:
        for line in f:
            try:
                done.add(json.loads(line)["id"])
            except Exception:
                pass
print(f"[resume] {len(done)} already fetched")

ids = [l.strip() for l in open(args.ids) if l.strip()]
ids = [i for i in ids if i not in done][: args.max]
print(f"[fetch] {len(ids)} to go")

ok = miss = err = 0
with open(args.out, "a", encoding="utf-8") as out:
    for n, tid in enumerate(ids):
        url = (f"https://cdn.syndication.twimg.com/tweet-result?id={tid}"
               f"&token={tok36(tid)}")
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        rec = {"id": tid}
        try:
            with urllib.request.urlopen(req, timeout=15) as r:
                d = json.load(r)
            if d.get("__typename") == "TweetTombstone" or "text" not in d:
                rec["status"] = "tombstone"
                miss += 1
            else:
                u = d.get("user", {})
                ents = d.get("entities", {})
                rec.update({
                    "status": "ok",
                    "text": d.get("text", ""),
                    "favs": d.get("favorite_count", 0),
                    "replies": d.get("conversation_count", 0),
                    "created_at": d.get("created_at", ""),
                    "lang": d.get("lang", ""),
                    "screen_name": u.get("screen_name", ""),
                    "verified": bool(u.get("is_blue_verified", False)),
                    "has_photo": bool(d.get("photos")),
                    "has_video": bool(d.get("video")),
                    "n_urls": len(ents.get("urls", [])),
                    "n_hashtags": len(ents.get("hashtags", [])),
                    "n_mentions": len(ents.get("user_mentions", [])),
                    "is_reply": "in_reply_to_status_id_str" in d or bool(d.get("parent")),
                })
                ok += 1
        except urllib.error.HTTPError as e:
            rec["status"] = f"http{e.code}"
            miss += 1
        except Exception as e:  # noqa: BLE001
            rec["status"] = f"err:{type(e).__name__}"
            err += 1
        out.write(json.dumps(rec, ensure_ascii=False) + "\n")
        out.flush()
        if (n + 1) % 50 == 0:
            print(f"[{n+1}/{len(ids)}] ok={ok} gone={miss} err={err}")
        time.sleep(args.delay)
print(f"[done] ok={ok} gone={miss} err={err}")