"""Amostra de artigos SciELO Brasil em Ciencias Humanas e Sociais Aplicadas.

1. Lista periodicos da colecao 'scl' e guarda a area (campo v441) em cache.
2. Lista identificadores de artigos dos periodicos selecionados.
3. Sorteia artigos (ano extraido do PID) e baixa titulo e resumo.

Uso:
  python collect_scielo.py --years 2015-2025 --n 40 --out ../pilot/scielo.jsonl
"""
import argparse, json, os, random, time, urllib.request
from concurrent.futures import ThreadPoolExecutor

API = "http://articlemeta.scielo.org/api/v1"
AREAS = {"Human Sciences", "Applied Social Sciences"}
HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.join(HERE, "..", "data")


def get(url, tries=4):
    for i in range(tries):
        try:
            with urllib.request.urlopen(url, timeout=90) as r:
                return json.load(r)
        except Exception:
            if i == tries - 1:
                raise
            time.sleep(2 * (i + 1))


def journals():
    path = os.path.join(CACHE, "scielo_journals.json")
    if os.path.exists(path):
        return json.load(open(path, encoding="utf-8"))
    ids, off = [], 0
    while True:
        d = get(f"{API}/journal/identifiers/?collection=scl&limit=1000&offset={off}")
        ids += [o["code"] for o in d["objects"]]
        off += 1000
        if off >= d["meta"]["total"]:
            break

    def one(issn):
        j = get(f"{API}/journal/?collection=scl&issn={issn}")
        j = j[0] if isinstance(j, list) else j
        val = lambda k: [x.get("_") for x in j.get(k, [])]
        return {"issn": issn, "titulo": (val("v100") or [""])[0],
                "areas": val("v441"), "status": (val("v50") or [""])[0]}

    with ThreadPoolExecutor(8) as ex:
        out = list(ex.map(one, ids))
    os.makedirs(CACHE, exist_ok=True)
    json.dump(out, open(path, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
    return out


def article_ids(issn):
    ids, off = [], 0
    while True:
        d = get(f"{API}/article/identifiers/?collection=scl&issn={issn}&limit=1000&offset={off}")
        ids += [o["code"] for o in d["objects"]]
        off += 1000
        if off >= d["meta"]["total"]:
            return ids


def text_of(entries, lang):
    for e in entries or []:
        if e.get("l") == lang:
            return e.get("a") or e.get("_") or ""
    return ""


def fetch(pid, jmeta):
    a = get(f"{API}/article/?code={pid}&collection=scl")
    art = a.get("article", {})
    return {
        "fonte": "scielo", "id": f"scielo-{pid}", "pid": pid,
        "ano": a.get("publication_year"), "tipo_documento": a.get("document_type"),
        "periodico": jmeta["titulo"], "issn": jmeta["issn"], "areas_periodico": jmeta["areas"],
        "titulo": text_of(art.get("v12"), "pt") or text_of(art.get("v12"), "en"),
        "resumo_pt": text_of(art.get("v83"), "pt"),
        "resumo_en": text_of(art.get("v83"), "en"),
    }


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--years", default="2015-2025")
    ap.add_argument("--n", type=int, default=40)
    ap.add_argument("--min-chars", type=int, default=400)
    ap.add_argument("--seed", type=int, default=20261013)
    ap.add_argument("--out", required=True)
    a = ap.parse_args()
    y0, y1 = map(int, a.years.split("-"))
    rng = random.Random(a.seed)
    js = [j for j in journals() if AREAS & set(j["areas"])]
    print(f"periodicos selecionados: {len(js)}")
    pool = []
    with ThreadPoolExecutor(6) as ex:
        for j, ids in zip(js, ex.map(lambda j: article_ids(j["issn"]), js)):
            pool += [(pid, j) for pid in ids if pid[10:14].isdigit() and y0 <= int(pid[10:14]) <= y1]
    print(f"artigos no recorte: {len(pool)}")
    rng.shuffle(pool)
    out, i = [], 0
    while len(out) < a.n and i < len(pool):
        batch = pool[i:i + 2 * (a.n - len(out))]
        i += len(batch)
        with ThreadPoolExecutor(6) as ex:
            for rec in ex.map(lambda p: fetch(*p), batch):
                if rec["tipo_documento"] == "research-article" and len(rec["resumo_pt"]) >= a.min_chars:
                    out.append(rec)
    with open(a.out, "w", encoding="utf-8") as f:
        for r in out[: a.n]:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"gravados: {min(len(out), a.n)}")


if __name__ == "__main__":
    main()