"""Structural validation and deterministic consolidation of coded batches.

Only records owned by a batch are accepted. A valid sub-batch record takes
precedence over its parent; incomplete placeholders never count as coding.
Raw files remain unchanged by this reader. Enum/content doubts are audited
separately from structural coverage.
"""
import json
from pathlib import Path

REQUIRED = set("id tema_politico temas eixo_economico eixo_cultural evidencia_posicao postura_normativa alvo_critica referencial_teorico tipo_pesquisa pergunta_explicita hipotese_testavel dados_descritos desenho_inferencia conclusao_tipo premissa_valorativa confianca politica_avaliada veredito_politica direcao_critica".split())
BOOL_FIELDS = ("tema_politico", "pergunta_explicita", "hipotese_testavel", "dados_descritos", "premissa_valorativa")


def normalize(record):
    if not isinstance(record, dict):
        return record, []
    row = dict(record)
    changes = []
    for field in ("eixo_economico", "eixo_cultural"):
        if row.get(field) in ("-2", "-1", "0", "1", "2"):
            row[field] = int(row[field])
            changes.append(field + ":numeric_string")
    if ("direcao_critica" not in row and "politica_avaliada" in row
            and "veredito_politica" in row and row["politica_avaliada"] is None
            and row["veredito_politica"] is None):
        row["direcao_critica"] = None
        changes.append("direcao_critica:null_without_policy")
    if row.get("postura_normativa") == "explícita":
        row["postura_normativa"] = "explicita"
        changes.append("postura_normativa:accent")
    return row, changes


def complete(row):
    if not isinstance(row, dict) or not REQUIRED <= row.keys():
        return False
    if not isinstance(row["id"], str) or not row["id"]:
        return False
    if any(type(row[field]) is not bool for field in BOOL_FIELDS):
        return False
    if any(row[field] is not None and (type(row[field]) is not int or not -2 <= row[field] <= 2)
           for field in ("eixo_economico", "eixo_cultural")):
        return False
    return isinstance(row["temas"], list) and all(isinstance(row[f], str)
        for f in ("postura_normativa", "tipo_pesquisa", "desenho_inferencia", "conclusao_tipo"))


def read_rows(path):
    if not Path(path).exists():
        return
    with open(path, encoding="utf-8-sig") as handle:
        for number, line in enumerate(handle, 1):
            if not line.strip():
                continue
            try:
                yield number, json.loads(line)
            except json.JSONDecodeError:
                yield number, None


def load_coded(data_dir, allowed_ids=None):
    data = Path(data_dir)
    coded, owners = {}, {}
    diagnostics = {"invalid_json": [], "incomplete_records": [],
                   "foreign_batch_ids": [], "normalizations": [], "overrides": []}
    for batch in sorted((data / "batches").glob("b_*.jsonl")):
        expected = {r["id"] for _, r in read_rows(batch) if isinstance(r, dict) and "id" in r}
        output = data / "coded" / batch.name
        for line, raw in read_rows(output):
            if raw is None:
                diagnostics["invalid_json"].append({"file": batch.name, "line": line})
                continue
            row, changes = normalize(raw)
            rid = row.get("id") if isinstance(row, dict) else None
            if rid not in expected or (allowed_ids is not None and rid not in allowed_ids):
                diagnostics["foreign_batch_ids"].append({"file": batch.name, "line": line, "id": rid})
                continue
            if not complete(row):
                diagnostics["incomplete_records"].append({"file": batch.name, "line": line, "id": rid})
                continue
            if changes:
                diagnostics["normalizations"].append({"file": batch.name, "id": rid, "changes": changes})
            if rid in coded and row != coded[rid]:
                diagnostics["overrides"].append({"id": rid, "before": owners[rid], "after": batch.name})
            coded[rid], owners[rid] = row, batch.name
    return dict(sorted(coded.items())), diagnostics