#!/usr/bin/env python3
"""collect_candidates.py — сбор кандидатов для контент-плана блога.

Самодостаточный: только стандартная библиотека Python 3.9+.
Собирает материалы из открытых источников (RSS, Hacker News, arXiv, Reddit,
Dev.to, Lobsters, GitHub trending, YouTube через yt-dlp), оценивает объём,
дедуплицирует против уже опубликованных постов и отдаёт JSON + markdown-заготовку
контент-плана.

Примеры:
    python3 collect_candidates.py --blog-dir ~/blog/posts --out plan.json --md plan.md
    python3 collect_candidates.py --sources rss,hn,arxiv --limit 25
    python3 collect_candidates.py --proxy socks5://127.0.0.1:1080 --deep 10
    python3 collect_candidates.py --help

Ничего не публикует и не коммитит: только собирает и печатает.
"""
from __future__ import annotations

import argparse
import html
import json
import os
import re
import socket
import ssl
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta, timezone

USER_AGENT = "content-plan-collector/1.0 (+contact: blog owner)"
DEFAULT_TIMEOUT = 12

# ─────────────────────────── КОНФИГ (правь здесь) ───────────────────────────

RSS_FEEDS = [
    ("OpenAI Blog", "https://openai.com/blog/rss.xml"),
    ("Hugging Face", "https://huggingface.co/blog/feed.xml"),
    ("Google DeepMind", "https://deepmind.google/blog/rss.xml"),
    ("Import AI", "https://importai.substack.com/feed"),
    ("The Gradient", "https://thegradient.pub/rss/"),
    ("Simon Willison", "https://simonwillison.net/atom/everything/"),
    ("Last Week in AI", "https://lastweekin.ai/feed"),
    ("One Useful Thing", "https://www.oneusefulthing.org/feed"),
    ("BAIR Blog", "https://bair.berkeley.edu/blog/feed.xml"),
    ("Distill", "https://distill.pub/rss.xml"),
    ("Lilian Weng", "https://lilianweng.github.io/index.xml"),
    ("Google AI", "https://blog.google/innovation-and-ai/technology/ai/rss/"),
    ("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"),
    ("PyTorch Blog", "https://pytorch.org/blog/feed.xml"),
    ("NVIDIA Developer", "https://developer.nvidia.com/blog/category/artificial-intelligence/feed/"),
]

REDDIT_SUBS = ["LocalLLaMA", "MachineLearning", "ArtificialIntelligence"]

YOUTUBE_QUERIES = [
    "AI news this week",
    "LLM large language model explained",
    "machine learning breakthrough",
    "AI agent automation",
]

# Пороги «хватает ли материала на статью» (в словах)
MIN_WORDS = {"youtube": 3000, "rss": 1000, "github": 1000, "reddit": 2000,
             "hn": 2000, "arxiv": 150, "devto": 800, "lobsters": 800}

# Кандидат старше этого срока отбрасывается (новости), кроме arxiv/github/youtube
FRESH_DAYS = 7
FRESH_DAYS_EXEMPT = {"arxiv", "github", "youtube"}

# Вес источника при сортировке
SOURCE_WEIGHT = {"arxiv": 3.0, "rss": 2.5, "github": 2.5, "hn": 2.0,
                 "reddit": 1.5, "lobsters": 1.5, "devto": 1.0,
                 "youtube": 2.0, "producthunt": 0.5}

# ─────────────────────────── HTTP ───────────────────────────

def _opener(proxy: str | None):
    handlers = []
    if proxy:
        if proxy.startswith("socks"):
            try:
                import socks  # PySocks: pip install pysocks
            except ImportError:
                print("предупреждение: PySocks не установлен, socks-прокси игнорируется "
                      "(pip install pysocks)", file=sys.stderr)
            else:
                parsed = urllib.parse.urlparse(proxy)
                kind = socks.SOCKS5 if (parsed.scheme or "socks5").endswith("5") else socks.SOCKS4
                socks.set_default_proxy(kind, parsed.hostname, parsed.port or 1080)
                socket.socket = socks.socksocket
                handlers.append(urllib.request.ProxyHandler({}))
        else:
            handlers.append(urllib.request.ProxyHandler({"http": proxy, "https": proxy}))
    ctx = ssl.create_default_context()
    return urllib.request.build_opener(*handlers, urllib.request.HTTPSHandler(context=ctx))


def fetch(url: str, opener, timeout: int = DEFAULT_TIMEOUT) -> str:
    req = urllib.request.Request(url, headers={
        "User-Agent": USER_AGENT,
        "Accept": "application/rss+xml, application/atom+xml, application/json, text/html;q=0.9, */*;q=0.5",
        "Accept-Language": "en-US,en;q=0.8",
    })
    with opener.open(req, timeout=timeout) as resp:
        raw = resp.read()
    for enc in ("utf-8", "latin-1"):
        try:
            return raw.decode(enc)
        except UnicodeDecodeError:
            continue
    return raw.decode("utf-8", "replace")


# ─────────────────────────── ТЕКСТ ───────────────────────────

TAG_RE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.S | re.I)
ANY_TAG_RE = re.compile(r"<[^>]+>")


def html_to_text(raw: str) -> str:
    text = TAG_RE.sub(" ", raw or "")
    text = ANY_TAG_RE.sub(" ", text)
    text = html.unescape(text)
    return re.sub(r"\s+", " ", text).strip()


def word_count(text: str) -> int:
    return len(re.findall(r"\w+", text or ""))


def parse_date(value: str | None) -> datetime | None:
    if not value:
        return None
    value = value.strip()
    fmts = ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S.%fZ",
            "%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%a, %d %b %Y %H:%M:%S %z",
            "%a, %d %b %Y %H:%M:%S %Z", "%Y-%m-%dT%H:%M:%S%z")
    for fmt in fmts:
        try:
            dt = datetime.strptime(value.replace("+00:00", "+0000"), fmt)
            return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
        except ValueError:
            continue
    try:
        return datetime.fromtimestamp(float(value), tz=timezone.utc)
    except (ValueError, TypeError):
        return None


def item(title, url, source, kind, summary="", published=None, words=None, extra=None):
    return {
        "title": html.unescape(re.sub(r"\s+", " ", title or "").strip()),
        "url": url,
        "source": source,
        "kind": kind,
        "summary_words": words if words is not None else word_count(summary),
        "published": published.isoformat() if isinstance(published, datetime) else published,
        "extra": extra or {},
    }


# ─────────────────────────── ИСТОЧНИКИ ───────────────────────────

def strip_ns(tag: str) -> str:
    return tag.split("}", 1)[-1].lower()


def collect_rss(opener, per_feed=6):
    out, errors = [], []
    for name, url in RSS_FEEDS:
        try:
            xml = fetch(url, opener)
            root = ET.fromstring(xml.encode("utf-8", "replace"))
        except Exception as exc:  # битый/мёртвый фид — не роняем сбор
            errors.append(f"RSS {name}: {type(exc).__name__}: {exc}")
            continue
        entries = [el for el in root.iter() if strip_ns(el.tag) in ("item", "entry")][:per_feed]
        for entry in entries:
            data = {strip_ns(child.tag): child for child in entry}
            title = (data.get("title").text if data.get("title") is not None else "") or ""
            link = ""
            if data.get("link") is not None:
                link = data["link"].text or data["link"].attrib.get("href", "")
            date = None
            for key in ("pubdate", "published", "updated", "date"):
                if data.get(key) is not None and data[key].text:
                    date = parse_date(data[key].text)
                    break
            summary = ""
            for key in ("encoded", "content", "summary", "description"):
                if data.get(key) is not None and data[key].text:
                    summary = data[key].text
                    break
            if title and link:
                out.append(item(title, link, f"rss/{name}", "rss",
                                summary=summary, published=date))
    return out, errors


def collect_hn(opener, min_points=80, days=7, limit=25):
    since = int((datetime.now(timezone.utc) - timedelta(days=days)).timestamp())
    url = ("https://hn.algolia.com/api/v1/search?tags=story"
           f"&numericFilters=points>{min_points},created_at_i>{since}&hitsPerPage={limit}")
    try:
        data = json.loads(fetch(url, opener))
    except Exception as exc:
        return [], [f"HN: {type(exc).__name__}: {exc}"]
    out = []
    for hit in data.get("hits", []):
        link = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID')}"
        out.append(item(hit.get("title") or "", link, "hackernews", "hn",
                        summary=hit.get("story_text") or "",
                        published=parse_date(str(hit.get("created_at_i"))),
                        extra={"points": hit.get("points"),
                               "comments": hit.get("num_comments"),
                               "discussion": f"https://news.ycombinator.com/item?id={hit.get('objectID')}"}))
    return out, []


def collect_arxiv(opener, limit=15):
    url = ("http://export.arxiv.org/api/query?"
           "search_query=cat:cs.AI+OR+cat:cs.LG+OR+cat:cs.CL"
           f"&sortBy=submittedDate&sortOrder=descending&max_results={limit}")
    try:
        root = ET.fromstring(fetch(url, opener).encode("utf-8", "replace"))
    except Exception as exc:
        return [], [f"arXiv: {type(exc).__name__}: {exc}"]
    out = []
    for entry in [el for el in root.iter() if strip_ns(el.tag) == "entry"]:
        data = {strip_ns(c.tag): c for c in entry}
        title = data["title"].text if data.get("title") is not None else ""
        summary = data["summary"].text if data.get("summary") is not None else ""
        link = data["id"].text if data.get("id") is not None else ""
        published = parse_date(data["published"].text) if data.get("published") is not None else None
        authors = [a.text for a in entry if strip_ns(a.tag) == "name"][:4]
        if title and link:
            out.append(item(title, link, "arxiv", "arxiv", summary=summary,
                            published=published,
                            extra={"authors": authors, "full_text": link.replace("/abs/", "/html/")}))
    return out, []


def collect_reddit(opener, per_sub=8):
    """Reddit: JSON-эндпоинты часто отдают 403, RSS-фид работает — читаем его."""
    out, errors = [], []
    for sub in REDDIT_SUBS:
        url = f"https://www.reddit.com/r/{sub}/top/.rss?t=week"
        try:
            root = ET.fromstring(fetch(url, opener).encode("utf-8", "replace"))
        except Exception as exc:
            errors.append(f"Reddit r/{sub}: {type(exc).__name__}: {exc}")
            continue
        for entry in [el for el in root.iter() if strip_ns(el.tag) == "entry"][:per_sub]:
            data = {strip_ns(c.tag): c for c in entry}
            title = data["title"].text if data.get("title") is not None else ""
            link = ""
            for child in entry:
                if strip_ns(child.tag) == "link" and child.attrib.get("href"):
                    link = child.attrib["href"]
                    break
            published = None
            for key in ("updated", "published"):
                if data.get(key) is not None and data[key].text:
                    published = parse_date(data[key].text)
                    break
            body = ""
            if data.get("content") is not None and data["content"].text:
                body = html_to_text(data["content"].text)
            if title and link:
                out.append(item(title, link, f"reddit/r/{sub}", "reddit",
                                summary=body, published=published))
    return out, errors


def collect_devto(opener, limit=15):
    try:
        data = json.loads(fetch(f"https://dev.to/api/articles?tag=ai&top=7&per_page={limit}", opener))
    except Exception as exc:
        return [], [f"Dev.to: {type(exc).__name__}: {exc}"]
    out = [item(a.get("title", ""), a.get("url", ""), "dev.to", "devto",
                summary=a.get("description", ""), published=parse_date(a.get("published_at")),
                extra={"reactions": a.get("public_reactions_count")})
           for a in data if a.get("title") and a.get("url")]
    return out, []


def collect_lobsters(opener, limit=15):
    try:
        data = json.loads(fetch("https://lobste.rs/t/ai.json", opener))
    except Exception as exc:
        return [], [f"Lobsters: {type(exc).__name__}: {exc}"]
    out = []
    for story in (data if isinstance(data, list) else [])[:limit]:
        url = story.get("url") or story.get("comments_url", "")
        if story.get("title") and url:
            out.append(item(story["title"], url, "lobsters", "lobsters",
                            published=parse_date(story.get("created_at")),
                            extra={"score": story.get("score"),
                                   "comments_url": story.get("comments_url")}))
    return out, []


GITHUB_TRENDING_RE = re.compile(
    r'href="/(?P<repo>[^"/]+/[^"]+?)"[^>]*>\s*(?:<[^>]+>\s*)*</a>.*?'
    r'(?:<p[^>]*>\s*(?P<desc>.*?)</p>)?', re.S)


def collect_github(opener, limit=12):
    """GitHub trending: HTML без стабильного API, поэтому парсим аккуратно."""
    try:
        page = fetch("https://github.com/trending?since=daily&spoken_language_code=en", opener)
    except Exception as exc:
        return [], [f"GitHub trending: {type(exc).__name__}: {exc}"]
    out = []
    for block in re.split(r'<article class="Box-row"', page)[1:]:
        m = re.search(r'<h2[^>]*>.*?href="/([^"?]+)"', block, re.S)
        if not m:
            continue
        repo = m.group(1).strip()
        desc = re.search(r'<p class="col-9[^"]*"[^>]*>(.*?)</p>', block, re.S)
        stars = re.search(r'aria-label="([\d,]+) users starred', block)
        out.append(item(repo, f"https://github.com/{repo}", "github/trending", "github",
                        summary=desc.group(1) if desc else "",
                        extra={"stars": stars.group(1) if stars else None}))
        if len(out) >= limit:
            break
    if not out:
        return [], ["GitHub trending: разметка не распознана (GitHub меняет HTML)"]
    return out, []


def collect_youtube(limit=8):
    """YouTube: нужен yt-dlp. Без него источник просто пропускается."""
    import shutil
    import subprocess
    if not shutil.which("yt-dlp"):
        return [], ["YouTube: yt-dlp не установлен (pip install yt-dlp) — источник пропущен"]
    out, errors = [], []
    for query in YOUTUBE_QUERIES:
        try:
            proc = subprocess.run(["yt-dlp", "--dump-single-json", "--flat-playlist",
                                   f"ytsearch{limit}:{query}"],
                                  capture_output=True, text=True, timeout=120)
            if proc.returncode != 0:
                errors.append(f"YouTube ({query}): {proc.stderr.strip()[:120]}")
                continue
            data = json.loads(proc.stdout)
        except Exception as exc:
            errors.append(f"YouTube ({query}): {type(exc).__name__}: {exc}")
            continue
        for entry in data.get("entries", []):
            if not entry or not entry.get("id"):
                continue
            out.append(item(entry.get("title", ""), f"https://youtube.com/watch?v={entry['id']}",
                            "youtube/search", "youtube",
                            words=0,
                            extra={"channel": entry.get("channel"), "query": query,
                                   "duration": entry.get("duration")}))
    return out, errors


COLLECTORS = {
    "rss": lambda opener, args: collect_rss(opener),
    "hn": lambda opener, args: collect_hn(opener),
    "arxiv": lambda opener, args: collect_arxiv(opener),
    "reddit": lambda opener, args: collect_reddit(opener),
    "devto": lambda opener, args: collect_devto(opener),
    "lobsters": lambda opener, args: collect_lobsters(opener),
    "github": lambda opener, args: collect_github(opener),
    "youtube": lambda opener, args: collect_youtube(),
}


# ─────────────────────────── ОЦЕНКА ───────────────────────────

def deep_check(entry, opener, timeout=DEFAULT_TIMEOUT):
    """Скачать страницу и посчитать слова: проверяем объём, а не предполагаем."""
    if entry["kind"] == "youtube":
        import shutil
        import subprocess
        if not shutil.which("yt-dlp"):
            return entry
        try:
            proc = subprocess.run(["yt-dlp", "--skip-download", "--write-auto-subs",
                                   "--write-subs", "--sub-lang", "en", "--sub-format", "vtt",
                                   "--dump-json", entry["url"]],
                                  capture_output=True, text=True, timeout=180)
            meta = json.loads(proc.stdout.splitlines()[0]) if proc.stdout.strip() else {}
            entry["extra"]["has_subs"] = bool(meta.get("subtitles") or meta.get("automatic_captions"))
            entry["extra"]["duration"] = meta.get("duration") or entry["extra"].get("duration")
            entry["full_words"] = None  # транскрипт проверяется отдельной командой yt-dlp
        except Exception as exc:
            entry["extra"]["deep_error"] = f"{type(exc).__name__}: {exc}"
        return entry
    if entry["kind"] == "arxiv":
        entry["full_words"] = entry["summary_words"]
        return entry
    try:
        text = html_to_text(fetch(entry["url"], opener, timeout=timeout))
        entry["full_words"] = word_count(text)
    except Exception as exc:
        entry["full_words"] = None
        entry["extra"]["deep_error"] = f"{type(exc).__name__}: {exc}"
    return entry


def score(entry):
    words = entry.get("full_words") or entry.get("summary_words") or 0
    kind = entry["kind"]
    threshold = MIN_WORDS.get(kind, 1000)
    ratio = min(words / max(threshold, 1), 3.0)
    age_days = None
    if entry.get("published"):
        dt = parse_date(entry["published"])
        if dt:
            age_days = max((datetime.now(timezone.utc) - dt).days, 0)
    fresh_bonus = 0.0 if age_days is None else max(0.0, 2.0 - age_days / 3)
    engagement = 0.0
    for key in ("points", "score", "reactions", "comments"):
        value = entry.get("extra", {}).get(key)
        if isinstance(value, (int, float)):
            engagement += min(value / 500.0, 1.5)
    return round(SOURCE_WEIGHT.get(kind, 1.0) + ratio + fresh_bonus + engagement, 2), {
        "words": words, "threshold": threshold, "age_days": age_days,
        "enough_content": words >= threshold,
    }


# ─────────────────────────── ДЕДУП ───────────────────────────

STOPWORDS = {"the", "a", "an", "and", "or", "of", "to", "in", "for", "on", "with",
             "is", "are", "was", "were", "how", "why", "what", "new", "ai", "и", "в",
             "на", "с", "для", "как", "что", "это", "не", "по", "из"}


def tokens(text: str) -> set:
    words = re.findall(r"[\wа-яё]+", (text or "").lower())
    return {w for w in words if len(w) > 2 and w not in STOPWORDS}


def load_published(blog_dir: str) -> list:
    """Читает уже опубликованные посты: slug, заголовки и frontmatter-ссылки."""
    published = []
    if not blog_dir or not os.path.isdir(blog_dir):
        return published
    for name in sorted(os.listdir(blog_dir)):
        if not name.endswith((".md", ".markdown")):
            continue
        path = os.path.join(blog_dir, name)
        try:
            text = open(path, encoding="utf-8", errors="replace").read()
        except OSError:
            continue
        title = ""
        fm = re.match(r"^---\s*\n(.*?)\n---", text, re.S)
        if fm:
            m = re.search(r'^title:\s*["\']?(.+?)["\']?\s*$', fm.group(1), re.M)
            if m:
                title = m.group(1)
        links = re.findall(r"https?://\S+", (fm.group(1) if fm else ""))
        published.append({
            "slug": name.rsplit(".", 1)[0],
            "title": title,
            "tokens": tokens(title + " " + name.rsplit(".", 1)[0]),
            "links": links,
        })
    return published


def is_duplicate(entry, published, seen_urls: set, threshold=0.5) -> str | None:
    url = entry["url"].rstrip("/")
    if url in seen_urls:
        return "url уже использовался"
    for post in published:
        if any(url == link.rstrip("/") for link in post["links"]):
            return f"url в frontmatter поста {post['slug']}"
    cand = tokens(entry["title"])
    if not cand:
        return None
    for post in published:
        if not post["tokens"]:
            continue
        overlap = len(cand & post["tokens"]) / max(len(cand), 1)
        if overlap >= threshold:
            return f"тема пересекается с {post['slug']} ({overlap:.0%})"
    return None


# ─────────────────────────── ВЫВОД ───────────────────────────

def write_markdown(path, plan, stats):
    lines = [f"# Контент-план ({datetime.now().strftime('%Y-%m-%d')})", ""]
    lines.append(f"Источников опрошено: {stats['sources_ok']}, ошибок: {len(stats['errors'])}. "
                 f"Кандидатов: {stats['raw']}, после дедупа: {stats['kept']}.")
    lines += ["", "## Темы", ""]
    for i, entry in enumerate(plan, 1):
        ev = entry["evaluation"]
        lines += [
            f"### {i}. {entry['title'][:80]}",
            f"- **Слот:** {entry['kind']}",
            f"- **Источник:** {entry['source']} — {entry['url']}",
            f"- **Объём:** {ev['words']} слов (порог {ev['threshold']}), "
            f"{'достаточно' if ev['enough_content'] else 'НЕДОСТАТОЧНО'}",
            f"- **Дата:** {entry.get('published') or '—'}",
            f"- **Оценка:** {entry['score']}",
            "",
        ]
    lines += ["## Отклонено", ""]
    for reason, items in stats["rejected"].items():
        if items:
            lines.append(f"- {reason}: {len(items)}")
    lines.append("")
    if stats["errors"]:
        lines += ["## Ошибки источников", ""]
        lines += [f"- {err}" for err in stats["errors"]]
        lines.append("")
    open(path, "w", encoding="utf-8").write("\n".join(lines))


def main():
    ap = argparse.ArgumentParser(description="Сбор кандидатов для контент-плана блога")
    ap.add_argument("--sources", default="rss,hn,arxiv,reddit,devto,lobsters,github,youtube",
                    help="через запятую: rss, hn, arxiv, reddit, devto, lobsters, github, youtube")
    ap.add_argument("--blog-dir", default="", help="каталог с уже опубликованными постами (.md)")
    ap.add_argument("--seen-file", default="", help="JSON со списком уже использованных URL")
    ap.add_argument("--limit", type=int, default=40, help="сколько кандидатов оставить")
    ap.add_argument("--deep", type=int, default=0, help="сколько кандидатов проверить скачиванием")
    ap.add_argument("--proxy", default="", help="например socks5://127.0.0.1:1080 или http://host:port")
    ap.add_argument("--out", default="", help="куда сохранить JSON (по умолчанию — stdout)")
    ap.add_argument("--md", default="", help="куда сохранить markdown-заготовку плана")
    args = ap.parse_args()

    opener = _opener(args.proxy or None)
    wanted = [s.strip() for s in args.sources.split(",") if s.strip()]
    entries, errors, sources_ok = [], [], 0

    for name in wanted:
        collector = COLLECTORS.get(name)
        if not collector:
            errors.append(f"неизвестный источник: {name}")
            continue
        t0 = time.time()
        found, errs = collector(opener, args)
        errors.extend(errs)
        entries.extend(found)
        if found:
            sources_ok += 1
        print(f"[{name}] {len(found)} кандидатов за {time.time() - t0:.1f}s "
              f"({len(errs)} ошибок)", file=sys.stderr)

    seen_urls = set()
    if args.seen_file and os.path.exists(args.seen_file):
        try:
            seen_urls = {u.rstrip("/") for u in json.load(open(args.seen_file))}
        except Exception as exc:
            errors.append(f"seen-file: {type(exc).__name__}: {exc}")
    published = load_published(args.blog_dir)

    raw = len(entries)
    rejected: dict[str, list] = {}
    kept = []
    for entry in entries:
        if not entry["url"] or not entry["title"]:
            rejected.setdefault("пустой заголовок или ссылка", []).append(entry)
            continue
        reason = is_duplicate(entry, published, seen_urls)
        if reason:
            rejected.setdefault(reason.split("(")[0].strip(), []).append(entry)
            continue
        kept.append(entry)

    for entry in kept:
        entry["score"], entry["evaluation"] = score(entry)

    # отсечь «не новости» старше FRESH_DAYS (arxiv/github/youtube не трогаем)
    fresh = []
    for entry in kept:
        age = entry["evaluation"]["age_days"]
        if (age is not None and age > FRESH_DAYS
                and entry["kind"] not in FRESH_DAYS_EXEMPT):
            rejected.setdefault(f"старше {FRESH_DAYS} дней", []).append(entry)
            continue
        fresh.append(entry)
    kept = fresh

    # Глубокую проверку (скачивание страницы и подсчёт слов) делаем для лучших
    # по предварительной оценке, а не для первых попавшихся.
    kept.sort(key=lambda e: e["score"], reverse=True)
    deep_checked = 0
    for entry in kept[:max(args.deep, 0)]:
        deep_check(entry, opener)
        entry["score"], entry["evaluation"] = score(entry)
        deep_checked += 1

    kept.sort(key=lambda e: e["score"], reverse=True)
    kept = kept[:args.limit]

    stats = {"raw": raw, "kept": len(kept), "sources_ok": sources_ok,
             "deep_checked": deep_checked, "errors": errors, "rejected": rejected}
    payload = {"generated_at": datetime.now(timezone.utc).isoformat(),
               "stats": {k: (len(v) if isinstance(v, dict) else v) for k, v in stats.items()},
               "candidates": kept}

    text = json.dumps(payload, ensure_ascii=False, indent=2)
    if args.out:
        open(args.out, "w", encoding="utf-8").write(text)
    else:
        print(text)
    if args.md:
        write_markdown(args.md, kept, stats)

    print(f"\nИтого: {raw} собрано → {len(kept)} отобрано "
          f"(дедуп против {len(published)} постов, ошибок: {len(errors)})", file=sys.stderr)


if __name__ == "__main__":
    main()