ohmu/third/server/tolkfetch/tolke_fetch.py

#!/usr/bin/env python3
"""tolke_fetch.py — URL-i lugemine tõlkeboti jaoks.

Ahel: curl (otsefetch) -> vajadusel FlareSolverr (Cloudflare) -> Trafilatura (DOM).

Kasutus käsureal:  echo '{"url":"..."}' | python3 tolke_fetch.py
Raamatukoguna:     from tolke_fetch import fetch;  fetch(url)
"""
import json
import os
import re
import subprocess
import sys

UA = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
)
FLARE = os.environ.get("TOLKE_FLARE_URL", "http://100.87.1.24:8191/v1")

# Tegelikud blokitunnused. NB: "challenge-platform" esineb KA edukal
# Cloudflare-lehel (precursor-skript) -> seda üksi ei tohi blokiks lugeda.
HARD_BLOCK_RE = re.compile(
    r"just a moment|Enable JavaScript and cookies|Attention Required|"
    r"_cf_chl_opt|cf-mitigated|Ray ID|cf-error-details|"
    r"checking your browser|Verifying you are human|DDoS protection by",
    re.I,
)
# 200-vastuse sees olev turvakontrolli tekst
SOFT_BLOCK_RE = re.compile(
    r"security service to protect itself|triggered the security solution|"
    r"verify you are human|checking your browser",
    re.I,
)

_extract = None
_IMPORT_ERR = ""


def _load_extractor():
    global _extract, _IMPORT_ERR
    if _extract is not None:
        return _extract
    try:
        from trafilatura import extract as _e  # noqa

        _extract = _e
    except Exception as e:  # pragma: no cover
        _IMPORT_ERR = str(e)[:200]
    return _extract


def run(cmd, timeout=180):
    try:
        p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
        return p.stdout
    except Exception as e:
        return "__ERR__" + str(e)[:200]


def direct(url):
    return run(
        ["curl", "-sL", "--max-time", "60", "-A", UA,
         "-H", "Accept-Language: et,en;q=0.9", "--compressed", url],
        timeout=75,
    )


def solverr(url):
    body = json.dumps({"cmd": "request.get", "url": url, "maxTimeout": 120000})
    out = run(
        ["curl", "-s", "--max-time", "145", "-X", "POST", FLARE,
         "-H", "Content-Type: application/json", "-d", body],
        timeout=160,
    )
    if out.startswith("__ERR__"):
        return ""
    try:
        return (json.loads(out).get("solution") or {}).get("response") or ""
    except Exception:
        return ""


def is_blocked(html):
    if not html or len(html) < 2000:
        return True
    head = html[:60000]
    if HARD_BLOCK_RE.search(head):
        return True
    if len(html) < 20000 and SOFT_BLOCK_RE.search(head):
        return True
    return False


def extract(html):
    """Trafilatura Python API. Tagastab (title, body_markdown)."""
    ex = _load_extractor()
    if not html or ex is None:
        return "", ""
    h = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.S | re.I)
    h = re.sub(r"<!--.*?-->", "", h, flags=re.S)
    try:
        body = ex(h, output_format="markdown", include_comments=False) or ""
    except Exception:
        return "", ""
    title = ""
    m = re.match(r"^#\s+(.+?)\s*$", body, re.M)
    if m:
        title = m.group(1).strip()
    return title, re.sub(r"\n{3,}", "\n\n", body).strip()


def fetch(url, mode="auto"):
    """Tagastab dict: {ok, url, title, body, chars, via, reason}."""
    url = (url or "").strip()
    if not url:
        return {"ok": False, "url": "", "reason": "url puudub"}
    if _load_extractor() is None:
        return {"ok": False, "url": url,
                "reason": "trafilatura import: " + _IMPORT_ERR}

    via = "direct"
    html = "" if mode == "solverr" else direct(url)
    if mode == "auto" and is_blocked(html):
        solved = solverr(url)
        if solved and not is_blocked(solved):
            html, via = solved, "solverr"

    title, body = extract(html)

    # Ka 200-vastuse tekst võib olla blokiteade -> proovi Solverrit
    if (not body or len(body) < 120) and mode == "auto" and via == "direct":
        solved = solverr(url)
        if solved and not is_blocked(solved):
            html, via = solved, "solverr"
            title, body = extract(html)

    if body and len(body) > 120 and SOFT_BLOCK_RE.search(body[:800]):
        return {"ok": False, "url": url, "via": via,
                "reason": "leht nõuab turvakontrolli (Cloudflare)",
                "chars": len(body)}

    if not body or len(body) < 120:
        return {"ok": False, "url": url, "via": via,
                "reason": "sisu ei õnnestunud ekstraktida (tõke või tühi leht)",
                "chars": len(body or "")}

    return {"ok": True, "url": url, "title": title, "body": body,
            "chars": len(body), "via": via}


def main():
    try:
        data = json.loads(sys.stdin.read() or "{}")
    except Exception:
        data = {}
    res = fetch(data.get("url") or "", data.get("mode") or "auto")
    print(json.dumps(res, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    sys.exit(main())