varundus 2026-09-26 07:00

third/server/tolkfetch/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+# Trafilatura (DOM-teadlik ekstraktimine) + curl otsefetchiks
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl ca-certificates \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip install --no-cache-dir trafilatura==2.2.0
+
+WORKDIR /app
+COPY tolke_fetch.py server.py ./
+
+EXPOSE 8192
+CMD ["python", "server.py"]

third/server/tolkfetch/docker-compose.yml

@@ -0,0 +1,20 @@
+name: tolkfetch
+
+services:
+  tolkfetch:
+    build: .
+    image: tolkfetch:1.0
+    container_name: tolkfetch
+    restart: unless-stopped
+    environment:
+      - TZ=Europe/Tallinn
+      - TOLKE_FLARE_URL=http://100.87.1.24:8191/v1
+    networks:
+      - ai_network
+    # Porte ei avaldata: teenus on kättesaadav AINULT main_ai_network võrgus
+    # (n8n ja runner on samas võrgus). Internetist ligipääs puudub.
+
+networks:
+  ai_network:
+    external: true
+    name: main_ai_network

third/server/tolkfetch/server.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""server.py — väike HTTP-teenus tõlkeboti fetch-mootorile.
+
+Endpointid:
+  POST /fetch   body {"url": "...", "mode": "auto|direct|solverr"} -> fetch() tulemus
+  GET  /health  -> {"ok": true}
+"""
+import json
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from tolke_fetch import fetch
+
+PORT = 8192
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "tolkfetch/1.0"
+
+    def _send(self, code, payload):
+        body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self):
+        if self.path.startswith("/health"):
+            self._send(200, {"ok": True, "service": "tolkfetch"})
+        else:
+            self._send(404, {"ok": False, "reason": "not found"})
+
+    def do_POST(self):
+        if not self.path.startswith("/fetch"):
+            self._send(404, {"ok": False, "reason": "not found"})
+            return
+        try:
+            n = int(self.headers.get("Content-Length") or 0)
+            data = json.loads(self.rfile.read(n) or b"{}")
+        except Exception as e:
+            self._send(400, {"ok": False, "reason": "halb JSON: " + str(e)[:120]})
+            return
+        try:
+            res = fetch(data.get("url") or "", data.get("mode") or "auto")
+        except Exception as e:
+            res = {"ok": False, "url": data.get("url", ""),
+                   "reason": "teenuse viga: " + str(e)[:200]}
+        self._send(200, res)
+
+    def log_message(self, fmt, *args):
+        pass
+
+
+if __name__ == "__main__":
+    srv = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
+    srv.serve_forever()

third/server/tolkfetch/tolke_fetch.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""tolke_fetch.py — URL-i lugemine tõlkeboti jaoks.
+
+Ahel: curl (otsefetch) -> vajadusel FlareSolverr (Cloudflare) -> Trafilatura (DOM).
+
+Kasutus käsureal:  echo '{"url":"..."}' | python3 tolke_fetch.py
+Raamatukoguna:     from tolke_fetch import fetch;  fetch(url)
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+
+UA = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+FLARE = os.environ.get("TOLKE_FLARE_URL", "http://100.87.1.24:8191/v1")
+
+# Tegelikud blokitunnused. NB: "challenge-platform" esineb KA edukal
+# Cloudflare-lehel (precursor-skript) -> seda üksi ei tohi blokiks lugeda.
+HARD_BLOCK_RE = re.compile(
+    r"just a moment|Enable JavaScript and cookies|Attention Required|"
+    r"_cf_chl_opt|cf-mitigated|Ray ID|cf-error-details|"
+    r"checking your browser|Verifying you are human|DDoS protection by",
+    re.I,
+)
+# 200-vastuse sees olev turvakontrolli tekst
+SOFT_BLOCK_RE = re.compile(
+    r"security service to protect itself|triggered the security solution|"
+    r"verify you are human|checking your browser",
+    re.I,
+)
+
+_extract = None
+_IMPORT_ERR = ""
+
+
+def _load_extractor():
+    global _extract, _IMPORT_ERR
+    if _extract is not None:
+        return _extract
+    try:
+        from trafilatura import extract as _e  # noqa
+
+        _extract = _e
+    except Exception as e:  # pragma: no cover
+        _IMPORT_ERR = str(e)[:200]
+    return _extract
+
+
+def run(cmd, timeout=180):
+    try:
+        p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+        return p.stdout
+    except Exception as e:
+        return "__ERR__" + str(e)[:200]
+
+
+def direct(url):
+    return run(
+        ["curl", "-sL", "--max-time", "60", "-A", UA,
+         "-H", "Accept-Language: et,en;q=0.9", "--compressed", url],
+        timeout=75,
+    )
+
+
+def solverr(url):
+    body = json.dumps({"cmd": "request.get", "url": url, "maxTimeout": 120000})
+    out = run(
+        ["curl", "-s", "--max-time", "145", "-X", "POST", FLARE,
+         "-H", "Content-Type: application/json", "-d", body],
+        timeout=160,
+    )
+    if out.startswith("__ERR__"):
+        return ""
+    try:
+        return (json.loads(out).get("solution") or {}).get("response") or ""
+    except Exception:
+        return ""
+
+
+def is_blocked(html):
+    if not html or len(html) < 2000:
+        return True
+    head = html[:60000]
+    if HARD_BLOCK_RE.search(head):
+        return True
+    if len(html) < 20000 and SOFT_BLOCK_RE.search(head):
+        return True
+    return False
+
+
+def extract(html):
+    """Trafilatura Python API. Tagastab (title, body_markdown)."""
+    ex = _load_extractor()
+    if not html or ex is None:
+        return "", ""
+    h = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.S | re.I)
+    h = re.sub(r"<!--.*?-->", "", h, flags=re.S)
+    try:
+        body = ex(h, output_format="markdown", include_comments=False) or ""
+    except Exception:
+        return "", ""
+    title = ""
+    m = re.match(r"^#\s+(.+?)\s*$", body, re.M)
+    if m:
+        title = m.group(1).strip()
+    return title, re.sub(r"\n{3,}", "\n\n", body).strip()
+
+
+def fetch(url, mode="auto"):
+    """Tagastab dict: {ok, url, title, body, chars, via, reason}."""
+    url = (url or "").strip()
+    if not url:
+        return {"ok": False, "url": "", "reason": "url puudub"}
+    if _load_extractor() is None:
+        return {"ok": False, "url": url,
+                "reason": "trafilatura import: " + _IMPORT_ERR}
+
+    via = "direct"
+    html = "" if mode == "solverr" else direct(url)
+    if mode == "auto" and is_blocked(html):
+        solved = solverr(url)
+        if solved and not is_blocked(solved):
+            html, via = solved, "solverr"
+
+    title, body = extract(html)
+
+    # Ka 200-vastuse tekst võib olla blokiteade -> proovi Solverrit
+    if (not body or len(body) < 120) and mode == "auto" and via == "direct":
+        solved = solverr(url)
+        if solved and not is_blocked(solved):
+            html, via = solved, "solverr"
+            title, body = extract(html)
+
+    if body and len(body) > 120 and SOFT_BLOCK_RE.search(body[:800]):
+        return {"ok": False, "url": url, "via": via,
+                "reason": "leht nõuab turvakontrolli (Cloudflare)",
+                "chars": len(body)}
+
+    if not body or len(body) < 120:
+        return {"ok": False, "url": url, "via": via,
+                "reason": "sisu ei õnnestunud ekstraktida (tõke või tühi leht)",
+                "chars": len(body or "")}
+
+    return {"ok": True, "url": url, "title": title, "body": body,
+            "chars": len(body), "via": via}
+
+
+def main():
+    try:
+        data = json.loads(sys.stdin.read() or "{}")
+    except Exception:
+        data = {}
+    res = fetch(data.get("url") or "", data.get("mode") or "auto")
+    print(json.dumps(res, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())