Initial commit

.env

@@ -0,0 +1,3 @@
+DEEPSEEK_API_KEY=sk-c6766d328c2446f78bfe509d3c7ad4b3
+WEAVIATE_URL=http://100.80.222.54:9020
+PDF_SOURCE_DIR=./data/pdfs

.gitignore

@@ -60,4 +60,5 @@ target/
 
 # Ardo
 data/logs
-
+data/pdfs
+..env.swp

README.md

@@ -1,19 +1,19 @@
-# Teadusartiklite Töötlussüsteem
-
-Süsteem teadusartiklite automaatseks töötlemiseks, analüüsiks ja salvestamiseks Weaviate'i baasi.
-
-## Funktsioonid
-
-1. **PDF töötlus**: Automaatne tekstieraldus ja struktureerimine
-2. **DeepSeek analüüs**: Põhjalike kokkuvõtete loomine eesti keeles
-3. **Võtmesõnade eraldamine**: Olulisemate mõistete identifitseerimine
-4. **Embeddingu loomine**: SentenceTransformers abil semantiliste vektorite genereerimine
-5. **Weaviate'i integreerimine**: Struktureeritud salvestamine ja otsing
-
-## Paigaldus
-
-1. Klooni repository:
-```bash
-git clone [repository-url]
-cd transpordi_artiklid
-```
+# Teadusartiklite Töötlussüsteem
+
+Süsteem teadusartiklite automaatseks töötlemiseks, analüüsiks ja salvestamiseks Weaviate'i baasi.
+
+## Funktsioonid
+
+1. **PDF töötlus**: Automaatne tekstieraldus ja struktureerimine
+2. **DeepSeek analüüs**: Põhjalike kokkuvõtete loomine eesti keeles
+3. **Võtmesõnade eraldamine**: Olulisemate mõistete identifitseerimine
+4. **Embeddingu loomine**: SentenceTransformers abil semantiliste vektorite genereerimine
+5. **Weaviate'i integreerimine**: Struktureeritud salvestamine ja otsing
+
+## Paigaldus
+
+1. Klooni repository:
+```bash
+git clone [repository-url]
+cd transpordi_artiklid
+```

check_weaviate.py

@@ -0,0 +1,23 @@
+# check_weaviate.py
+from src.weaviate_client import WeaviateClient
+import sys
+sys.path.insert(0, './src')
+
+client = WeaviateClient()
+collection = client.client.collections.get("ScientificArticle")
+
+# Loendi kokku
+count_response = collection.aggregate.over_all(total_count=True)
+total = count_response.total_count
+print(f"\n✅ Weaviate'is on {total} artiklit.")
+
+# Võta mõni näidis välja
+if total > 0:
+    print("\n📄 Esimesed 3 artiklit:")
+    response = collection.query.fetch_objects(limit=3)
+    for i, obj in enumerate(response.objects):
+        print(f"\n{i+1}. ID: {obj.properties.get('article_id', 'N/A')}")
+        print(f"   Pealkiri: {obj.properties.get('title', 'N/A')}")
+        print(f"   Autorid: {obj.properties.get('authors', ['N/A'])}")
+
+client.close()

peamine.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+Peamine skript artiklite töötlemiseks ja Weaviate'i salvestamiseks
+"""
+
+import sys
+import os
+
+# Lisa src kaust Pythoni teele
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
+
+from src.pipeline import ArticleProcessingPipeline
+from src.utils import setup_logging
+
+def main():
+    """Peamine funktsioon"""
+    # Seadista logimine
+    logger = setup_logging()
+    
+    try:
+        logger.info("ALUSTAN ARTIKLITE TÖÖTLUST")
+        logger.info("="*60)
+        
+        # Käivita pipeline
+        pipeline = ArticleProcessingPipeline()
+        results = pipeline.run()
+        
+        logger.info("TÖÖTLUS LÕPETATUD")
+        
+        if results:
+            logger.info(f"Edukalt töödeldud {len(results)} artiklit")
+            # Näita esimese artikli näidet
+            if len(results) > 0:
+                first_article = results[0]
+                logger.info("\nESIMESE ARTIKLI NÄIDE:")
+                logger.info(f"Pealkiri: {first_article.get('title', 'N/A')}")
+                logger.info(f"Autorid: {', '.join(first_article.get('authors', []))}")
+                logger.info(f"Võtmesõnu: {len(first_article.get('key_concepts', []))}")
+                logger.info(f"Kokkuvõtte pikkus: {len(first_article.get('summary_et', ''))} märki")
+        
+        # Sulge ressursid
+        pipeline.close()
+        
+    except Exception as e:
+        logger.error(f"Viga peaprogrammis: {str(e)}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,8 @@
+sentence-transformers
+torch
+transformers
+chromadb
+langchain
+langchain-community
+pypdf
+PyPDF2

src/config.py

@@ -0,0 +1,53 @@
+import os
+from dotenv import load_dotenv
+from dataclasses import dataclass
+from typing import Optional, List, Dict
+
+# Lae keskkonnamuutujad
+load_dotenv()
+
+@dataclass
+class Config:
+    """Rakenduse konfiguratsioon"""
+    
+    # DeepSeek API
+    deepseek_api_key: str = os.getenv("DEEPSEEK_API_KEY", "")
+    deepseek_base_url: str = "https://api.deepseek.com"
+    deepseek_model: str = "deepseek-chat"
+    
+    # Weaviate
+    weaviate_url: str = os.getenv("WEAVIATE_URL", "http://localhost:8080")
+    weaviate_api_key: Optional[str] = os.getenv("WEAVIATE_API_KEY")
+    
+    # Failisüsteem
+    pdf_source_dir: str = os.getenv("PDF_SOURCE_DIR", "./data/pdfs")
+    processed_dir: str = os.getenv("PROCESSED_DIR", "./data/processed")
+    log_dir: str = os.getenv("LOG_DIR", "./data/logs")
+    
+    # Töötlusparameetrid
+    max_tokens: int = int(os.getenv("MAX_TOKENS", "4000"))
+    batch_size: int = int(os.getenv("BATCH_SIZE", "5"))
+    
+    # Embedding mudel
+    embedding_model: str = "all-MiniLM-L6-v2"
+    
+    # Veel seadeid
+    enable_logging: bool = True
+    language: str = "et"  # eesti keel kokkuvõtete jaoks
+    
+    def validate(self):
+        """Valideeri konfiguratsioon"""
+        if not self.deepseek_api_key:
+            raise ValueError("DeepSeek API võti on puudu! Lisa see .env faili.")
+        
+        if not os.path.exists(self.pdf_source_dir):
+            os.makedirs(self.pdf_source_dir, exist_ok=True)
+            print(f"Loodi PDF kaust: {self.pdf_source_dir}")
+        
+        for directory in [self.processed_dir, self.log_dir]:
+            os.makedirs(directory, exist_ok=True)
+        
+        return self
+
+# Loo globaalne konfiguratsioon
+config = Config().validate()

src/deepseek_client.py

@@ -0,0 +1,212 @@
+import time
+import json
+from typing import Dict, List, Optional, Any
+from tenacity import retry, stop_after_attempt, wait_exponential
+import logging
+from openai import OpenAI
+
+from .config import config
+
+logger = logging.getLogger(__name__)
+
+class DeepSeekClient:
+    """DeepSeek API kliendi klass"""
+    
+    def __init__(self):
+        self.client = OpenAI(
+            api_key=config.deepseek_api_key,
+            base_url=config.deepseek_base_url
+        )
+        self.logger = logging.getLogger(__name__)
+        self.max_tokens = config.max_tokens
+    
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=4, max=10)
+    )
+    def call_api(self, messages: List[Dict], temperature: float = 0.7) -> str:
+        """Kutsu DeepSeek API-d"""
+        try:
+            response = self.client.chat.completions.create(
+                model=config.deepseek_model,
+                messages=messages,
+                max_tokens=self.max_tokens,
+                temperature=temperature
+            )
+            
+            content = response.choices[0].message.content
+            tokens_used = response.usage.total_tokens if hasattr(response, 'usage') else 0
+            
+            self.logger.info(f"API kutsutud, tokenid: {tokens_used}")
+            return content
+            
+        except Exception as e:
+            self.logger.error(f"API viga: {str(e)}")
+            raise
+    
+    def create_summary(self, text: str, context: Dict) -> Dict:
+        """Loo põhjalik kokkuvõte artikkelist"""
+        self.logger.info("Loon artikli kokkuvõtte DeepSeek-iga...")
+        
+        # Koosta süsteemipromt
+        system_prompt = f"""Sa oled transpordiplaneerimise spetsialist ja teadusartiklite analüütik. 
+Loo põhjalik kokkuvõte antud teadusartiklist EESTI KEELES. 
+Kokkuvõte peaks sisaldama järgmisi osi:
+
+1. ARTIKLI PEAMISED PUNKTID:
+   - Uurimisküsimused ja eesmärgid
+   - Teaduslik tähtsus
+   - Uudne panus valdkonda
+
+2. KASUTATUD MEETODID:
+   - Andmete kogumise meetodid
+   - Analüüsimeetodid
+   - Mudelid ja algoritmid
+
+3. PEAMISED TULEMUSED:
+   - Olulisemad leidud
+   - Statistilised tulemused
+   - Mudeli täpsus ja piirangud
+
+4. JÄRELDUSED JA SOOVITUSED:
+   - Peamised järeldused
+   - Rakendussoovitused
+   - Edasised uurimissuunad
+
+5. TRANSFORDIPLANEERIMISE KONTEKST:
+   - Kuidas aitab see artikkel paremat transpordisüsteemi kujundada?
+   - Millised on praktilised rakendused?
+   - Mida võiks Eesti tingimustes rakendada?
+
+Kasuta selget ja asjakohast keelt. Olge konkreetne ja viida konkreetsetele leidudele.
+"""
+        
+        # Koosta kasutaja promt
+        user_prompt = f"""Loo põhjalik kokkuvõte järgmisest teadusartiklist:
+
+ARTIKLI INFO:
+Pealkiri: {context.get('title', 'Teadmata')}
+Autorid: {', '.join(context.get('authors', []))}
+Aasta: {context.get('year', 'Teadmata')}
+Žurnaal: {context.get('journal', 'Teadmata')}
+
+ARTIKLI SISU (esimesed 6000 märki):
+{text[:6000]}
+
+Palun loo struktureeritud kokkuvõte ülaltoodud nõuete järgi.
+"""
+        
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+        
+        summary = self.call_api(messages, temperature=0.7)
+        
+        return {
+            "summary_et": summary,
+            "summary_length": len(summary),
+            "language": "et"
+        }
+    
+    def extract_key_concepts(self, text: str, summary: str) -> List[str]:
+        """Eralda artikli olulisemad mõisted ja võtmesõnad"""
+        self.logger.info("Eraldan võtmesõnu...")
+        
+        system_prompt = """Sa oled teadusartiklite analüütik. Eralda antud artiklist 5-10 olulisemat 
+mõistet/võtmesõna, mis iseloomustavad artikli põhisisu. Tagasta need komadega eraldatud nimekirjana.
+"""
+        
+        user_prompt = f"""Artikli kokkuvõte:
+{summary[:1000]}
+
+Artikli tekst (osa):
+{text[:2000]}
+
+Palun eralda olulisemad mõisted. Tagasta VAID komadega eraldatud nimekiri.
+"""
+        
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+        
+        response = self.call_api(messages, temperature=0.3)
+        
+        # Puhasta vastus
+        concepts = [c.strip() for c in response.split(',') if c.strip()]
+        concepts = concepts[:10]  # Piirangu 10 mõistele
+        
+        return concepts
+    
+    def identify_methods(self, text: str) -> List[str]:
+        """Tuvasta artiklis kasutatud meetodid"""
+        self.logger.info("Tuvastan kasutatud meetodeid...")
+        
+        system_prompt = """Sa oled teadusmeetodite spetsialist. Tuvasta antud teadusartiklist 
+kasutatud peamised meetodid (nt: regressioonanalüüs, simulatsioon, juhtumianalüüs jne).
+Tagasta komadega eraldatud nimekiri.
+"""
+        
+        user_prompt = f"""Artikli tekst (osa):
+{text[:3000]}
+
+Palun tuvasta kasutatud meetodid. Tagasta VAID komadega eraldatud nimekiri.
+"""
+        
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+        
+        response = self.call_api(messages, temperature=0.3)
+        
+        methods = [m.strip() for m in response.split(',') if m.strip()]
+        return methods
+    
+    def analyze_transport_context(self, summary: str) -> Dict:
+        """Analüüsi artikli tähtsus transpordiplaneerimise kontekstis"""
+        self.logger.info("Analüüsin transpordi konteksti...")
+        
+        system_prompt = """Sa oled transpordiplaneerimise ekspert. Analüüsi antud teadusartikli 
+tähtsust ja rakendatavust transpordisüsteemide planeerimisel.
+
+Hinda järgmisi aspekte:
+1. Teoreetiline panus
+2. Praktiline rakendatavus
+3. Reaalsete probleemide lahendamine
+4. Piirangud ja edasised vajadused
+"""
+        
+        user_prompt = f"""Artikli kokkuvõte:
+{summary}
+
+Palun analüüsi artikli tähtsust transpordiplaneerimise kontekstis. Tagasta JSON formaadis:
+{{
+  "theoretical_contribution": "lühikirjeldus",
+  "practical_applicability": "lühikirjeldus",
+  "problem_solving": "lühikirjeldus",
+  "limitations": "lühikirjeldus",
+  "relevance_score": 1-10 (kus 10 on kõige relevantsem)
+}}
+"""
+        
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
+        
+        response = self.call_api(messages, temperature=0.5)
+        
+        try:
+            # Proovi parsida JSON vastust
+            if response.startswith('{') and response.endswith('}'):
+                return json.loads(response)
+        except:
+            pass
+        
+        # Kui JSON parsimine ei õnnestu, tagasta struktureeritud tekst
+        return {
+            "analysis": response,
+            "relevance_score": 5  # Vaikimisi skoor
+        }

src/embedding_generator.py

@@ -0,0 +1,104 @@
+import numpy as np
+from typing import List, Dict
+from sentence_transformers import SentenceTransformer
+import logging
+
+from .config import config
+
+logger = logging.getLogger(__name__)
+
+class EmbeddingGenerator:
+    """Embeddingute genereerimine SentenceTransformers abil"""
+    
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self.model_name = config.embedding_model
+        self.model = None
+        
+    def load_model(self):
+        """Lae embedding mudel"""
+        if self.model is None:
+            self.logger.info(f"Laadin embedding mudelit: {self.model_name}")
+            self.model = SentenceTransformer(self.model_name)
+            self.logger.info("Embedding mudel laetud")
+        return self.model
+    
+    def generate_embedding(self, text: str) -> List[float]:
+        """Genereeri embedding ühele tekstile"""
+        model = self.load_model()
+        
+        # Kui tekst on tühi, tagasta nullvektor
+        if not text or len(text.strip()) < 10:
+            return [0.0] * 384  # all-MiniLM-L6-v2 annab 384 dimensiooni
+        
+        # Lühikesteks tekstidele lisa kontekst
+        if len(text) < 50:
+            text = f"teadusartikkel: {text}"
+        
+        try:
+            embedding = model.encode(text, convert_to_numpy=True).tolist()
+            return embedding
+        except Exception as e:
+            self.logger.error(f"Viga embeddingu genereerimisel: {str(e)}")
+            return [0.0] * 384
+    
+    def generate_embeddings_batch(self, texts: List[str]) -> List[List[float]]:
+        """Genereeri embeddingud partii kaupa"""
+        model = self.load_model()
+        
+        # Filtreeri tühjad tekstid
+        valid_texts = []
+        indices = []
+        for i, text in enumerate(texts):
+            if text and len(text.strip()) > 10:
+                valid_texts.append(text)
+                indices.append(i)
+        
+        if not valid_texts:
+            return [[] for _ in range(len(texts))]
+        
+        try:
+            embeddings = model.encode(valid_texts, convert_to_numpy=True)
+            
+            # Aseta embeddingud tagasi õigetesse kohtadesse
+            result = [[] for _ in range(len(texts))]
+            for idx, embedding in zip(indices, embeddings):
+                result[idx] = embedding.tolist()
+            
+            return result
+        except Exception as e:
+            self.logger.error(f"Viga batch embeddingu genereerimisel: {str(e)}")
+            return [[] for _ in range(len(texts))]
+    
+    def generate_article_embeddings(self, article_data: Dict) -> Dict:
+        """Genereeri embeddingud kogu artikli jaoks"""
+        embeddings = {}
+        
+        # Kokkuvõtte embedding
+        if 'summary_et' in article_data:
+            summary = article_data['summary_et']
+            embeddings['summary'] = self.generate_embedding(summary)
+        
+        # Abstrakti embedding
+        if 'abstract_en' in article_data:
+            abstract = article_data.get('abstract_en', '')
+            embeddings['abstract'] = self.generate_embedding(abstract)
+        
+        # Pealkirja embedding
+        if 'title' in article_data:
+            title = article_data['title']
+            embeddings['title'] = self.generate_embedding(title)
+        
+        # Sektsioonide embeddingud
+        if 'sections' in article_data:
+            section_embeddings = {}
+            for section in article_data['sections']:
+                section_type = section.get('section_type', 'unknown')
+                content = section.get('content', '')
+                if content:
+                    section_embeddings[section_type] = self.generate_embedding(content)
+            
+            if section_embeddings:
+                embeddings['sections'] = section_embeddings
+        
+        return embeddings

src/pdf_processor.py

@@ -0,0 +1,229 @@
+import os
+import re
+import hashlib
+import PyPDF2
+from typing import Dict, List, Optional, Tuple
+from datetime import datetime
+import json
+from dataclasses import dataclass, asdict
+import logging
+
+from .config import config
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class PDFMetadata:
+    """PDF faili metainfo"""
+    filename: str
+    filepath: str
+    file_hash: str
+    file_size: int
+    page_count: int
+    creation_date: str
+    modification_date: str
+
+@dataclass
+class ArticleSection:
+    """Artikli sektsioon"""
+    section_type: str  # abstract, introduction, methodology, results, conclusion
+    content: str
+    page_start: int
+    page_end: int
+
+class PDFProcessor:
+    """PDF failide töötlemine"""
+    
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        
+    def calculate_file_hash(self, filepath: str) -> str:
+        """Arvuta faili hash kontrollimaks duplikaate"""
+        with open(filepath, 'rb') as f:
+            file_hash = hashlib.md5(f.read()).hexdigest()
+        return file_hash
+    
+    def extract_metadata(self, filepath: str) -> PDFMetadata:
+        """Eralda PDF faili metainfo"""
+        filename = os.path.basename(filepath)
+        file_size = os.path.getsize(filepath)
+        file_hash = self.calculate_file_hash(filepath)
+        
+        # Saada faili muutmise ja loomise kuupäevad
+        creation_time = os.path.getctime(filepath)
+        modification_time = os.path.getmtime(filepath)
+        
+        # Loe PDF
+        with open(filepath, 'rb') as file:
+            pdf_reader = PyPDF2.PdfReader(file)
+            page_count = len(pdf_reader.pages)
+        
+        return PDFMetadata(
+            filename=filename,
+            filepath=filepath,
+            file_hash=file_hash,
+            file_size=file_size,
+            page_count=page_count,
+            creation_date=datetime.fromtimestamp(creation_time).isoformat(),
+            modification_date=datetime.fromtimestamp(modification_time).isoformat()
+        )
+    
+    def extract_text_from_pdf(self, filepath: str) -> Tuple[str, List[ArticleSection]]:
+        """
+        Eralda tekst PDF failist ja proovi automaatselt struktureerida
+        
+        Tagastab:
+            - Kogu tekst
+            - Struktureeritud sektsioonid
+        """
+        self.logger.info(f"Eraldan teksti failist: {filepath}")
+        
+        with open(filepath, 'rb') as file:
+            pdf_reader = PyPDF2.PdfReader(file)
+            
+            # Kogu kogu teksti
+            full_text = ""
+            sections = []
+            current_section = None
+            
+            for page_num, page in enumerate(pdf_reader.pages, 1):
+                page_text = page.extract_text()
+                if not page_text:
+                    continue
+                
+                full_text += f"\n--- Page {page_num} ---\n{page_text}\n"
+                
+                # Proovi leida sektsioonide pealkirju
+                section_patterns = {
+                    'abstract': r'(?i)abstract|kokkuvõte|abr?strakt',
+                    'introduction': r'(?i)introduction|sissejuhatus|1\.\s*introduction|1\.\s*sissejuhatus',
+                    'methodology': r'(?i)methodology|meetodid|methods|2\.\s*method|3\.\s*meetod',
+                    'results': r'(?i)results|tulemused|4\.\s*results|5\.\s*tulemused',
+                    'conclusion': r'(?i)conclusion|järeldused|discussion|6\.\s*conclusion|7\.\s*järeldused',
+                    'references': r'(?i)references|viited|bibliography|kirjandus'
+                }
+                
+                lines = page_text.split('\n')
+                for line in lines:
+                    line_lower = line.lower().strip()
+                    
+                    for section_type, pattern in section_patterns.items():
+                        if re.search(pattern, line_lower) and len(line) < 200:
+                            # Sulge eelmine sektsioon kui oli avatud
+                            if current_section:
+                                current_section.content = current_section.content.strip()
+                                sections.append(current_section)
+                            
+                            # Alusta uut sektsiooni
+                            current_section = ArticleSection(
+                                section_type=section_type,
+                                content=f"## {line}\n",
+                                page_start=page_num,
+                                page_end=page_num
+                            )
+                            break
+                    
+                    # Lisa rida praegusesse sektsiooni
+                    if current_section:
+                        current_section.content += line + "\n"
+                        current_section.page_end = page_num
+            
+            # Lisa viimane sektsioon
+            if current_section:
+                current_section.content = current_section.content.strip()
+                sections.append(current_section)
+        
+        self.logger.info(f"Eraldati {len(sections)} sektsiooni")
+        
+        return full_text, sections
+    
+    def extract_structured_metadata(self, text: str) -> Dict:
+        """Proovi eraldada struktureeritud metainfo tekstist"""
+        metadata = {
+            'title': '',
+            'authors': [],
+            'year': '',
+            'journal': '',
+            'doi': '',
+            'keywords': []
+        }
+        
+        # Otsi pealkirja (esimene suurem rida)
+        lines = text.split('\n')
+        for line in lines:
+            line = line.strip()
+            if len(line) > 20 and len(line) < 200 and not line.startswith('http'):
+                if not metadata['title'] and line[0].isupper():
+                    metadata['title'] = line
+                break
+        
+        # Otsi autoreid (tüüpiline muster)
+        for i, line in enumerate(lines):
+            if 'author' in line.lower() or 'authors' in line.lower():
+                # Proovi järgmised 3 rida
+                for j in range(1, 4):
+                    if i + j < len(lines):
+                        author_line = lines[i + j].strip()
+                        if author_line and len(author_line) < 300:
+                            # Eralda nimed komade või 'and' järgi
+                            authors = re.split(r',|\band\b|;', author_line)
+                            metadata['authors'] = [a.strip() for a in authors if a.strip()]
+                            break
+        
+        # Otsi aastat
+        year_pattern = r'\((\d{4})\)|(\d{4})\s*[A-Z]'
+        for line in lines:
+            match = re.search(year_pattern, line)
+            if match:
+                metadata['year'] = match.group(1) or match.group(2)
+                break
+        
+        # Otsi DOI
+        doi_pattern = r'doi:\s*([^\s]+|10\.\d{4,9}/[-._;()/:A-Z0-9]+)'
+        for line in lines:
+            match = re.search(doi_pattern, line, re.IGNORECASE)
+            if match:
+                metadata['doi'] = match.group(1)
+                break
+        
+        # Otsi võtmesõnu
+        for line in lines:
+            if 'keyword' in line.lower():
+                # Proovi järgmised read
+                for j in range(1, 3):
+                    if i + j < len(lines):
+                        kw_line = lines[i + j].strip()
+                        if kw_line:
+                            metadata['keywords'] = [k.strip() for k in re.split(r',|;', kw_line)]
+                            break
+        
+        return metadata
+    
+    def process_pdf(self, filepath: str) -> Dict:
+        """Töötle üks PDF fail"""
+        try:
+            # Eralda metainfo
+            metadata = self.extract_metadata(filepath)
+            
+            # Eralda tekst
+            full_text, sections = self.extract_text_from_pdf(filepath)
+            
+            # Eralda struktureeritud metainfo
+            structured_meta = self.extract_structured_metadata(full_text)
+            
+            # Ühenda kõik andmed
+            result = {
+                'pdf_metadata': asdict(metadata),
+                'structured_metadata': structured_meta,
+                'full_text': full_text[:5000],  # Säästa mälu, salvesta ainult algus
+                'sections': [asdict(s) for s in sections],
+                'processing_date': datetime.now().isoformat(),
+                'word_count': len(full_text.split())
+            }
+            
+            self.logger.info(f"PDF töödeldud: {metadata.filename}")
+            return result
+            
+        except Exception as e:
+            self.logger.error(f"Viga PDF töötlemisel {filepath}: {str(e)}")
+            raise

src/pipeline.py

@@ -0,0 +1,199 @@
+import os
+import json
+import time
+from typing import List, Dict, Optional
+import logging
+from datetime import datetime
+from tqdm import tqdm
+
+from .pdf_processor import PDFProcessor
+from .deepseek_client import DeepSeekClient
+from .embedding_generator import EmbeddingGenerator
+from .weaviate_client import WeaviateClient
+from .config import config
+from .utils import setup_logging, save_processed_article
+
+logger = logging.getLogger(__name__)
+
+class ArticleProcessingPipeline:
+    """Artikli töötluspipeline"""
+    
+    def __init__(self):
+        self.pdf_processor = PDFProcessor()
+        self.deepseek_client = DeepSeekClient()
+        self.embedding_generator = EmbeddingGenerator()
+        self.weaviate_client = WeaviateClient()
+        
+        self.stats = {
+            'processed': 0,
+            'saved': 0,
+            'skipped': 0,
+            'errors': 0
+        }
+    
+    def process_single_article(self, pdf_path: str) -> Optional[Dict]:
+        """Töötle üks artikkel"""
+        try:
+            logger.info(f"Alustan töötlemist: {pdf_path}")
+            
+            # 1. PDF töötlus
+            pdf_data = self.pdf_processor.process_pdf(pdf_path)
+            
+            # 2. Eralda abstrakt (kui leiad)
+            abstract_en = ""
+            for section in pdf_data['sections']:
+                if section['section_type'] == 'abstract':
+                    abstract_en = section['content']
+                    break
+            
+            # 3. DeepSeek analüüs
+            summary_data = self.deepseek_client.create_summary(
+                text=pdf_data['full_text'],
+                context={
+                    'title': pdf_data['structured_metadata'].get('title', ''),
+                    'authors': pdf_data['structured_metadata'].get('authors', []),
+                    'year': pdf_data['structured_metadata'].get('year', ''),
+                    'journal': pdf_data['structured_metadata'].get('journal', '')
+                }
+            )
+            
+            # 4. Eralda võtmesõnad
+            key_concepts = self.deepseek_client.extract_key_concepts(
+                text=pdf_data['full_text'],
+                summary=summary_data['summary_et']
+            )
+            
+            # 5. Tuvasta meetodid
+            methods_used = self.deepseek_client.identify_methods(pdf_data['full_text'])
+            
+            # 6. Analüüsi transpordi kontekst
+            transport_context = self.deepseek_client.analyze_transport_context(
+                summary_data['summary_et']
+            )
+            
+            # 7. Koosta lõplik artikkel
+            article_data = {
+                # PDF metainfo
+                'pdf_metadata': pdf_data['pdf_metadata'],
+                'source_file': pdf_path,
+                'file_hash': pdf_data['pdf_metadata']['file_hash'],
+                
+                # Struktureeritud metainfo
+                'title': pdf_data['structured_metadata'].get('title', ''),
+                'authors': pdf_data['structured_metadata'].get('authors', []),
+                'year': pdf_data['structured_metadata'].get('year', ''),
+                'journal': pdf_data['structured_metadata'].get('journal', ''),
+                'doi': pdf_data['structured_metadata'].get('doi', ''),
+                
+                # Tekstisisu
+                'abstract_en': abstract_en,
+                'summary_et': summary_data['summary_et'],
+                'key_concepts': key_concepts,
+                'methods_used': methods_used,
+                
+                # Analüüs
+                'transport_context': transport_context,
+                'relevance_score': transport_context.get('relevance_score', 5),
+                
+                # Töötlusinfo
+                'processing_date': datetime.now().isoformat(),
+                'word_count': pdf_data['word_count'],
+                'section_count': len(pdf_data['sections'])
+            }
+            
+            # 8. Genereeri embeddingud
+            embeddings = self.embedding_generator.generate_article_embeddings(article_data)
+            
+            # 9. Salvesta Weaviate'i
+            saved = self.weaviate_client.save_article(article_data, embeddings)
+            
+            # 10. Salvesta töödeldud andmed faili
+            processed_data = {
+                'article': article_data,
+                'embeddings_summary': embeddings.get('summary', []),
+                'processing_stats': {
+                    'pdf_pages': pdf_data['pdf_metadata']['page_count'],
+                    'summary_length': summary_data['summary_length'],
+                    'concepts_count': len(key_concepts),
+                    'methods_count': len(methods_used)
+                }
+            }
+            
+            save_processed_article(processed_data)
+            
+            # Uuenda statistikat
+            if saved:
+                self.stats['saved'] += 1
+                logger.info(f"Artikkel edukalt salvestatud: {article_data['title']}")
+            else:
+                self.stats['skipped'] += 1
+                logger.info(f"Artikkel jäeti vahele (duplikaat): {article_data['title']}")
+            
+            self.stats['processed'] += 1
+            return article_data
+            
+        except Exception as e:
+            self.stats['errors'] += 1
+            logger.error(f"Viga artikli töötlemisel {pdf_path}: {str(e)}")
+            return None
+    
+    def process_batch(self, pdf_files: List[str]) -> List[Dict]:
+        """Töötle partii PDF faile"""
+        results = []
+        
+        for pdf_file in tqdm(pdf_files, desc="Töötlen artikleid"):
+            result = self.process_single_article(pdf_file)
+            if result:
+                results.append(result)
+            
+            # Väike paus, et mitte üle koormata API-d
+            time.sleep(1)
+        
+        return results
+    
+    def run(self):
+        """Käivita terve pipeline"""
+        logger.info("Käivitan artiklite töötluspipeline")
+        
+        # Leia PDF failid
+        pdf_files = self.get_pdf_files()
+        
+        if not pdf_files:
+            logger.warning(f"Ei leidnud PDF faile kaustas: {config.pdf_source_dir}")
+            return
+        
+        logger.info(f"Leidsin {len(pdf_files)} PDF faili")
+        
+        # Töötle failid
+        results = self.process_batch(pdf_files)
+        
+        # Kuva statistikat
+        self.print_stats()
+        
+        return results
+    
+    def get_pdf_files(self) -> List[str]:
+        """Leia kõik PDF failid kaustast"""
+        pdf_files = []
+        
+        for root, dirs, files in os.walk(config.pdf_source_dir):
+            for file in files:
+                if file.lower().endswith('.pdf'):
+                    pdf_files.append(os.path.join(root, file))
+        
+        return sorted(pdf_files)
+    
+    def print_stats(self):
+        """Prindi töötlusstatistika"""
+        logger.info("\n" + "="*50)
+        logger.info("TÖÖTLUSSTATISTIKA")
+        logger.info("="*50)
+        logger.info(f"Töödeldud faile: {self.stats['processed']}")
+        logger.info(f"Salvestatud artikleid: {self.stats['saved']}")
+        logger.info(f"Vahele jäetud (duplikaadid): {self.stats['skipped']}")
+        logger.info(f"Vigaseid töötlusi: {self.stats['errors']}")
+        logger.info("="*50)
+    
+    def close(self):
+        """Sulge ressursid"""
+        self.weaviate_client.close()

src/utils.py

@@ -0,0 +1,65 @@
+import os
+import json
+import logging
+from datetime import datetime
+from typing import Dict, Any, List
+
+from .config import config
+
+def setup_logging():
+    """Seadista logimine"""
+    log_file = os.path.join(config.log_dir, f"processing_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log")
+    
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.FileHandler(log_file, encoding='utf-8'),
+            logging.StreamHandler()
+        ]
+    )
+    
+    return logging.getLogger(__name__)
+
+def save_processed_article(data: Dict[str, Any]):
+    """Salvesta töödeldud artikkel JSON failina"""
+    try:
+        # Genereeri failinimi
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = f"article_{timestamp}.json"
+        filepath = os.path.join(config.processed_dir, filename)
+        
+        # Salvesta JSON failina
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+        
+        logger = logging.getLogger(__name__)
+        logger.info(f"Töödeldud andmed salvestatud: {filepath}")
+        
+    except Exception as e:
+        logger = logging.getLogger(__name__)
+        logger.error(f"Viga andmete salvestamisel: {str(e)}")
+
+def load_processed_articles() -> List[Dict]:
+    """Lae töödeldud artiklid"""
+    articles = []
+    
+    for filename in os.listdir(config.processed_dir):
+        if filename.endswith('.json'):
+            filepath = os.path.join(config.processed_dir, filename)
+            try:
+                with open(filepath, 'r', encoding='utf-8') as f:
+                    articles.append(json.load(f))
+            except:
+                continue
+    
+    return articles
+
+def clean_filename(filename: str) -> str:
+    """Puhasta failinimi erimärkidest"""
+    import re
+    # Eemalda erimärgid, jäta ainult tähed, numbrid, tühikud, punktid ja sidekriipsud
+    clean = re.sub(r'[^\w\s\.\-]', '', filename)
+    # Asenda mitmik tühikud ühega
+    clean = re.sub(r'\s+', ' ', clean)
+    return clean.strip()

src/weaviate_client.py

@@ -0,0 +1,340 @@
+import weaviate
+import hashlib
+import json
+from typing import Dict, List, Optional, Any
+import logging
+from urllib.parse import urlparse
+
+from .config import config
+
+logger = logging.getLogger(__name__)
+
+class WeaviateClient:
+    """Weaviate kliendi klass versioon 4.19.0"""
+    
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self.client = self._connect_to_weaviate()
+        self.class_name = "ScientificArticle"
+        self._setup_schema()
+    
+    def _connect_to_weaviate(self):
+        """Ühenda Weaviate'iga Weaviate 4.19.0 süntaksiga"""
+        try:
+            url = config.weaviate_url
+            self.logger.info(f"Ühendan Weaviate: {url}")
+            
+            # Eemalda http:// või https:// kui on
+            parsed_url = urlparse(url)
+            if not parsed_url.scheme:
+                # Kui pole protokolli, lisa http://
+                url = f"http://{url}"
+                parsed_url = urlparse(url)
+            
+            host = parsed_url.hostname
+            port = parsed_url.port or (443 if parsed_url.scheme == 'https' else 80)
+            secure = parsed_url.scheme == 'https'
+            
+            self.logger.info(f"Parsitud: host={host}, port={port}, secure={secure}")
+            
+            # Uus süntaks Weaviate 4.19.0 jaoks
+            if config.weaviate_api_key:
+                # Kui on API võti
+                auth_credentials = weaviate.auth.AuthApiKey(config.weaviate_api_key)
+                client = weaviate.WeaviateClient(
+                    connection_params=weaviate.ConnectionParams.from_params(
+                        http_host=host,
+                        http_port=port,
+                        http_secure=secure,
+                        grpc_host=host,
+                        grpc_port=50051,
+                        grpc_secure=secure,
+                        auth_credentials=auth_credentials
+                    )
+                )
+            else:
+                # Ilma autentimiseta
+                client = weaviate.WeaviateClient(
+                    connection_params=weaviate.ConnectionParams.from_params(
+                        http_host=host,
+                        http_port=port,
+                        http_secure=secure,
+                        grpc_host=host,
+                        grpc_port=50051,
+                        grpc_secure=secure
+                    )
+                )
+            
+            # Ühenda
+            client.connect()
+            
+            self.logger.info(f"Ühendatud Weaviate'iga: {host}:{port}")
+            return client
+            
+        except Exception as e:
+            self.logger.error(f"Viga Weaviate'iga ühendumisel: {str(e)}")
+            self.logger.info("Proovin alternatiivset ühendusviisi...")
+            return self._connect_fallback()
+    
+    def _connect_fallback(self):
+        """Alternatiivne ühendusviis"""
+        try:
+            url = config.weaviate_url
+            self.logger.info(f"Alternatiivne ühendus: {url}")
+            
+            # Lihtsam viis
+            if not url.startswith(("http://", "https://")):
+                url = f"http://{url}"
+            
+            # Kasutame otse weaviate.connect_to_weaviate funktsiooni
+            if config.weaviate_api_key:
+                client = weaviate.connect_to_weaviate(
+                    cluster_url=url,
+                    auth_credentials=weaviate.auth.AuthApiKey(config.weaviate_api_key)
+                )
+            else:
+                # Kui on localhost, kasuta connect_to_local
+                if "localhost" in url or "127.0.0.1" in url:
+                    # Eralda host ja port
+                    parsed = urlparse(url)
+                    host = parsed.hostname
+                    port = parsed.port or 8080
+                    client = weaviate.connect_to_local(
+                        host=host,
+                        port=port
+                    )
+                else:
+                    client = weaviate.connect_to_weaviate(
+                        cluster_url=url
+                    )
+            
+            self.logger.info(f"Alternatiivne ühendus õnnestus: {url}")
+            return client
+            
+        except Exception as e:
+            self.logger.error(f"Mõlemad ühendusviisid ebaõnnestusid: {str(e)}")
+            raise ConnectionError(f"Ei saanud Weaviate'iga ühendust: {str(e)}")
+    
+    def _setup_schema(self):
+        """Loo või kontrolli Weaviate skeemi"""
+        try:
+            # Kontrolli, kas klass on olemas
+            if self.client.collections.exists(self.class_name):
+                self.logger.info(f"Klass {self.class_name} on juba olemas")
+                return
+            
+            # Loo uus klass
+            self.client.collections.create(
+                name=self.class_name,
+                # Kasutame oma embeddinguid
+                vectorizer_config=weaviate.classes.config.Configure.Vectorizer.none(),
+                properties=[
+                    weaviate.classes.config.Property(
+                        name="article_id",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Artikli unikaalne ID"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="title",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Artikli pealkiri"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="authors",
+                        data_type=weaviate.classes.data.DataType.TEXT_ARRAY,
+                        description="Artikli autorid"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="year",
+                        data_type=weaviate.classes.data.DataType.INT,
+                        description="Avaldamisaasta"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="journal",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Žurnaal"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="doi",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="DOI identifikaator"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="abstract_en",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Inglise keelne abstrakt"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="summary_et",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Eesti keelne kokkuvõte"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="key_concepts",
+                        data_type=weaviate.classes.data.DataType.TEXT_ARRAY,
+                        description="Võtmesõnad ja mõisted"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="methods_used",
+                        data_type=weaviate.classes.data.DataType.TEXT_ARRAY,
+                        description="Kasutatud meetodid"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="transport_context",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Transpordi konteksti analüüs"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="relevance_score",
+                        data_type=weaviate.classes.data.DataType.INT,
+                        description="Relevantsus skoor 1-10"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="processing_date",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Töötlemise kuupäev"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="source_file",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Algne PDF fail"
+                    ),
+                    weaviate.classes.config.Property(
+                        name="file_hash",
+                        data_type=weaviate.classes.data.DataType.TEXT,
+                        description="Faili hash duplikaatide kontrolliks"
+                    )
+                ],
+                # Vector index seaded
+                vector_index_config=weaviate.classes.config.Configure.VectorIndex.hnsw(
+                    distance_metric=weaviate.classes.config.VectorDistances.COSINE
+                )
+            )
+            
+            self.logger.info(f"Loodi klass: {self.class_name}")
+            
+        except Exception as e:
+            self.logger.error(f"Viga skeemi seadistamisel: {str(e)}")
+            # Võib olla klass juba olemas
+            pass
+    
+    def generate_article_id(self, article_data: Dict) -> str:
+        """Genereeri artikli unikaalne ID"""
+        unique_string = ""
+        
+        if article_data.get('doi'):
+            unique_string = article_data['doi']
+        elif article_data.get('file_hash'):
+            unique_string = article_data['file_hash']
+        else:
+            title = article_data.get('title', '')
+            authors = article_data.get('authors', [])
+            unique_string = f"{title}_{'_'.join(authors[:3])}"
+        
+        # Loo MD5 hash
+        article_id = hashlib.md5(unique_string.encode()).hexdigest()
+        return article_id
+    
+    def article_exists(self, article_id: str) -> bool:
+        """Kontrolli, kas artikkel on juba baasis"""
+        try:
+            collection = self.client.collections.get(self.class_name)
+            
+            response = collection.query.fetch_objects(
+                filters=weaviate.classes.query.Filter.by_property("article_id").equal(article_id),
+                limit=1
+            )
+            
+            return len(response.objects) > 0
+            
+        except Exception as e:
+            self.logger.error(f"Viga artikli olemasolu kontrollimisel: {str(e)}")
+            return False
+    
+    def save_article(self, article_data: Dict, embeddings: Dict) -> bool:
+        """Salvesta artikkel Weaviate'i"""
+        try:
+            # Genereeri artikkel ID
+            article_id = self.generate_article_id(article_data)
+            
+            # Kontrolli duplikaati
+            if self.article_exists(article_id):
+                self.logger.info(f"Artikkel {article_id} on juba olemas, jätan vahele")
+                return False
+            
+            # Valmistame andmed ette
+            properties = {
+                "article_id": article_id,
+                "title": article_data.get('title', ''),
+                "authors": article_data.get('authors', []),
+                "year": int(article_data.get('year', 0)) if str(article_data.get('year', '0')).isdigit() else 0,
+                "journal": article_data.get('journal', ''),
+                "doi": article_data.get('doi', ''),
+                "abstract_en": article_data.get('abstract_en', ''),
+                "summary_et": article_data.get('summary_et', ''),
+                "key_concepts": article_data.get('key_concepts', []),
+                "methods_used": article_data.get('methods_used', []),
+                "transport_context": json.dumps(article_data.get('transport_context', {}), ensure_ascii=False),
+                "relevance_score": article_data.get('relevance_score', 5),
+                "processing_date": article_data.get('processing_date', ''),
+                "source_file": article_data.get('source_file', ''),
+                "file_hash": article_data.get('file_hash', '')
+            }
+            
+            # Kasutame kokkuvõtte embeddingut vektorina
+            vector = embeddings.get('summary', [])
+            
+            # Salvesta Weaviate'i
+            collection = self.client.collections.get(self.class_name)
+            
+            # Lisa objekt koos vektoriga
+            collection.data.insert(
+                properties=properties,
+                vector=vector
+            )
+            
+            self.logger.info(f"Artikkel salvestatud: {article_id}")
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"Viga artikli salvestamisel: {str(e)}")
+            return False
+    
+    def search_articles(self, query: str, limit: int = 10) -> List[Dict]:
+        """Otsi artikleid"""
+        try:
+            collection = self.client.collections.get(self.class_name)
+            
+            # Lihtne otsing
+            response = collection.query.bm25(
+                query=query,
+                query_properties=["title", "summary_et", "abstract_en", "key_concepts"],
+                limit=limit
+            )
+            
+            results = []
+            for obj in response.objects:
+                article_data = {
+                    'article_id': obj.properties.get('article_id'),
+                    'title': obj.properties.get('title'),
+                    'authors': obj.properties.get('authors', []),
+                    'year': obj.properties.get('year'),
+                    'summary': obj.properties.get('summary_et', '')[:500] + '...',
+                    'relevance_score': obj.properties.get('relevance_score'),
+                    'score': obj.metadata.score
+                }
+                results.append(article_data)
+            
+            return results
+            
+        except Exception as e:
+            self.logger.error(f"Viga otsingul: {str(e)}")
+            return []
+    
+    def close(self):
+        """Sulge Weaviate ühendus"""
+        if hasattr(self, 'client') and self.client:
+            try:
+                self.client.close()
+            except:
+                pass