import os
import sys
import re
from datetime import datetime
from reportlab.lib.pagesizes import A4
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak, Table, TableStyle
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.lib.enums import TA_JUSTIFY, TA_LEFT
from reportlab.lib import colors
from reportlab.lib.units import cm
import json
import traceback
# Lisa src kaust Pythoni teele
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
from src.weaviate_client import WeaviateClient
# ============================================================================
# STRINGI PUHASTAMISE FUNKTSIOONID
# ============================================================================
def clean_html_tags(text):
"""Puhasta tekst HTML/XML siltidest ReportLab-i jaoks"""
if not text:
return ""
# Eemalda kõik HTML/XML sildid
text = re.sub(r'<[^>]+>', '', text)
# Asenda erimärgid ReportLab-ile sobivate märkidega
replacements = {
' ': ' ',
'&': '&',
'<': '<',
'>': '>',
'"': '"',
'’': "'",
'‘': "'",
'\u00a0': ' ', # mitte-tühik
'\u2026': '...', # ellipsis
'\u2013': '-', # n-sild
'\u2014': '-', # m-sild
'\u2018': "'", # vasak ülakoma
'\u2019': "'", # parem ülakoma
'\u201c': '"', # vasak jutumärk
'\u201d': '"', # parem jutumärk
}
for old, new in replacements.items():
text = text.replace(old, new)
return text
def clean_markdown_for_pdf(text):
"""Konverteeri markdown ReportLab-ile sobivaks tekstiks"""
if not text:
return ""
# Kui ei ole string, konverteeri stringiks
if not isinstance(text, str):
text = str(text)
# Eemalda HTML sildid
text = clean_html_tags(text)
# Asenda markdown pealkirjad (# # # jne)
text = re.sub(r'#{1,6}\s+', '', text)
# Asenda bold, italic, strikethrough markeeringud
text = re.sub(r'\*\*(.+?)\*\*', r'\1', text) # **bold**
text = re.sub(r'\*(.+?)\*', r'\1', text) # *italic*
text = re.sub(r'__(.+?)__', r'\1', text) # __underline__
text = re.sub(r'~~(.+?)~~', r'\1', text) # ~~strikethrough~~
# Asenda loetelud
text = re.sub(r'^\s*[-*+]\s+', '• ', text, flags=re.MULTILINE)
text = re.sub(r'^\s*(\d+)\.\s+', r'\1. ', text, flags=re.MULTILINE)
# Eemalda koodiblokid
text = re.sub(r'```[^`]+```', '', text)
text = re.sub(r'`([^`]+)`', r'[\1]', text)
# Eemalda lingid (jäta ainult tekst)
text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)
# Eemalda liigsed tühikud
text = re.sub(r'\s+', ' ', text)
return text.strip()
def format_summary_for_pdf(summary):
"""Vorminda kokkuvõte PDF-ile sobivaks"""
if not summary:
return ""
# Kui ei ole string, konverteeri
if not isinstance(summary, str):
summary = str(summary)
# Eemalda kõik vormindus
summary = clean_markdown_for_pdf(summary)
# Lisa uued read peamiste sektsioonide ette
summary = summary.replace('1. ARTIKLI PEAMISED PUNKTID:', '\n1. ARTIKLI PEAMISED PUNKTID:\n')
summary = summary.replace('2. KASUTATUD MEETODID:', '\n\n2. KASUTATUD MEETODID:\n')
summary = summary.replace('3. PEAMISED TULEMUSED:', '\n\n3. PEAMISED TULEMUSED:\n')
summary = summary.replace('4. JÄRELDUSED JA SOOVITUSED:', '\n\n4. JÄRELDUSED JA SOOVITUSED:\n')
summary = summary.replace('5. TRANSFORDIPLANEERIMISE KONTEKST:', '\n\n5. TRANSFORDIPLANEERIMISE KONTEKST:\n')
# Asenda liigsed reavahetused
summary = re.sub(r'\n{3,}', '\n\n', summary)
# Lõika liiga pikk tekst
if len(summary) > 4000:
summary = summary[:4000] + "... [kokkuvõte lõigatud, liiga pikk]"
return summary
# ============================================================================
# TRANSPORT KONTEKSTI PARSING
# ============================================================================
def extract_json_field(json_string, field_name):
"""
Eralda JSON stringist konkreetne väli regex abil.
Args:
json_string: JSON tekst stringina
field_name: välja nimi (nt "theoretical_contribution")
Returns:
Välja väärtus või None
"""
if not json_string:
return None
pattern = rf'"{field_name}":\s*"([^"]*(?:\\"[^"]*)*)"'
match = re.search(pattern, json_string)
if match:
return match.group(1)
return None
def extract_relevance_score(json_string):
"""Eralda relevance_score JSON stringist või tekstist."""
if not json_string:
return None
pattern = r'"relevance_score":\s*(\d+(?:\.\d+)?)'
match = re.search(pattern, json_string)
if match:
return match.group(1)
return None
def format_transport_context(transport_context):
"""
Vorminda transpordi kontekst PDF-ile.
"""
if not transport_context:
return None
# PARANDUS: Leia analüüsi tekst
analysis_text = None
# Kui on dict ja sisaldab 'analysis' võtit
if isinstance(transport_context, dict):
if 'analysis' in transport_context:
analysis_text = transport_context['analysis']
else:
# Võib-olla kogu dict ON juba analysis?
analysis_text = str(transport_context)
elif isinstance(transport_context, str):
analysis_text = transport_context
else:
return None
if not analysis_text:
return None
txt = str(analysis_text)
# EEMALDA ```json ... ``` markerid
txt = re.sub(r'```json\s*', '', txt)
txt = re.sub(r'\s*```', '', txt)
txt = txt.strip()
# DEBUG: Prindi välja pärast puhastamist
#print("DEBUG after cleanup:", txt[:200])
parsed = None
# 1) Proovi parsida JSON otse
try:
parsed = json.loads(txt)
#print("✅ JSON parsed successfully!")
#print(f"DEBUG parsed keys: {list(parsed.keys())}")
except json.JSONDecodeError as e:
#print(f"❌ JSON parse failed: {e}")
# Kui ei õnnestu, proovi leida {...} blokk
m = re.search(r'\{.*\}', txt, flags=re.DOTALL)
if m:
json_candidate = m.group(0)
try:
parsed = json.loads(json_candidate)
#print("✅ JSON parsed from candidate!")
except json.JSONDecodeError as e2:
print(f"❌ Candidate parse also failed: {e2}")
parsed = None
formatted_parts = []
# Kui JSON parsimine õnnestus
if isinstance(parsed, dict):
#print("✅ Using parsed JSON dict")
# KUI parsed sisaldab 'analysis' võtit, siis see on VEEL ÜKS string!
# Peame seda UUESTI parsima!
# KUI parsed sisaldab 'analysis' võtit, siis see on VEEL ÜKS string!
if 'analysis' in parsed and isinstance(parsed['analysis'], str):
#print("⚠️ 'analysis' is still a string, parsing again...")
inner_txt = parsed['analysis']
# Eemalda ```json markerid uuesti
inner_txt = re.sub(r'```json\s*', '', inner_txt)
inner_txt = re.sub(r'\s*```', '', inner_txt)
inner_txt = inner_txt.strip()
# PARANDUS: Asenda valed escaped quotes
# \"word\" → "word" (ainult siis kui on tähtede vahel)
inner_txt = re.sub(r'\\"([a-züõäöA-ZÜÕÄÖ]+)\\"', r'"\1"', inner_txt)
# PARANDUS: Escape jutumärgid, mis on stringi väärtuste sees
# Leia kõik "key": "value" paare ja escape "value" sees olevad jutumärgid
def escape_quotes_in_values(match):
key = match.group(1)
value = match.group(2)
# Escape jutumärgid value sees
value_escaped = value.replace('"', '\\"')
return f'"{key}": "{value_escaped}"'
inner_txt = re.sub(r'"(\w+)":\s*"([^"]*(?:"[^"]*)*)"', escape_quotes_in_values, inner_txt)
try:
parsed = json.loads(inner_txt)
#print("✅ Inner JSON parsed successfully!")
#print(f"DEBUG inner parsed keys: {list(parsed.keys())}")
except json.JSONDecodeError as e:
#print(f"❌ Inner JSON parse failed: {e}")
# AGRESSIIVNE PARANDUS: kasuta regex fallback'i
#print("⚠️ Falling back to regex extraction...")
# Taasta originaal inner_txt (ilma escapimiseta)
inner_txt = parsed['analysis']
inner_txt = re.sub(r'```json\s*', '', inner_txt)
inner_txt = re.sub(r'\s*```', '', inner_txt)
inner_txt = inner_txt.strip()
# Kasuta regex'i otse inner_txt pealt
temp_parts = []
match = re.search(r'"theoretical_contribution":\s*"(.*?)"(?=\s*,\s*")', inner_txt, flags=re.DOTALL)
if match:
temp_parts.append("TEOREETILINE PANUS:\n" + match.group(1))
match = re.search(r'"practical_applicability":\s*"(.*?)"(?=\s*,\s*")', inner_txt, flags=re.DOTALL)
if match:
temp_parts.append("PRAKTILINE RAKENDATAVUS:\n" + match.group(1))
match = re.search(r'"problem_solving":\s*"(.*?)"(?=\s*,\s*")', inner_txt, flags=re.DOTALL)
if match:
temp_parts.append("PROBLEEMILAHENDUS:\n" + match.group(1))
match = re.search(r'"limitations":\s*"(.*?)"(?=\s*,\s*")', inner_txt, flags=re.DOTALL)
if match:
temp_parts.append("PIIRANGUD:\n" + match.group(1))
match = re.search(r'"relevance_score":\s*(\d+)', inner_txt)
if match:
temp_parts.append(f"RELEVANTSUSE SKOOR: {match.group(1)}/10")
if temp_parts:
#print(f"✅ Regex extracted {len(temp_parts)} parts")
return "\n\n".join(temp_parts)
# Nüüd kasuta parsed dict'i
if parsed.get("theoretical_contribution"):
formatted_parts.append(
"TEOREETILINE PANUS:\n" + str(parsed["theoretical_contribution"])
)
if parsed.get("practical_applicability"):
formatted_parts.append(
"PRAKTILINE RAKENDATAVUS:\n" + str(parsed["practical_applicability"])
)
if parsed.get("problem_solving"):
formatted_parts.append(
"PROBLEEMILAHENDUS:\n" + str(parsed["problem_solving"])
)
if parsed.get("limitations"):
formatted_parts.append(
"PIIRANGUD:\n" + str(parsed["limitations"])
)
if "relevance_score" in parsed:
formatted_parts.append(
f"RELEVANTSUSE SKOOR: {parsed['relevance_score']}/10"
)
return "\n\n".join(formatted_parts) if formatted_parts else None
# Kui JSON ei õnnestunud → kasuta regex-i
print("⚠️ Using regex fallback")
# Regex peab nüüd käsitlema newline't – kasuta re.DOTALL
match = re.search(r'"theoretical_contribution":\s*"(.*?)"(?=\s*,|\s*})', txt, flags=re.DOTALL)
if match:
content = match.group(1).replace('\\n', '\n').replace('\\"', '"')
if content:
formatted_parts.append(f"TEOREETILINE PANUS:\n{content}")
match = re.search(r'"practical_applicability":\s*"(.*?)"(?=\s*,|\s*})', txt, flags=re.DOTALL)
if match:
content = match.group(1).replace('\\n', '\n').replace('\\"', '"')
if content:
formatted_parts.append(f"PRAKTILINE RAKENDATAVUS:\n{content}")
match = re.search(r'"problem_solving":\s*"(.*?)"(?=\s*,|\s*})', txt, flags=re.DOTALL)
if match:
content = match.group(1).replace('\\n', '\n').replace('\\"', '"')
if content:
formatted_parts.append(f"PROBLEEMILAHENDUS:\n{content}")
match = re.search(r'"limitations":\s*"(.*?)"(?=\s*,|\s*})', txt, flags=re.DOTALL)
if match:
content = match.group(1).replace('\\n', '\n').replace('\\"', '"')
if content:
formatted_parts.append(f"PIIRANGUD:\n{content}")
match = re.search(r'"relevance_score":\s*(\d+(?:\.\d+)?)', txt)
if match:
score = match.group(1)
formatted_parts.append(f"RELEVANTSUSE SKOOR: {score}/10")
return "\n\n".join(formatted_parts) if formatted_parts else None
# ============================================================================
# WEAVIATE ANDMEBAASIST PÄRING
# ============================================================================
def get_all_articles_from_weaviate():
"""Toob kõik artiklid Weaviate andmebaasist"""
client = WeaviateClient()
articles = []
try:
collection = client.client.collections.get("ScientificArticle")
# Loendi kokku
count_response = collection.aggregate.over_all(total_count=True)
total = count_response.total_count
print(f"Weaviate'is leidsin {total} artiklit")
if total > 0:
# Toob kõik artiklid
response = collection.query.fetch_objects(limit=total)
for obj in response.objects:
try:
print(obj.properties.get('abstract_en', ''))
article = {
'article_id': obj.properties.get('article_id', 'N/A'),
'title': clean_markdown_for_pdf(obj.properties.get('title', 'N/A')),
'authors': obj.properties.get('authors', []),
'year': obj.properties.get('year', 'N/A'),
'journal': clean_markdown_for_pdf(obj.properties.get('journal', 'N/A')),
'doi': obj.properties.get('doi', ''),
'abstract_en': clean_markdown_for_pdf(obj.properties.get('abstract_en', '')),
'summary_et': clean_markdown_for_pdf(obj.properties.get('summary_et', '')),
'key_concepts': [clean_markdown_for_pdf(c) for c in obj.properties.get('key_concepts', [])],
'methods_used': [clean_markdown_for_pdf(m) for m in obj.properties.get('methods_used', [])],
'transport_context': obj.properties.get('transport_context', {}),
'relevance_score': obj.properties.get('relevance_score', 'N/A'),
'processing_date': obj.properties.get('processing_date', ''),
'source_file': obj.properties.get('source_file', '')
}
articles.append(article)
except Exception as e:
print(f"⚠️ Viga artikli {obj.properties.get('article_id', 'unknown')} töötlemisel: {e}")
# Jätka järgmise artikliga
continue
except Exception as e:
print(f"❌ Viga artiklite toomisel: {e}")
traceback.print_exc()
finally:
client.close()
return articles
# ============================================================================
# PDF GENEREERIMINE
# ============================================================================
def create_pdf_from_articles(articles, output_filename):
"""Loob PDF faili artiklitest"""
# Loo PDF dokument
doc = SimpleDocTemplate(
output_filename,
pagesize=A4,
rightMargin=72,
leftMargin=72,
topMargin=72,
bottomMargin=72
)
# Stiilide loomine
styles = getSampleStyleSheet()
# Kohandatud stiilid
title_style = ParagraphStyle(
'CustomTitle',
parent=styles['Heading1'],
fontSize=14,
spaceAfter=12,
textColor=colors.HexColor('#2c3e50'),
alignment=TA_LEFT
)
subtitle_style = ParagraphStyle(
'CustomSubtitle',
parent=styles['Heading2'],
fontSize=12,
spaceAfter=6,
textColor=colors.HexColor('#34495e'),
alignment=TA_LEFT
)
section_style = ParagraphStyle(
'CustomSection',
parent=styles['Heading3'],
fontSize=11,
spaceAfter=6,
spaceBefore=12,
textColor=colors.HexColor('#7f8c8d'),
alignment=TA_LEFT
)
normal_style = ParagraphStyle(
'CustomNormal',
parent=styles['Normal'],
fontSize=10,
spaceAfter=6,
alignment=TA_JUSTIFY,
leading=14
)
metadata_style = ParagraphStyle(
'CustomMetadata',
parent=styles['Normal'],
fontSize=9,
spaceAfter=3,
textColor=colors.HexColor('#5d6d7e'),
alignment=TA_LEFT
)
# Elementide kogumine
elements = []
# Pealkiri ja kokkuvõte
elements.append(Paragraph("TEADUSARTIKLITE ANDMEBAAS", title_style))
elements.append(Spacer(1, 12))
today = datetime.now().strftime("%d.%m.%Y %H:%M")
elements.append(Paragraph(f"Eksporditud: {today}", metadata_style))
elements.append(Paragraph(f"Artikleid kokku: {len(articles)}", metadata_style))
elements.append(Spacer(1, 24))
# ========================================================================
# Iga artikli jaoks
# ========================================================================
for i, article in enumerate(articles):
# ARTIKLI PEALKIRI
elements.append(Paragraph(f"{i+1}. {article['title']}", title_style))
print(f"✅ {i+1}. {article['title']}")
# AUTORID
if article['authors']:
authors_text = ", ".join(article['authors'])
elements.append(Paragraph(f"Autorid: {authors_text}", subtitle_style))
# METAANDMED (aasta, žurnaal, DOI, relevantsus)
metadata_data = []
if article['year'] and article['year'] != 'N/A':
metadata_data.append(['Aasta:', str(article['year'])])
if article['journal'] and article['journal'] != 'N/A':
metadata_data.append(['Žurnaal:', article['journal']])
if article['doi']:
metadata_data.append(['DOI:', article['doi']])
if article['relevance_score'] and article['relevance_score'] != 'N/A':
metadata_data.append(['Relevantsus:', f"{article['relevance_score']}/10"])
if metadata_data:
metadata_table = Table(metadata_data, colWidths=[2*cm, 12*cm])
metadata_table.setStyle(TableStyle([
('FONTNAME', (0, 0), (-1, -1), 'Helvetica'),
('FONTSIZE', (0, 0), (-1, -1), 9),
('BOTTOMPADDING', (0, 0), (-1, -1), 6),
('TOPPADDING', (0, 0), (-1, -1), 6),
('VALIGN', (0, 0), (-1, -1), 'TOP'),
('LEFTPADDING', (0, 0), (0, -1), 0),
]))
elements.append(metadata_table)
elements.append(Spacer(1, 12))
# VÕTMESÕNAD JA MEETODID
if article['key_concepts'] or article['methods_used']:
# Luua väike pealkiri stiil
small_label_style = ParagraphStyle(
'SmallLabel',
parent=styles['Normal'],
fontSize=8,
textColor=colors.HexColor('#7f8c8d'),
spaceAfter=2,
)
content_style = ParagraphStyle(
'TagContent',
parent=styles['Normal'],
fontSize=9,
spaceAfter=4,
)
if article['key_concepts']:
concepts_text = ", ".join(article['key_concepts'][:10])
elements.append(Paragraph("Võtmesõnad: " + concepts_text, content_style))
if article['methods_used']:
methods_text = ", ".join(article['methods_used'][:8])
elements.append(Paragraph("Meetodid: " + methods_text, content_style))
elements.append(Spacer(1, 6))
# ABSTRAKT (inglise keeles)
if article['abstract_en']:
elements.append(Paragraph("ABSTRAKT (inglise keeles):", section_style))
abstract_text = article['abstract_en']
if len(abstract_text) > 800:
abstract_text = abstract_text[:800] + "..."
elements.append(Paragraph(abstract_text, normal_style))
elements.append(Spacer(1, 12))
# KOKKUVÕTE (eesti keeles)
if article['summary_et']:
elements.append(Paragraph("KOKKUVÕTE (eesti keeles):", section_style))
summary = format_summary_for_pdf(article['summary_et'])
elements.append(Paragraph(summary, normal_style))
elements.append(Spacer(1, 12))
# ====================================================================
# TRANSPORDI PLANEERIMISE KONTEKST
# ====================================================================
if article['transport_context']:
elements.append(Paragraph("TRANSFORDIPLANEERIMISE KONTEKST:", section_style))
# DEBUG-REA – prindi üks-ühele objekt konsooli
#print("DEBUG transport_context:", article['article_id'], article['transport_context'])
context_text = format_transport_context(article['transport_context'])
#print(f"DEBUG context_text returned: {context_text}") # <-- LISA SEE RIDA
if context_text:
# Jaga osadeks ja lisa eraldi paragrahfidena
parts = context_text.split('\n\n') # Jaga tühjast reaga
for part in parts:
if part.strip():
# Asenda \n
tag'iga
part_html = part.replace('\n', '
')
try:
elements.append(Paragraph(part_html, normal_style))
elements.append(Spacer(1, 6))
except Exception as e:
print(f"❌ Failed to add part to PDF: {e}")
# Kui HTML tag ei tööta, proovi ilma
part_plain = part.replace('\n', ' ')
elements.append(Paragraph(part_plain, normal_style))
elements.append(Spacer(1, 6))
print("✅ Context added to PDF successfully")
else:
elements.append(Paragraph("Analüüsi andmed puuduvad", normal_style))
elements.append(Spacer(1, 12))
# ====================================================================
# FOOTER INFO (allikfail, töötlemise kuupäev)
# ====================================================================
footer_info = []
if article['source_file']:
source_name = os.path.basename(article['source_file'])
footer_info.append(f"Allikfail: {source_name}")
if article['processing_date']:
try:
date_str = article['processing_date']
if '.' in date_str:
date_str = date_str.split('.')[0]
date_str = date_str.replace('Z', '+00:00')
date_obj = datetime.fromisoformat(date_str)
footer_info.append(f"Töödeldud: {date_obj.strftime('%d.%m.%Y %H:%M')}")
except Exception:
footer_info.append(f"Töödeldud: {article['processing_date'][:19]}")
if footer_info:
elements.append(Spacer(1, 6))
elements.append(Paragraph(" | ".join(footer_info), metadata_style))
# Lisa lehevahetus (välja arvatud viimase artikli puhul)
if i < len(articles) - 1:
elements.append(PageBreak())
else:
elements.append(Spacer(1, 24))
# LÕPPINFO
elements.append(Paragraph("=" * 80, metadata_style))
elements.append(Spacer(1, 6))
elements.append(Paragraph(f"Kokku eksporditud artikleid: {len(articles)}", metadata_style))
elements.append(Paragraph("Eksporditud Weaviate teadusartiklite andmebaasist", metadata_style))
elements.append(Paragraph(f"PDF genereeritud: {datetime.now().strftime('%d.%m.%Y %H:%M:%S')}", metadata_style))
# Koosta PDF
doc.build(elements)
return len(articles)
# ============================================================================
# PEAMINE FUNKTSIOON
# ============================================================================
def main():
"""Peamine funktsioon - käivitab kogu protsessi"""
print("=" * 60)
print("ARTIKLITE EKSPORT PDF FAILI")
print("=" * 60)
# Toob artiklid Weaviate'ist
print("Toon artikleid Weaviate'ist...")
articles = get_all_articles_from_weaviate()
if not articles:
print("❌ Ei leidnud ühtegi artiklit Weaviate'is!")
return
print(f"✓ Leidsin {len(articles)} artiklit")
# Genereeri PDF failinimi
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_dir = "./data/exports"
os.makedirs(output_dir, exist_ok=True)
output_filename = os.path.join(output_dir, f"artiklid_eksport_{timestamp}.pdf")
# Loo PDF
print(f"Loon PDF faili: {output_filename}")
try:
article_count = create_pdf_from_articles(articles, output_filename)
print("=" * 60)
print(f"✅ VALMIS! Loodud PDF fail: {output_filename}")
print(f" - Eksporditud artikleid: {article_count}")
print(f" - Faili suurus: {os.path.getsize(output_filename) / 1024:.1f} KB")
print("=" * 60)
if articles:
print("\nEsimesed artiklid:")
for i, article in enumerate(articles[:3]):
title_preview = article['title']
if len(title_preview) > 60:
title_preview = title_preview[:60] + "..."
print(f" {i+1}. {title_preview}")
except Exception as e:
print(f"\n❌ VIGA PDF loomisel: {e}")
traceback.print_exc()
if __name__ == "__main__":
main()