Philipp MockandClaude Opus 5.5
d54b3e5b8b
- prepare_corpus.py: split double-page scans (split_spreads.py), re-OCR with OCRmyPDF/Tesseract only where the existing text layer is worse, Docling layout extraction with heading hierarchy, QA report per PDF - docling_loader.py: cleanup (margin noise, split paragraphs, footnote marks, OCR comma/period repair), page mapping back to the original PDF, chunk headers with author, title and section - local_rag.py: bge-m3 embeddings, configurable chunking and loader, index settings check (index_info.json), hybrid/union BM25 search with Reciprocal Rank Fusion and bge-reranker-v2-m3 cross-encoder; query instruction text removed after it lowered recall - eval_retrieval.py: CLI options, same retrieval path as the chat, tolerant file name matching; goldsets and evaluation reports for the Kafka corpus - grobid_test.py, grobid_qa.py, extract_docling.py, compare_extractors.py: extractor tests and quality checks - .gitignore: corpus PDFs, vector stores and extractor output stay local - server.py: uses ./vectorstore_docling_hdr Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
4.9 KiB
Python
110 lines
4.9 KiB
Python
"""
|
|
Vergleicht die Extraktoren mit denselben Prüfungen aus grobid_qa.py.
|
|
|
|
Für jeden Titel aus extract_docling.TITLES werden bewertet:
|
|
pypdf der jetzige PyPDFLoader (eine Einheit pro Seite, keine Struktur)
|
|
grobid grobid_out/data/<pdf>.tei.xml
|
|
docling extract_out/docling/<pdf>.json
|
|
docling_ocr extract_out/docling_ocr/<pdf>.json
|
|
docling_tess extract_out/docling_tess/<pdf>.json
|
|
docling_ocrmypdf extract_out/docling_ocrmypdf/<pdf>.json (Referenz: extract_out/ocrmypdf/<pdf>)
|
|
Fehlende Ausgaben werden ausgelassen.
|
|
|
|
Aufruf (Projektumgebung genügt, Docling wird hier nicht gebraucht):
|
|
conda activate local_rag
|
|
python compare_extractors.py
|
|
|
|
Ausgabe: Vergleichstabelle im Terminal und in extract_out/vergleich/vergleich.md
|
|
(+ .csv), Einzelberichte mit Belegstellen in extract_out/vergleich/<extraktor>/.
|
|
"""
|
|
|
|
import csv
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import grobid_qa as qa
|
|
|
|
PDF_DIR = Path("data")
|
|
OUT_DIR = Path("extract_out") / "vergleich"
|
|
|
|
# Titel wie in extract_docling.py, ohne Docling importieren zu müssen
|
|
TITLES = re.findall(r'^\s*"([^"]+)",', Path("extract_docling.py").read_text(encoding="utf-8"), re.M)
|
|
|
|
EXTRACTORS = {
|
|
"pypdf": lambda stem: None,
|
|
"grobid": lambda stem: Path("grobid_out/data") / f"{stem}.tei.xml",
|
|
"docling": lambda stem: Path("extract_out/docling") / f"{stem}.json",
|
|
"docling_ocr": lambda stem: Path("extract_out/docling_ocr") / f"{stem}.json",
|
|
"docling_tess": lambda stem: Path("extract_out/docling_tess") / f"{stem}.json",
|
|
"docling_ocrmypdf": lambda stem: Path("extract_out/docling_ocrmypdf") / f"{stem}.json",
|
|
"ocrmypdf_bereinigt": lambda stem: Path("extract_out/docling_ocrmypdf") / f"{stem}.json",
|
|
}
|
|
# Diese Varianten werden so geprüft, wie docling_loader.py sie bereinigt an den Index gibt.
|
|
CLEANED = {"ocrmypdf_bereinigt"}
|
|
# Für diese Varianten ist das neu erkannte PDF die Referenz: geprüft wird, ob Docling
|
|
# dessen Textschicht vollständig übernimmt, und wie gut die neue OCR ist.
|
|
REFERENCE_PDF = {"docling_ocrmypdf": Path("extract_out/ocrmypdf"),
|
|
"ocrmypdf_bereinigt": Path("extract_out/ocrmypdf")}
|
|
|
|
# Spalten der Vergleichstabelle: Kennzahl -> Kurzname
|
|
COLUMNS = {
|
|
"absaetze": "Absätze", "ueberschriften": "Überschr.", "fussnoten": "Fußnoten",
|
|
"giant_para_pct": "Riesen%", "lost_pct": "Verlust%", "misfiled_pct": "Fehlabl%",
|
|
"contam_per_100k": "Fremd", "fnmark_per_10k": "FN-Zch", "ocr_per_10k": "OCR",
|
|
"hyphen_per_10k": "Trenn", "broken_per_100p": "Zerriss", "order": "Reihenf",
|
|
}
|
|
|
|
|
|
def mark(m: dict, key: str) -> str:
|
|
value = m.get(key)
|
|
light = m.get("ampel", {}).get(key)
|
|
suffix = {"ROT": "!", "GELB": "?"}.get(light, "")
|
|
return f"{value}{suffix}"
|
|
|
|
|
|
def main():
|
|
pdfs = [p for p in sorted(PDF_DIR.glob("*.pdf")) if any(p.name.startswith(t) for t in TITLES)]
|
|
rows = []
|
|
for pdf in pdfs:
|
|
for name, locate in EXTRACTORS.items():
|
|
out = locate(pdf.stem)
|
|
if out is not None and not out.exists():
|
|
continue
|
|
ref = REFERENCE_PDF[name] / pdf.name if name in REFERENCE_PDF else pdf
|
|
m, ex = qa.check(ref, out, cleaned=name in CLEANED)
|
|
report_dir = OUT_DIR / name
|
|
report_dir.mkdir(parents=True, exist_ok=True)
|
|
qa.write_report(m, ex, report_dir / f"{pdf.stem}.md")
|
|
rows.append({**m, "titel": pdf.name.split("_")[0].rstrip(","), "datei": pdf.name,
|
|
"extraktor": name})
|
|
print(f"{pdf.name[:50]:50} {name:12} Extraktor {m['grobid']}", flush=True)
|
|
|
|
header = ["Titel", "Extraktor", "Urteil", *COLUMNS.values()]
|
|
lines = ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)]
|
|
for r in rows:
|
|
cells = [r["titel"], r["extraktor"], r["grobid"], *(mark(r, k) for k in COLUMNS)]
|
|
lines.append("| " + " | ".join(map(str, cells)) + " |")
|
|
legend = ("\n? = gelb, ! = rot. Absätze/Überschriften/Fußnoten: Anzahl erkannter Elemente. "
|
|
"Bei pypdf ist jede Seite eine Einheit, Verlust und Reihenfolge sind dort trivial.")
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
(OUT_DIR / "vergleich.md").write_text("# Extraktoren im Vergleich\n\n" + "\n".join(lines) + legend + "\n",
|
|
encoding="utf-8")
|
|
with open(OUT_DIR / "vergleich.csv", "w", newline="", encoding="utf-8") as f:
|
|
w = csv.DictWriter(f, fieldnames=["datei", "extraktor", "grobid", "pdf", *COLUMNS],
|
|
extrasaction="ignore", delimiter=";")
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
|
|
print()
|
|
widths = [12, 16, 6] + [9] * len(COLUMNS)
|
|
print(" ".join(h[:w].ljust(w) for h, w in zip(header, widths)))
|
|
for r in rows:
|
|
cells = [r["titel"], r["extraktor"], r["grobid"], *(mark(r, k) for k in COLUMNS)]
|
|
print(" ".join(str(c)[:w].ljust(w) for c, w in zip(cells, widths)))
|
|
print(legend.strip())
|
|
print("Details:", OUT_DIR)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|