Add Docling corpus pipeline, retrieval evaluation and hybrid search with reranking
- prepare_corpus.py: split double-page scans (split_spreads.py), re-OCR with OCRmyPDF/Tesseract only where the existing text layer is worse, Docling layout extraction with heading hierarchy, QA report per PDF - docling_loader.py: cleanup (margin noise, split paragraphs, footnote marks, OCR comma/period repair), page mapping back to the original PDF, chunk headers with author, title and section - local_rag.py: bge-m3 embeddings, configurable chunking and loader, index settings check (index_info.json), hybrid/union BM25 search with Reciprocal Rank Fusion and bge-reranker-v2-m3 cross-encoder; query instruction text removed after it lowered recall - eval_retrieval.py: CLI options, same retrieval path as the chat, tolerant file name matching; goldsets and evaluation reports for the Kafka corpus - grobid_test.py, grobid_qa.py, extract_docling.py, compare_extractors.py: extractor tests and quality checks - .gitignore: corpus PDFs, vector stores and extractor output stay local - server.py: uses ./vectorstore_docling_hdr Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Vergleicht die Extraktoren mit denselben Prüfungen aus grobid_qa.py.
|
||||
|
||||
Für jeden Titel aus extract_docling.TITLES werden bewertet:
|
||||
pypdf der jetzige PyPDFLoader (eine Einheit pro Seite, keine Struktur)
|
||||
grobid grobid_out/data/<pdf>.tei.xml
|
||||
docling extract_out/docling/<pdf>.json
|
||||
docling_ocr extract_out/docling_ocr/<pdf>.json
|
||||
docling_tess extract_out/docling_tess/<pdf>.json
|
||||
docling_ocrmypdf extract_out/docling_ocrmypdf/<pdf>.json (Referenz: extract_out/ocrmypdf/<pdf>)
|
||||
Fehlende Ausgaben werden ausgelassen.
|
||||
|
||||
Aufruf (Projektumgebung genügt, Docling wird hier nicht gebraucht):
|
||||
conda activate local_rag
|
||||
python compare_extractors.py
|
||||
|
||||
Ausgabe: Vergleichstabelle im Terminal und in extract_out/vergleich/vergleich.md
|
||||
(+ .csv), Einzelberichte mit Belegstellen in extract_out/vergleich/<extraktor>/.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import grobid_qa as qa
|
||||
|
||||
PDF_DIR = Path("data")
|
||||
OUT_DIR = Path("extract_out") / "vergleich"
|
||||
|
||||
# Titel wie in extract_docling.py, ohne Docling importieren zu müssen
|
||||
TITLES = re.findall(r'^\s*"([^"]+)",', Path("extract_docling.py").read_text(encoding="utf-8"), re.M)
|
||||
|
||||
EXTRACTORS = {
|
||||
"pypdf": lambda stem: None,
|
||||
"grobid": lambda stem: Path("grobid_out/data") / f"{stem}.tei.xml",
|
||||
"docling": lambda stem: Path("extract_out/docling") / f"{stem}.json",
|
||||
"docling_ocr": lambda stem: Path("extract_out/docling_ocr") / f"{stem}.json",
|
||||
"docling_tess": lambda stem: Path("extract_out/docling_tess") / f"{stem}.json",
|
||||
"docling_ocrmypdf": lambda stem: Path("extract_out/docling_ocrmypdf") / f"{stem}.json",
|
||||
"ocrmypdf_bereinigt": lambda stem: Path("extract_out/docling_ocrmypdf") / f"{stem}.json",
|
||||
}
|
||||
# Diese Varianten werden so geprüft, wie docling_loader.py sie bereinigt an den Index gibt.
|
||||
CLEANED = {"ocrmypdf_bereinigt"}
|
||||
# Für diese Varianten ist das neu erkannte PDF die Referenz: geprüft wird, ob Docling
|
||||
# dessen Textschicht vollständig übernimmt, und wie gut die neue OCR ist.
|
||||
REFERENCE_PDF = {"docling_ocrmypdf": Path("extract_out/ocrmypdf"),
|
||||
"ocrmypdf_bereinigt": Path("extract_out/ocrmypdf")}
|
||||
|
||||
# Spalten der Vergleichstabelle: Kennzahl -> Kurzname
|
||||
COLUMNS = {
|
||||
"absaetze": "Absätze", "ueberschriften": "Überschr.", "fussnoten": "Fußnoten",
|
||||
"giant_para_pct": "Riesen%", "lost_pct": "Verlust%", "misfiled_pct": "Fehlabl%",
|
||||
"contam_per_100k": "Fremd", "fnmark_per_10k": "FN-Zch", "ocr_per_10k": "OCR",
|
||||
"hyphen_per_10k": "Trenn", "broken_per_100p": "Zerriss", "order": "Reihenf",
|
||||
}
|
||||
|
||||
|
||||
def mark(m: dict, key: str) -> str:
|
||||
value = m.get(key)
|
||||
light = m.get("ampel", {}).get(key)
|
||||
suffix = {"ROT": "!", "GELB": "?"}.get(light, "")
|
||||
return f"{value}{suffix}"
|
||||
|
||||
|
||||
def main():
|
||||
pdfs = [p for p in sorted(PDF_DIR.glob("*.pdf")) if any(p.name.startswith(t) for t in TITLES)]
|
||||
rows = []
|
||||
for pdf in pdfs:
|
||||
for name, locate in EXTRACTORS.items():
|
||||
out = locate(pdf.stem)
|
||||
if out is not None and not out.exists():
|
||||
continue
|
||||
ref = REFERENCE_PDF[name] / pdf.name if name in REFERENCE_PDF else pdf
|
||||
m, ex = qa.check(ref, out, cleaned=name in CLEANED)
|
||||
report_dir = OUT_DIR / name
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
qa.write_report(m, ex, report_dir / f"{pdf.stem}.md")
|
||||
rows.append({**m, "titel": pdf.name.split("_")[0].rstrip(","), "datei": pdf.name,
|
||||
"extraktor": name})
|
||||
print(f"{pdf.name[:50]:50} {name:12} Extraktor {m['grobid']}", flush=True)
|
||||
|
||||
header = ["Titel", "Extraktor", "Urteil", *COLUMNS.values()]
|
||||
lines = ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)]
|
||||
for r in rows:
|
||||
cells = [r["titel"], r["extraktor"], r["grobid"], *(mark(r, k) for k in COLUMNS)]
|
||||
lines.append("| " + " | ".join(map(str, cells)) + " |")
|
||||
legend = ("\n? = gelb, ! = rot. Absätze/Überschriften/Fußnoten: Anzahl erkannter Elemente. "
|
||||
"Bei pypdf ist jede Seite eine Einheit, Verlust und Reihenfolge sind dort trivial.")
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(OUT_DIR / "vergleich.md").write_text("# Extraktoren im Vergleich\n\n" + "\n".join(lines) + legend + "\n",
|
||||
encoding="utf-8")
|
||||
with open(OUT_DIR / "vergleich.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["datei", "extraktor", "grobid", "pdf", *COLUMNS],
|
||||
extrasaction="ignore", delimiter=";")
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
|
||||
print()
|
||||
widths = [12, 16, 6] + [9] * len(COLUMNS)
|
||||
print(" ".join(h[:w].ljust(w) for h, w in zip(header, widths)))
|
||||
for r in rows:
|
||||
cells = [r["titel"], r["extraktor"], r["grobid"], *(mark(r, k) for k in COLUMNS)]
|
||||
print(" ".join(str(c)[:w].ljust(w) for c, w in zip(cells, widths)))
|
||||
print(legend.strip())
|
||||
print("Details:", OUT_DIR)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user