Add Docling corpus pipeline, retrieval evaluation and hybrid search with reranking
- prepare_corpus.py: split double-page scans (split_spreads.py), re-OCR with OCRmyPDF/Tesseract only where the existing text layer is worse, Docling layout extraction with heading hierarchy, QA report per PDF - docling_loader.py: cleanup (margin noise, split paragraphs, footnote marks, OCR comma/period repair), page mapping back to the original PDF, chunk headers with author, title and section - local_rag.py: bge-m3 embeddings, configurable chunking and loader, index settings check (index_info.json), hybrid/union BM25 search with Reciprocal Rank Fusion and bge-reranker-v2-m3 cross-encoder; query instruction text removed after it lowered recall - eval_retrieval.py: CLI options, same retrieval path as the chat, tolerant file name matching; goldsets and evaluation reports for the Kafka corpus - grobid_test.py, grobid_qa.py, extract_docling.py, compare_extractors.py: extractor tests and quality checks - .gitignore: corpus PDFs, vector stores and extractor output stay local - server.py: uses ./vectorstore_docling_hdr Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+777
@@ -0,0 +1,777 @@
|
||||
"""
|
||||
Qualitätsprüfung der GROBID-Ausgabe, ohne Goldstandard.
|
||||
|
||||
Vergleicht für jedes PDF den GROBID-Output (TEI-XML) mit dem Rohtext, den pypdf
|
||||
aus derselben Datei liest, und sucht gezielt nach den Fehlern, die für RAG
|
||||
schaden:
|
||||
|
||||
- Textverlust: Passagen, die im PDF stehen, im TEI aber nirgends auftauchen.
|
||||
Verlorene Passagen werden sortiert: Vermerke/Metadaten, Tabellen,
|
||||
Literaturangaben und pypdf-Lesefehler sind unkritisch und zählen
|
||||
nicht in die Ampel — nur verlorener Fließtext.
|
||||
- Verunreinigung: Lizenz-/Download-Vermerke, Kolumnentitel, Literaturangaben und
|
||||
Fragmente im Fließtext
|
||||
- Doppelte Absätze
|
||||
- Fehlklassierung: Fließtext, der als Abbildung/Tabelle/Notiz einsortiert wurde
|
||||
- Absatzerkennung: Fließtext in Riesenabsätzen (ganze Seiten oder Kapitel am Stück)
|
||||
- Zerrissene Sätze, Trennstrich-Reste, Kodierungsfehler, OCR-Fehler,
|
||||
Fußnotenzeichen im Text (am Wort klebend oder abgesetzt)
|
||||
- Lesereihenfolge: stimmt die Absatzfolge mit der Reihenfolge im PDF überein?
|
||||
- Fehlender Textlayer (Scans ohne OCR — dann kann GROBID nichts extrahieren)
|
||||
|
||||
Die Ampel-Schwellen unten sind Startwerte, kalibriert an drei englischsprachigen
|
||||
Zeitschriftenartikeln und dem Kafka-Korpus (deutsche Monografien, OCR-Scans).
|
||||
|
||||
Wo pypdf selbst keinen brauchbaren Text liefert (Sperrsatz, verschluckte
|
||||
Leerzeichen), werden die Seiten aus dem Abgleich ausgenommen; ist das bei zu
|
||||
vielen Seiten der Fall, bleibt die Verlustmessung leer statt falsch.
|
||||
|
||||
Aufruf (in der Projektumgebung, weil pypdf gebraucht wird):
|
||||
conda activate local_rag
|
||||
python grobid_qa.py # PDFs aus data_test/
|
||||
python grobid_qa.py data # PDFs aus einem anderen Ordner
|
||||
|
||||
Ausgabe: Übersicht im Terminal, grobid_out/<ordner>/qa/summary.csv (eine Zeile
|
||||
pro PDF, für Vergleiche über die Zeit) und grobid_out/<ordner>/qa/<pdf>.md mit
|
||||
Belegstellen. Bereits vorhandenes TEI wird wiederverwendet.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import unicodedata
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from pypdf import PdfReader
|
||||
|
||||
from grobid_test import GROBID_URL, process_pdf
|
||||
|
||||
# ---------- Einstellungen ----------
|
||||
|
||||
PDF_DIR = Path("data_test") # oder als Argument: python grobid_qa.py data
|
||||
OUT_ROOT = Path("grobid_out") # TEI landet in grobid_out/<pdf-ordner>/, Berichte in .../qa/
|
||||
REPROCESS = False # True: TEI immer neu von GROBID holen, auch wenn vorhanden
|
||||
MAX_EXAMPLES = 8 # Belegstellen pro Kategorie im Bericht
|
||||
SAMPLE_PARAGRAPHS = 5 # zufällige Fließtextabsätze zum Gegenlesen
|
||||
SHINGLE = 5 # Wortfolgen dieser Länge dienen zum Abgleich PDF <-> TEI
|
||||
MIN_LOST_RUN = 20 # ab so vielen Wörtern am Stück gilt eine Passage als verloren
|
||||
GIANT_PARA = 3000 # Absätze über so vielen Zeichen gelten als nicht segmentiert
|
||||
MAX_UNUSABLE_REF = 30 # ist pypdf auf mehr als so viel % der Seiten unlesbar, keine Verlustmessung
|
||||
|
||||
# Ampel: (gelb ab, rot ab). Bei "order" und "body_share" ist kleiner schlechter.
|
||||
THRESHOLDS = {
|
||||
"scan_pages_pct": (1, 10), # Seiten ohne nennenswerten Textlayer, %
|
||||
"lost_pct": (2, 5), # Fließtext-Wörter des PDFs, die im TEI fehlen, %
|
||||
"contam_per_100k": (2, 8), # Fremdtext-Funde je 100.000 Zeichen Fließtext
|
||||
"duplicate_pct": (0.5, 2), # doppelt vorkommende Fließtextabsätze, %
|
||||
"misfiled_pct": (1, 5), # Fließtext, der in Abbildungen/Notizen steckt, %
|
||||
"giant_para_pct": (10, 30), # Fließtext in Riesenabsätzen (> GIANT_PARA Zeichen), %
|
||||
"broken_per_100p": (2, 6), # zerrissene Sätze je 100 Absätze
|
||||
"hyphen_per_10k": (2, 10), # Trennstrich-Reste je 10.000 Wörter
|
||||
"encoding_per_10k": (0.2, 3), # Kodierungsfehler je 10.000 Zeichen
|
||||
"fnmark_per_10k": (2, 10), # Fußnotenzeichen im Fließtext je 10.000 Wörter
|
||||
"ocr_per_10k": (5, 20), # OCR-Fehlerbilder je 10.000 Wörter
|
||||
"order": (0.97, 0.90), # Anteil korrekt geordneter Absatzpaare
|
||||
}
|
||||
LOWER_IS_WORSE = {"order"}
|
||||
|
||||
# Zwei getrennte Urteile: "GROBID" bewertet, was GROBID aus dem PDF macht;
|
||||
# "PDF" bewertet Mängel der Vorlage, die kein Extraktor beheben kann (Scans ohne
|
||||
# Textlayer, falsch kodierte Schriften, schlechte OCR). Kodierungs- und OCR-Fehler
|
||||
# zählen nur dann zum PDF, wenn sie schon im pypdf-Rohtext stehen — sonst hat
|
||||
# GROBID sie verursacht.
|
||||
PDF_METRICS = {"scan_pages_pct"}
|
||||
|
||||
# ---------- TEI lesen ----------
|
||||
|
||||
NS = {"t": "http://www.tei-c.org/ns/1.0"}
|
||||
T = "{http://www.tei-c.org/ns/1.0}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Segment:
|
||||
kind: str # body, head, header, footnote, body_note, figure, table, back, reference
|
||||
text: str
|
||||
page: int | None
|
||||
|
||||
|
||||
def text_of(el) -> str:
|
||||
return " ".join("".join(el.itertext()).split())
|
||||
|
||||
|
||||
def first_page(el) -> int | None:
|
||||
coords = el.get("coords")
|
||||
if not coords:
|
||||
return None
|
||||
try:
|
||||
return int(coords.split(",")[0])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def read_tei(path: Path) -> tuple[list[Segment], dict]:
|
||||
root = ET.parse(path).getroot()
|
||||
segs: list[Segment] = []
|
||||
info = {}
|
||||
|
||||
title = root.find(".//t:titleStmt/t:title", NS)
|
||||
info["title"] = text_of(title) if title is not None else ""
|
||||
info["authors"] = len(root.findall(".//t:sourceDesc//t:author/t:persName", NS))
|
||||
for el in root.findall(".//t:teiHeader//t:title", NS):
|
||||
segs.append(Segment("header", text_of(el), None))
|
||||
for el in root.findall(".//t:profileDesc/t:abstract//t:p", NS):
|
||||
segs.append(Segment("header", text_of(el), first_page(el)))
|
||||
for el in root.findall(".//t:profileDesc/t:textClass//t:term", NS):
|
||||
segs.append(Segment("header", text_of(el), None))
|
||||
abstract = root.find(".//t:profileDesc/t:abstract", NS)
|
||||
info["abstract_chars"] = len(text_of(abstract)) if abstract is not None else 0
|
||||
|
||||
body = root.find(".//t:text/t:body", NS)
|
||||
info["footnote_refs"] = 0
|
||||
if body is not None:
|
||||
info["footnote_refs"] = len(body.findall(".//t:ref[@type='foot']", NS))
|
||||
# Reihenfolge des Dokuments beibehalten
|
||||
for el in body.iter():
|
||||
tag = el.tag.replace(T, "")
|
||||
if tag == "p" and not _inside(el, body, ("figure", "note")):
|
||||
segs.append(Segment("body", text_of(el), first_page(el)))
|
||||
elif tag == "head" and not _inside(el, body, ("figure",)):
|
||||
segs.append(Segment("head", text_of(el), first_page(el)))
|
||||
elif tag == "figure":
|
||||
kind = "table" if el.get("type") == "table" else "figure"
|
||||
segs.append(Segment(kind, text_of(el), first_page(el)))
|
||||
elif tag == "note" and el.get("place") != "foot" and not _inside(el, body, ("figure",)):
|
||||
segs.append(Segment("body_note", text_of(el), first_page(el)))
|
||||
|
||||
for el in root.findall(".//t:note[@place='foot']", NS):
|
||||
segs.append(Segment("footnote", text_of(el), first_page(el)))
|
||||
|
||||
back = root.find(".//t:text/t:back", NS)
|
||||
refs = []
|
||||
if back is not None:
|
||||
for div in back.findall("t:div", NS):
|
||||
if div.get("type") == "references":
|
||||
continue
|
||||
for p in div.iter():
|
||||
if p.tag in (T + "p", T + "head"):
|
||||
segs.append(Segment("back", text_of(p), first_page(p)))
|
||||
refs = back.findall(".//t:listBibl/t:biblStruct", NS)
|
||||
for b in refs:
|
||||
segs.append(Segment("reference", text_of(b), first_page(b)))
|
||||
|
||||
complete = 0
|
||||
for b in refs:
|
||||
has_title = any(text_of(t) for t in b.findall(".//t:title", NS))
|
||||
has_author = b.find(".//t:author", NS) is not None
|
||||
has_year = b.find(".//t:date", NS) is not None
|
||||
complete += has_title and has_author and has_year
|
||||
info["references"] = len(refs)
|
||||
info["refs_complete_pct"] = round(100 * complete / len(refs), 1) if refs else None
|
||||
return [s for s in segs if s.text], info
|
||||
|
||||
|
||||
_parent_cache: dict = {}
|
||||
|
||||
|
||||
def _inside(el, root, tags) -> bool:
|
||||
"""Liegt el (innerhalb von root) unter einem der genannten Elemente?"""
|
||||
parents = _parent_cache.get(id(root))
|
||||
if parents is None:
|
||||
parents = {c: p for p in root.iter() for c in p}
|
||||
_parent_cache.clear()
|
||||
_parent_cache[id(root)] = parents
|
||||
p = parents.get(el)
|
||||
while p is not None and p is not root:
|
||||
if p.tag.replace(T, "") in tags:
|
||||
return True
|
||||
p = parents.get(p)
|
||||
return False
|
||||
|
||||
# ---------- Docling-JSON und pypdf-Ausgangswert lesen ----------
|
||||
|
||||
# Docling-Labels -> Segmentarten der Prüfung
|
||||
DOCLING_KIND = {
|
||||
"title": "header", "section_header": "head", "text": "body", "paragraph": "body",
|
||||
"list_item": "body", "footnote": "footnote", "caption": "figure", "formula": "figure",
|
||||
"code": "figure", "reference": "reference", "document_index": "back",
|
||||
"page_header": "furniture", "page_footer": "furniture",
|
||||
}
|
||||
|
||||
|
||||
def read_docling(path: Path) -> tuple[list[Segment], dict]:
|
||||
"""Liest ein mit DoclingDocument.save_as_json gespeichertes Dokument in Lesereihenfolge."""
|
||||
import json
|
||||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||
segs: list[Segment] = []
|
||||
|
||||
def resolve(ref: str) -> dict:
|
||||
_, coll, idx = ref.split("/")
|
||||
return doc[coll][int(idx)]
|
||||
|
||||
def page(item) -> int | None:
|
||||
prov = item.get("prov") or []
|
||||
return prov[0].get("page_no") if prov else None
|
||||
|
||||
def walk(node, layer):
|
||||
for child in node.get("children", []):
|
||||
item = resolve(child["$ref"])
|
||||
coll = child["$ref"].split("/")[1]
|
||||
if coll == "texts":
|
||||
kind = DOCLING_KIND.get(item.get("label"), "body")
|
||||
if layer == "furniture" or item.get("content_layer") == "furniture":
|
||||
kind = "furniture"
|
||||
segs.append(Segment(kind, " ".join(item.get("text", "").split()), page(item)))
|
||||
elif coll == "tables":
|
||||
cells = (item.get("data") or {}).get("table_cells", [])
|
||||
segs.append(Segment("table", " ".join(c.get("text", "") for c in cells), page(item)))
|
||||
walk(item, layer) # Gruppen, Bildunterschriften, verschachtelte Listen
|
||||
|
||||
walk(doc.get("body", {}), "body")
|
||||
walk(doc.get("furniture", {}), "furniture")
|
||||
title = next((s.text for s in segs if s.kind == "header"), "")
|
||||
info = {"title": title, "authors": 0, "abstract_chars": 0, "footnote_refs": 0,
|
||||
"references": sum(s.kind == "reference" for s in segs), "refs_complete_pct": None}
|
||||
return [s for s in segs if s.text], info
|
||||
|
||||
|
||||
def read_pypdf_baseline(pdf: Path) -> tuple[list[Segment], dict]:
|
||||
"""Das, was der jetzige PyPDFLoader liefert: eine Einheit pro Seite, ohne Struktur."""
|
||||
segs = [Segment("body", " ".join(p.split()), i + 1) for i, p in enumerate(read_pdf_pages(pdf))]
|
||||
info = {"title": "", "authors": 0, "abstract_chars": 0, "footnote_refs": 0,
|
||||
"references": 0, "refs_complete_pct": None}
|
||||
return [s for s in segs if s.text], info
|
||||
|
||||
|
||||
def read_docling_cleaned(path: Path) -> tuple[list[Segment], dict]:
|
||||
"""Docling-JSON so, wie docling_loader.py es bereinigt an den Index gibt."""
|
||||
from docling_loader import clean_blocks
|
||||
kind = {"head": "head", "body": "body", "footnote": "footnote", "caption": "figure",
|
||||
"table": "table", "reference": "reference"}
|
||||
segs = [Segment(kind.get(b.kind, "body"), b.text, b.page) for b in clean_blocks(path)]
|
||||
info = {"title": "", "authors": 0, "abstract_chars": 0, "footnote_refs": 0,
|
||||
"references": sum(s.kind == "reference" for s in segs), "refs_complete_pct": None}
|
||||
return segs, info
|
||||
|
||||
|
||||
def read_segments(pdf: Path, out: Path | None, cleaned: bool = False) -> tuple[list[Segment], dict]:
|
||||
"""Extraktor-Ausgabe einlesen: GROBID-TEI, Docling-JSON (roh oder wie vom Loader
|
||||
bereinigt) oder (out=None) pypdf."""
|
||||
if out is None:
|
||||
return read_pypdf_baseline(pdf)
|
||||
if out.name.endswith(".json"):
|
||||
return read_docling_cleaned(out) if cleaned else read_docling(out)
|
||||
return read_tei(out)
|
||||
|
||||
# ---------- PDF-Rohtext ----------
|
||||
|
||||
|
||||
def read_pdf_pages(path: Path) -> list[str]:
|
||||
# Getrennte Doppelseiten (split_spreads.py): pypdf ignoriert den Zuschnitt und
|
||||
# läse auf jeder Hälfte die ganze Doppelseite, deshalb begrenzt über pdfium
|
||||
path = Path(path)
|
||||
if path.with_name(path.stem + ".pages.json").exists():
|
||||
from split_spreads import bounded_page_texts
|
||||
return bounded_page_texts(path)
|
||||
reader = PdfReader(path)
|
||||
pages = []
|
||||
for page in reader.pages:
|
||||
try:
|
||||
pages.append(page.extract_text() or "")
|
||||
except Exception:
|
||||
pages.append("")
|
||||
return pages
|
||||
|
||||
|
||||
def boilerplate_lines(pages: list[str]) -> set[str]:
|
||||
"""Zeilen, die oben/unten auf vielen Seiten wiederkehren (Kolumnentitel, Fußzeilen)."""
|
||||
def norm(line):
|
||||
return re.sub(r"\d+", "#", " ".join(line.split()).lower())
|
||||
counts = Counter()
|
||||
for text in pages:
|
||||
lines = [l for l in text.splitlines() if l.strip()]
|
||||
for line in set(lines[:3] + lines[-3:]):
|
||||
counts[norm(line)] += 1
|
||||
need = max(3, int(0.3 * len(pages)))
|
||||
return {l for l, c in counts.items() if c >= need and len(l) > 3}
|
||||
|
||||
|
||||
def strip_boilerplate(text: str, boiler: set[str]) -> str:
|
||||
keep = []
|
||||
for line in text.splitlines():
|
||||
if re.sub(r"\d+", "#", " ".join(line.split()).lower()) not in boiler:
|
||||
keep.append(line)
|
||||
return "\n".join(keep)
|
||||
|
||||
# ---------- Normalisierung für den Abgleich ----------
|
||||
|
||||
|
||||
def normalize(text: str, dehyphenate: bool = False) -> list[str]:
|
||||
text = unicodedata.normalize("NFKC", text).replace("", "")
|
||||
if dehyphenate:
|
||||
text = re.sub(r"(\w)-\s*\n\s*(?=[a-zäöüß])", r"\1", text)
|
||||
return re.findall(r"[^\W_]+", text.lower())
|
||||
|
||||
|
||||
def shingles(tokens: list[str]) -> list[tuple]:
|
||||
return [tuple(tokens[i:i + SHINGLE]) for i in range(len(tokens) - SHINGLE + 1)]
|
||||
|
||||
# ---------- Prüfungen ----------
|
||||
|
||||
# URLs und DOIs stehen in wissenschaftlicher Prosa legitim, sie zählen nicht als Fremdtext.
|
||||
BOILER_RE = re.compile(
|
||||
r"downloaded (from|by)|heruntergeladen|ip address|©|\bÓ\s?\d{4}|"
|
||||
r"all rights reserved|alle rechte vorbehalten|creative ?commons|distributed under|"
|
||||
r"published with license|licensed under|\bissn\b|"
|
||||
r"this article (is|was) published", re.I)
|
||||
YEAR_RE = re.compile(r"\b(1[5-9]|20)\d{2}[a-z]?\b")
|
||||
AUTHOR_RE = re.compile(r"[A-ZÄÖÜ][a-zäöüß]+,\s(?:[A-ZÄÖÜ]\.\s?)+")
|
||||
TERMINAL_RE = re.compile(r"[.!?:;…\"'“”‘’»«)\]]$")
|
||||
HYPHEN_RE = re.compile(r"\b([^\W\d_]{2,})- ([a-zäöüß]{2,})\b")
|
||||
HYPHEN_OK = {"und", "oder", "bis", "sowie", "als", "wie", "noch", "and", "or", "to", "nor", "versus"}
|
||||
FNMARK_RE = re.compile(r"\b[a-zäöüß]{3,}[.,;:]?\d{1,3}\b(?![.,]\d)")
|
||||
# abgesetzte Fußnotenzeichen aus OCR: "an Felice 6 , die" oder "Schreibart 1 6 :"
|
||||
FNMARK_SPACED_RE = re.compile(
|
||||
r"\b([A-Za-zÄÖÜäöüß]{3,})[.,;:!?»«\"”)]? (\d{1,3})(?: \d)? (?=[,;:]|[A-ZÄÖÜ„»])")
|
||||
FNMARK_SPACED_SKIP = {
|
||||
"seite", "seiten", "band", "bd", "nr", "nummer", "kap", "kapitel", "abschnitt", "teil", "vers",
|
||||
"zeile", "anm", "anmerkung", "heft", "abb", "tab", "tabelle", "punkt", "absatz", "satz", "page",
|
||||
"vol", "chapter", "section", "figure", "fig", "table", "experiment", "study", "studie",
|
||||
"hypothesis", "hypothese", "condition", "group", "gruppe", "item", "items", "grade", "grades",
|
||||
"klasse", "wave", "phase", "step", "level", "model", "modell", "booklet", "segment", "aufl"}
|
||||
# typische OCR-Fehlerbilder: Ziffer im Wort (erl10ben), Backslash (\Vort), Zeichen, die
|
||||
# in Prosa nicht vorkommen (| ~ ^), Satzzeichensalat (-----,~.). Binnenmajuskeln (KaJkas)
|
||||
# bleiben außen vor: zu viele Fehlalarme bei Namen und Werkzeugen (McNamara, CoMap).
|
||||
OCR_RE = re.compile(
|
||||
r"\b[a-zäöüß]+\d+[a-zäöüß]+\b|\\[A-Za-z]|[|~^]"
|
||||
r"|(?![^\w\s]*(?:\.\.\.|…))(?![.…\-–—]{4,})[^\w\s]{4,}") # Auslassungspunkte zählen nicht
|
||||
URL_RE = re.compile(r"https?://\S+|\b\S+\.(?:org|io|com|de|net)/\S*")
|
||||
ENCODING_RE = re.compile(
|
||||
"[�-ff-st\u0000-\u0008\u000b\u000c\u000e-\u001f]"
|
||||
r"|Ã[\x80-\xbf]|â€|Â(?=\s|\w)|(?<=\s)[¼½¾](?=\s)|(?<=\w) ?Ó ?(?=\w)|(?<=[\s(])À(?=\s?[\d.])")
|
||||
|
||||
|
||||
# Platzhalter, die pypdf für Glyphen ohne Unicode-Zuordnung ausgibt
|
||||
PDF_GLYPH_RE = re.compile(r"/(?:C\d{1,3}|G\d{1,3}|g\d{1,3}|uni[0-9A-Fa-f]{4}|cid:?\d+)\b|\(cid:\d+\)")
|
||||
|
||||
|
||||
def bibliography_like(t: str) -> bool:
|
||||
"""Kurzer Absatz, der überwiegend aus Literaturangaben besteht.
|
||||
Belegdichte Prosa ("vgl. Binder 1983, S. 170") soll nicht anschlagen."""
|
||||
if len(t) > 800:
|
||||
return False
|
||||
years = len(YEAR_RE.findall(t))
|
||||
authors = len(AUTHOR_RE.findall(t)) + len(re.findall(r"[A-ZÄÖÜ][a-zäöüß]+, [A-ZÄÖÜ][a-zäöüß]+[^:]{0,20}:", t))
|
||||
return years >= 2 and authors >= 1 and years * 1000 / len(t) >= 5
|
||||
|
||||
|
||||
def prose_like(t: str) -> bool:
|
||||
"""Grob: sieht nach Fließtext aus (lang, Sätze, wenig Ziffern)."""
|
||||
if len(t) < 300:
|
||||
return False
|
||||
digits = sum(c.isdigit() for c in t) / len(t)
|
||||
sentences = len(re.findall(r"[a-zäöüß][.!?]\s+[A-ZÄÖÜ]", t))
|
||||
return digits < 0.08 and sentences >= 2
|
||||
|
||||
|
||||
META_RE = re.compile(
|
||||
r"https?|orcid|doi|licen[cs]e|creative commons|copyright|correspondence|"
|
||||
r"email|e mail|university|universität|institut|published|issn|isbn|"
|
||||
r"received|accepted|keywords|supplement|journal|verlag|rights|contact|cite this|"
|
||||
r"gmbh|research cent|department|faculty|fakultät|lehrstuhl|seminar für", re.I)
|
||||
|
||||
|
||||
def classify_lost(tokens: list[str]) -> str:
|
||||
"""Ordnet eine verlorene Passage ein; nur 'Fließtext' ist kritisch."""
|
||||
n = len(tokens)
|
||||
if sum(len(t) for t in tokens) / n < 2.5:
|
||||
return "pypdf-Lesefehler (Sperrsatz)"
|
||||
if sum(t.isdigit() or bool(re.fullmatch(r"[a-z]?\d+", t)) for t in tokens) / n > 0.25:
|
||||
return "Zahlen/Tabelle"
|
||||
text = " ".join(tokens)
|
||||
if len(META_RE.findall(text)) >= 2:
|
||||
return "Vermerk/Metadaten"
|
||||
if len(YEAR_RE.findall(text)) >= 2 and re.search(r"\b(pp|ed|eds|hg|hrsg|vol|bd|s)\b", text):
|
||||
return "Literaturangabe"
|
||||
return "Fließtext"
|
||||
|
||||
|
||||
def page_unusable(text: str) -> bool:
|
||||
"""Liefert pypdf auf dieser Seite Unbrauchbares? (Sperrsatz, verschluckte Leerzeichen)"""
|
||||
words = text.split()
|
||||
if len(words) < 30:
|
||||
return False
|
||||
mean = sum(len(w) for w in words) / len(words)
|
||||
long_share = sum(len(w) > 20 for w in words) / len(words)
|
||||
return mean < 2.5 or mean > 9 or long_share > 0.08
|
||||
|
||||
|
||||
def light(metric: str, value) -> str:
|
||||
if value is None:
|
||||
return "–"
|
||||
yellow, red = THRESHOLDS[metric]
|
||||
if metric in LOWER_IS_WORSE:
|
||||
return "ROT" if value < red else "GELB" if value < yellow else "GRÜN"
|
||||
return "ROT" if value >= red else "GELB" if value >= yellow else "GRÜN"
|
||||
|
||||
|
||||
def check(pdf: Path, out: Path | None, cleaned: bool = False) -> tuple[dict, dict]:
|
||||
"""Prüft die Ausgabe eines Extraktors (TEI, Docling-JSON oder None für pypdf) gegen das PDF.
|
||||
cleaned=True prüft Docling-JSON nach der Bereinigung durch docling_loader.py."""
|
||||
segs, info = read_segments(pdf, out, cleaned)
|
||||
pages = read_pdf_pages(pdf)
|
||||
boiler = boilerplate_lines(pages)
|
||||
ex = defaultdict(list) # Belegstellen
|
||||
m = {"datei": pdf.name, "titel": info["title"][:80], "seiten": len(pages)}
|
||||
|
||||
# 1. Textlayer
|
||||
scan_pages = [i + 1 for i, p in enumerate(pages) if len(p.strip()) < 200]
|
||||
m["scan_pages_pct"] = round(100 * len(scan_pages) / max(1, len(pages)), 1)
|
||||
if scan_pages:
|
||||
ex["Seiten ohne nennenswerten Textlayer (Scan? GROBID macht kein OCR)"] = [
|
||||
f"Seite {', '.join(map(str, scan_pages[:40]))}"]
|
||||
|
||||
# 2. Abdeckung: welcher Teil des PDF-Texts findet sich wo im TEI wieder?
|
||||
priority = ["body", "head", "header", "footnote", "body_note", "back", "table", "figure",
|
||||
"reference", "furniture"]
|
||||
tei_index: dict[tuple, str] = {}
|
||||
for kind in reversed(priority): # höhere Priorität überschreibt
|
||||
for s in segs:
|
||||
if s.kind == kind:
|
||||
for sh in shingles(normalize(s.text)):
|
||||
tei_index[sh] = kind
|
||||
|
||||
unusable = [i + 1 for i, p in enumerate(pages) if page_unusable(p)]
|
||||
m["ref_unusable_pct"] = round(100 * len(unusable) / max(1, len(pages)), 1)
|
||||
if unusable:
|
||||
ex["Seiten, auf denen pypdf keinen brauchbaren Vergleichstext liefert (nicht geprüft)"] = [
|
||||
f"Seite {', '.join(map(str, unusable[:60]))}"]
|
||||
pdf_tokens, token_page = [], []
|
||||
for i, p in enumerate(pages):
|
||||
if i + 1 in unusable:
|
||||
continue
|
||||
toks = normalize(strip_boilerplate(p, boiler), dehyphenate=True)
|
||||
pdf_tokens += toks
|
||||
token_page += [i + 1] * len(toks)
|
||||
covered = [None] * len(pdf_tokens)
|
||||
for i, sh in enumerate(shingles(pdf_tokens)):
|
||||
kind = tei_index.get(sh)
|
||||
if kind:
|
||||
for j in range(i, i + SHINGLE):
|
||||
if covered[j] is None or priority.index(kind) < priority.index(covered[j]):
|
||||
covered[j] = kind
|
||||
n = max(1, len(pdf_tokens))
|
||||
share = Counter(c or "fehlt" for c in covered)
|
||||
m["pdf_woerter"] = len(pdf_tokens)
|
||||
m["missing_all_pct"] = round(100 * share["fehlt"] / n, 1)
|
||||
m["body_share_pct"] = round(100 * (share["body"] + share["head"]) / n, 1)
|
||||
m["verteilung"] = " ".join(f"{k}={100 * v / n:.0f}%" for k, v in share.most_common())
|
||||
|
||||
# verlorene Passagen am Stück
|
||||
runs, start = [], None
|
||||
for i, c in enumerate(covered + ["end"]):
|
||||
if c is None and start is None:
|
||||
start = i
|
||||
elif c is not None and start is not None:
|
||||
if i - start >= MIN_LOST_RUN:
|
||||
runs.append((start, i))
|
||||
start = None
|
||||
runs.sort(key=lambda r: r[1] - r[0], reverse=True)
|
||||
lost_by_class = Counter()
|
||||
for a, b in runs:
|
||||
cls = classify_lost(pdf_tokens[a:b])
|
||||
lost_by_class[cls] += b - a
|
||||
title = ("Verlorener FLIESSTEXT (kritisch)" if cls == "Fließtext"
|
||||
else "Verlorene Passagen, unkritisch (Vermerke, Tabellen, Literatur, Lesefehler)")
|
||||
if len(ex[title]) < MAX_EXAMPLES:
|
||||
words = " ".join(pdf_tokens[a:b])
|
||||
ex[title].append(f"[{cls}, S. {token_page[a]}, {b - a} Wörter] "
|
||||
f"{words[:400]}{' …' if len(words) > 400 else ''}")
|
||||
m["lost_pct"] = round(100 * lost_by_class["Fließtext"] / n, 2)
|
||||
if m["ref_unusable_pct"] > MAX_UNUSABLE_REF or not pdf_tokens:
|
||||
m["lost_pct"] = None # Referenz zu schlecht, lieber keine Aussage als eine falsche
|
||||
m["lost_runs"] = len(runs)
|
||||
m["verlust_nach_art"] = " ".join(f"{k}={100 * v / n:.1f}%" for k, v in lost_by_class.most_common())
|
||||
|
||||
# 3. Fließtext-Prüfungen
|
||||
body = [s for s in segs if s.kind == "body"]
|
||||
body_chars = sum(len(s.text) for s in body) or 1
|
||||
body_words = sum(len(s.text.split()) for s in body) or 1
|
||||
m["absaetze"] = len(body)
|
||||
m["ueberschriften"] = sum(s.kind == "head" for s in segs)
|
||||
m["fliesstext_zeichen"] = body_chars
|
||||
|
||||
# Gezählt werden Funde, nicht Absätze: bei schlechter Absatzerkennung (wenige
|
||||
# Riesenabsätze) wäre ein Anteil der Absätze sonst bedeutungslos.
|
||||
heads = [l for l in boiler if len(l) >= 15]
|
||||
hits = []
|
||||
for s in body:
|
||||
norm_text = re.sub(r"\d+", "#", s.text.lower())
|
||||
found = [("Vermerk", mt.start()) for mt in BOILER_RE.finditer(s.text)]
|
||||
for h in heads:
|
||||
pos = norm_text.find(h)
|
||||
while pos >= 0:
|
||||
found.append(("Kolumnentitel/Fußzeile", pos))
|
||||
pos = norm_text.find(h, pos + 1)
|
||||
last = -1000
|
||||
for reason, pos in sorted(found, key=lambda x: x[1]):
|
||||
if pos - last > 300: # ein Einschub trifft oft mehrere Muster
|
||||
hits.append((reason, s, pos))
|
||||
last = pos
|
||||
if bibliography_like(s.text):
|
||||
hits.append(("wie Literaturangabe", s, 0))
|
||||
elif len(s.text) < 40:
|
||||
hits.append(("Fragment", s, 0))
|
||||
for reason, s, where in hits[:MAX_EXAMPLES * 3]:
|
||||
a = max(0, where - 100)
|
||||
ex["Fremdtext im Fließtext"].append(
|
||||
f"[{reason}, S. {s.page}] {'…' if a else ''}{s.text[a:where + 200]}…")
|
||||
m["contam_per_100k"] = round(100000 * len(hits) / body_chars, 1)
|
||||
|
||||
seen, dups = set(), 0
|
||||
for s in body:
|
||||
key = s.text.lower()
|
||||
if len(key) > 60 and key in seen:
|
||||
dups += 1
|
||||
ex["Doppelte Absätze"].append(f"[S. {s.page}] {s.text[:200]}")
|
||||
seen.add(key)
|
||||
m["duplicate_pct"] = round(100 * dups / max(1, len(body)), 1)
|
||||
|
||||
# Fließtext, der als Abbildung, Tabelle oder Notiz einsortiert wurde
|
||||
misfiled = 0
|
||||
for s in segs:
|
||||
if s.kind in ("figure", "body_note") and prose_like(s.text):
|
||||
misfiled += len(s.text)
|
||||
ex["Möglicher Fließtext in Abbildung/Notiz (fehlt dem Index)"].append(
|
||||
f"[{s.kind}, S. {s.page}] {s.text[:300]}")
|
||||
m["misfiled_chars"] = misfiled
|
||||
m["misfiled_pct"] = round(100 * misfiled / (body_chars + misfiled), 1)
|
||||
|
||||
# Absatzerkennung: wie viel Fließtext steckt in Riesenabsätzen?
|
||||
giant = [s for s in body if len(s.text) > GIANT_PARA]
|
||||
m["giant_para_pct"] = round(100 * sum(len(s.text) for s in giant) / body_chars, 1)
|
||||
m["laengster_absatz"] = max((len(s.text) for s in body), default=0)
|
||||
for s in sorted(giant, key=lambda s: -len(s.text))[:MAX_EXAMPLES]:
|
||||
ex[f"Riesenabsätze (> {GIANT_PARA} Zeichen, Absatzerkennung versagt)"].append(
|
||||
f"[S. {s.page}, {len(s.text):,} Zeichen] {s.text[:200]} … {s.text[-120:]}".replace(",", "."))
|
||||
|
||||
# zerrissene Sätze: Absatz endet ohne Satzzeichen, nächster beginnt klein
|
||||
broken = 0
|
||||
for a, b in zip(body, body[1:]):
|
||||
if not TERMINAL_RE.search(a.text) and b.text[:1].islower():
|
||||
broken += 1
|
||||
ex["Über Absatzgrenzen zerrissene Sätze"].append(
|
||||
f"S. {a.page}→{b.page}: …{a.text[-120:]} ‖ {b.text[:120]}…")
|
||||
m["broken_per_100p"] = round(100 * broken / max(1, len(body)), 1)
|
||||
|
||||
all_body = "\n".join(s.text for s in body)
|
||||
hyph = [mt for mt in HYPHEN_RE.finditer(all_body) if mt.group(2) not in HYPHEN_OK]
|
||||
m["hyphen_per_10k"] = round(10000 * len(hyph) / body_words, 1)
|
||||
for mt in hyph[:MAX_EXAMPLES]:
|
||||
ex["Trennstrich-Reste"].append(all_body[max(0, mt.start() - 40):mt.end() + 40])
|
||||
|
||||
enc = list(ENCODING_RE.finditer(all_body))
|
||||
m["encoding_per_10k"] = round(10000 * len(enc) / body_chars, 1)
|
||||
# Stecken dieselben Zeichen schon im PDF-Rohtext, liegt es an der Schriftkodierung
|
||||
# des PDFs, nicht an GROBID — behebbar mit einer Ersetzungstabelle im Loader.
|
||||
# pypdf zeigt dieselben Schriftmängel teils anders, als Glyphen-Platzhalter wie "/C22".
|
||||
raw_text = "\n".join(pages)
|
||||
raw_enc = len(ENCODING_RE.findall(raw_text)) + len(PDF_GLYPH_RE.findall(raw_text))
|
||||
m["kodierung_im_pdf"] = "ja" if enc and raw_enc >= 0.5 * len(enc) else ("nein" if enc else "–")
|
||||
for mt in enc[:MAX_EXAMPLES]:
|
||||
ctx = all_body[max(0, mt.start() - 40):mt.end() + 40]
|
||||
ex["Kodierungsfehler"].append(f"{mt.group()!r}: {ctx}")
|
||||
|
||||
fnm = list(FNMARK_RE.finditer(all_body))
|
||||
fns = [mt for mt in FNMARK_SPACED_RE.finditer(all_body)
|
||||
if mt.group(1).lower() not in FNMARK_SPACED_SKIP]
|
||||
m["fnmark_per_10k"] = round(10000 * (len(fnm) + len(fns)) / body_words, 1)
|
||||
for mt in fnm[:MAX_EXAMPLES // 2] + fns[:MAX_EXAMPLES // 2]:
|
||||
ex["Fußnotenzeichen im Fließtext (verfälscht Suche, Fußnote nicht getrennt)"].append(
|
||||
all_body[max(0, mt.start() - 40):mt.end() + 40])
|
||||
|
||||
ocr = list(OCR_RE.finditer(URL_RE.sub(" ", all_body)))
|
||||
m["ocr_per_10k"] = round(10000 * len(ocr) / body_words, 1)
|
||||
raw_ocr = len(OCR_RE.findall(URL_RE.sub(" ", raw_text)))
|
||||
raw_words = len(raw_text.split()) or 1
|
||||
# gleiche oder höhere Dichte im Rohtext -> die OCR-Schicht des PDFs ist schuld
|
||||
m["ocr_im_pdf"] = ("ja" if ocr and raw_ocr / raw_words >= 0.5 * len(ocr) / body_words
|
||||
else "nein" if ocr else "–")
|
||||
for mt in ocr[:MAX_EXAMPLES]:
|
||||
ctx = URL_RE.sub(" ", all_body)
|
||||
ex["OCR-Fehlerbilder"].append(f"{mt.group()!r}: {ctx[max(0, mt.start() - 40):mt.end() + 40]}")
|
||||
|
||||
# 4. Lesereihenfolge: Position jedes Absatzes im PDF-Text
|
||||
pdf_pos: dict[tuple, int] = {}
|
||||
dup = set()
|
||||
for i, sh in enumerate(shingles(pdf_tokens)):
|
||||
if sh in pdf_pos:
|
||||
dup.add(sh)
|
||||
pdf_pos.setdefault(sh, i)
|
||||
positions = []
|
||||
for s in body:
|
||||
for sh in shingles(normalize(s.text))[:30]:
|
||||
if sh in pdf_pos and sh not in dup:
|
||||
positions.append((pdf_pos[sh], s))
|
||||
break
|
||||
pairs = list(zip(positions, positions[1:]))
|
||||
ordered = sum(a[0] < b[0] for a, b in pairs)
|
||||
m["order"] = round(ordered / len(pairs), 3) if pairs else None
|
||||
for a, b in pairs:
|
||||
if a[0] > b[0] and len(ex["Absätze außerhalb der PDF-Reihenfolge"]) < MAX_EXAMPLES:
|
||||
ex["Absätze außerhalb der PDF-Reihenfolge"].append(
|
||||
f"S. {a[1].page} „{a[1].text[:80]}…“ steht vor S. {b[1].page} „{b[1].text[:80]}…“")
|
||||
m["absaetze_ohne_pdf_fundstelle"] = len(body) - len(positions)
|
||||
|
||||
# 5. Struktur
|
||||
m["fussnoten"] = sum(s.kind == "footnote" for s in segs)
|
||||
m["fussnoten_verweise"] = info["footnote_refs"]
|
||||
m["literaturangaben"] = info["references"]
|
||||
m["literatur_vollstaendig_pct"] = info["refs_complete_pct"]
|
||||
m["autoren"] = info["authors"]
|
||||
m["abstract_zeichen"] = info["abstract_chars"]
|
||||
if m["fussnoten_verweise"] and not m["fussnoten"]:
|
||||
ex["Strukturhinweise"].append(
|
||||
f"{m['fussnoten_verweise']} Fußnotenverweise im Text, aber keine Fußnote erkannt")
|
||||
if not info["title"]:
|
||||
ex["Strukturhinweise"].append("Kein Titel erkannt")
|
||||
if m["ueberschriften"] == 0:
|
||||
ex["Strukturhinweise"].append("Keine Zwischenüberschriften erkannt")
|
||||
|
||||
# Stichprobe zum Gegenlesen
|
||||
rnd = random.Random(pdf.name)
|
||||
for s in rnd.sample(body, min(SAMPLE_PARAGRAPHS, len(body))):
|
||||
ex["Stichprobe: zufällige Fließtextabsätze zum Gegenlesen"].append(f"[S. {s.page}] {s.text}")
|
||||
|
||||
m["ampel"] = {k: light(k, m[k]) for k in THRESHOLDS if k in m}
|
||||
m["ursache"] = {k: "PDF" if k in PDF_METRICS else "Extraktor" for k in m["ampel"]}
|
||||
if m["kodierung_im_pdf"] == "ja":
|
||||
m["ursache"]["encoding_per_10k"] = "PDF"
|
||||
if m["ocr_im_pdf"] == "ja":
|
||||
m["ursache"]["ocr_per_10k"] = "PDF"
|
||||
m["grobid"] = worst(l for k, l in m["ampel"].items() if m["ursache"][k] == "Extraktor")
|
||||
m["pdf"] = worst(l for k, l in m["ampel"].items() if m["ursache"][k] == "PDF")
|
||||
return m, ex
|
||||
|
||||
|
||||
def worst(lights) -> str:
|
||||
lights = list(lights)
|
||||
return "ROT" if "ROT" in lights else "GELB" if "GELB" in lights else "GRÜN"
|
||||
|
||||
# ---------- Berichte ----------
|
||||
|
||||
LABELS = {
|
||||
"scan_pages_pct": "Seiten ohne Textlayer (%)",
|
||||
"lost_pct": "Verlorener Fließtext ggü. PDF (%)",
|
||||
"giant_para_pct": f"Fließtext in Riesenabsätzen > {GIANT_PARA} Zeichen (%)",
|
||||
"contam_per_100k": "Fremdtext-Funde je 100k Zeichen",
|
||||
"duplicate_pct": "Doppelte Absätze (%)",
|
||||
"misfiled_pct": "Fließtext in Abbildung/Notiz (%)",
|
||||
"broken_per_100p": "Zerrissene Sätze je 100 Absätze",
|
||||
"hyphen_per_10k": "Trennstrich-Reste je 10k Wörter",
|
||||
"encoding_per_10k": "Kodierungsfehler je 10k Zeichen",
|
||||
"fnmark_per_10k": "Fußnotenzeichen im Fließtext je 10k Wörter",
|
||||
"ocr_per_10k": "OCR-Fehlerbilder je 10k Wörter",
|
||||
"order": "Lesereihenfolge (Anteil geordnet)",
|
||||
}
|
||||
|
||||
|
||||
def write_report(m: dict, ex: dict, path: Path):
|
||||
out = [f"# QA: {m['datei']}", "", f"**{m['titel']}**", "",
|
||||
f"Extraktor: **{m['grobid']}** · PDF-Mängel: **{m['pdf']}** · {m['seiten']} Seiten · {m['absaetze']} Absätze · "
|
||||
f"{m['ueberschriften']} Überschriften · {m['fliesstext_zeichen']:,} Zeichen Fließtext".replace(",", "."),
|
||||
"", "| Prüfung | Wert | Ampel | zählt zu | gelb ab | rot ab |", "|---|---|---|---|---|---|"]
|
||||
for k, label in LABELS.items():
|
||||
y, r = THRESHOLDS[k]
|
||||
out.append(f"| {label} | {m[k]} | {m['ampel'][k]} | {m['ursache'][k]} | {y} | {r} |")
|
||||
out += ["", f"Verteilung der PDF-Wörter auf TEI-Bereiche: {m['verteilung']}", "",
|
||||
f"Nicht wiedergefunden insgesamt: {m['missing_all_pct']} %, davon in Passagen "
|
||||
f"≥ {MIN_LOST_RUN} Wörter: {m['verlust_nach_art'] or '–'}", "",
|
||||
f"Fußnoten: {m['fussnoten']} (Verweise im Text: {m['fussnoten_verweise']}) · "
|
||||
f"Literaturangaben: {m['literaturangaben']} (vollständig: {m['literatur_vollstaendig_pct']} %) · "
|
||||
f"Autoren: {m['autoren']} · Abstract: {m['abstract_zeichen']} Zeichen · "
|
||||
f"Absätze ohne PDF-Fundstelle: {m['absaetze_ohne_pdf_fundstelle']} · "
|
||||
f"Kodierungsfehler schon im PDF angelegt: {m['kodierung_im_pdf']} · "
|
||||
f"OCR-Fehler schon im PDF angelegt: {m['ocr_im_pdf']} · "
|
||||
f"Längster Absatz: {m['laengster_absatz']} Zeichen · "
|
||||
f"Seiten ohne brauchbaren pypdf-Vergleichstext: {m['ref_unusable_pct']} %"
|
||||
+ (" → Verlustmessung ausgesetzt" if m["lost_pct"] is None else ""), ""]
|
||||
for title, items in ex.items():
|
||||
out += [f"## {title}", ""]
|
||||
out += [f"- {i}" for i in items[:MAX_EXAMPLES if 'Stichprobe' not in title else None]]
|
||||
out.append("")
|
||||
path.write_text("\n".join(out), encoding="utf-8")
|
||||
|
||||
|
||||
def tei_is_current(tei: Path) -> bool:
|
||||
"""TEI vorhanden und mit den Optionen aus grobid_test.py erzeugt (Koordinaten, Rohzitate)?"""
|
||||
if not tei.exists():
|
||||
return False
|
||||
data = tei.read_bytes()
|
||||
return re.search(rb"<p [^>]*coords=", data) is not None and (
|
||||
b"raw_reference" in data or b"<listBibl/>" in data or b"listBibl" not in data)
|
||||
|
||||
|
||||
def main():
|
||||
pdf_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else PDF_DIR
|
||||
tei_dir = OUT_ROOT / pdf_dir.name
|
||||
report_dir = tei_dir / "qa"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
pdfs = sorted(pdf_dir.glob("*.pdf"))
|
||||
if not pdfs:
|
||||
raise SystemExit(f"Keine PDFs in {pdf_dir}/")
|
||||
rows, failed = [], []
|
||||
for i, pdf in enumerate(pdfs, 1):
|
||||
tei = tei_dir / f"{pdf.stem}.tei.xml"
|
||||
t0 = time.time()
|
||||
try:
|
||||
if REPROCESS or not tei_is_current(tei):
|
||||
print(f"[{i}/{len(pdfs)}] {pdf.name}: GROBID ...", end=" ", flush=True)
|
||||
tei.write_bytes(process_pdf(pdf))
|
||||
else:
|
||||
print(f"[{i}/{len(pdfs)}] {pdf.name}: vorhandenes TEI ...", end=" ", flush=True)
|
||||
m, ex = check(pdf, tei)
|
||||
except Exception as e: # ein kaputtes PDF soll den Lauf nicht abbrechen
|
||||
print(f"FEHLER: {e}", flush=True)
|
||||
failed.append((pdf.name, str(e)))
|
||||
continue
|
||||
write_report(m, ex, report_dir / f"{pdf.stem}.md")
|
||||
rows.append(m)
|
||||
print(f"GROBID {m['grobid']}, PDF {m['pdf']} ({time.time() - t0:.0f} s)", flush=True)
|
||||
if not rows:
|
||||
raise SystemExit("Kein PDF erfolgreich geprüft.")
|
||||
|
||||
cols = ["datei", "grobid", "pdf", "seiten", *LABELS, "body_share_pct", "absaetze", "ueberschriften",
|
||||
"fussnoten", "fussnoten_verweise", "literaturangaben", "literatur_vollstaendig_pct",
|
||||
"absaetze_ohne_pdf_fundstelle", "kodierung_im_pdf", "ocr_im_pdf", "laengster_absatz",
|
||||
"ref_unusable_pct", "misfiled_chars", "missing_all_pct", "verlust_nach_art", "lost_runs", "titel"]
|
||||
with open(report_dir / "summary.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore", delimiter=";")
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
w.writerows({"datei": name, "grobid": "FEHLER", "titel": err[:200]} for name, err in failed)
|
||||
|
||||
short = {"scan_pages_pct": "Scan%", "lost_pct": "Verlust%", "giant_para_pct": "Riesen%",
|
||||
"contam_per_100k": "Fremd", "misfiled_pct": "Fehlabl%", "duplicate_pct": "Doppelt",
|
||||
"broken_per_100p": "Zerriss", "hyphen_per_10k": "Trenn", "encoding_per_10k": "Kodier",
|
||||
"ocr_per_10k": "OCR", "fnmark_per_10k": "FN-Zch", "order": "Reihenf"}
|
||||
name_w = min(40, max(len(r["datei"]) for r in rows))
|
||||
print("\n" + "Datei".ljust(name_w), "GROBID".ljust(6), "PDF".ljust(5), *(v.rjust(8) for v in short.values()))
|
||||
for r in rows:
|
||||
cells = [f"{'P:' if r['ursache'][k] == 'PDF' and r['ampel'][k] != 'GRÜN' else ''}{r[k]}"
|
||||
f"{'!' if r['ampel'][k] == 'ROT' else '?' if r['ampel'][k] == 'GELB' else ''}"
|
||||
for k in short]
|
||||
print(r["datei"][:name_w].ljust(name_w), r["grobid"].ljust(6), r["pdf"].ljust(5),
|
||||
*(c.rjust(8) for c in cells))
|
||||
print("\n? = gelb, ! = rot, P: = Mangel der PDF-Vorlage, nicht von GROBID.")
|
||||
for name, err in failed:
|
||||
print(f"FEHLER bei {name}: {err}")
|
||||
print("Details und Belegstellen:", report_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user