Add Docling corpus pipeline, retrieval evaluation and hybrid search with reranking

- prepare_corpus.py: split double-page scans (split_spreads.py), re-OCR with
  OCRmyPDF/Tesseract only where the existing text layer is worse, Docling
  layout extraction with heading hierarchy, QA report per PDF
- docling_loader.py: cleanup (margin noise, split paragraphs, footnote marks,
  OCR comma/period repair), page mapping back to the original PDF, chunk
  headers with author, title and section
- local_rag.py: bge-m3 embeddings, configurable chunking and loader, index
  settings check (index_info.json), hybrid/union BM25 search with Reciprocal
  Rank Fusion and bge-reranker-v2-m3 cross-encoder; query instruction text
  removed after it lowered recall
- eval_retrieval.py: CLI options, same retrieval path as the chat, tolerant
  file name matching; goldsets and evaluation reports for the Kafka corpus
- grobid_test.py, grobid_qa.py, extract_docling.py, compare_extractors.py:
  extractor tests and quality checks
- .gitignore: corpus PDFs, vector stores and extractor output stay local
- server.py: uses ./vectorstore_docling_hdr

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

add_pdfs.py

@@ -1,14 +1,19 @@
 #!/usr/bin/env python3
-"""Add all PDFs under a folder to the RAG vector store. Run from project root."""
+"""Add all PDFs under a folder to the RAG vector store. Run from project root.
+
+The settings below are the defaults; they can be overridden on the command line, e.g.
+    python add_pdfs.py --data data --index ./vectorstore_docling --loader docling
+"""
+import argparse
 import shutil
 from pathlib import Path
 
-from local_rag import LocalRAG
+from local_rag import CHUNK_OVERLAP, CHUNK_SIZE, PDF_LOADER, PROCESSED_DIR, LocalRAG
 
 # Folder to scan for PDFs (recursively).
 DATA_ROOT = Path("/Users/Philipp/Desktop/workspace/python/gpt_publikationen/data_vs")
 
-# Must match server.py so the chat loads the same index.
+# The chat (server.py) currently loads ./vectorstore_docling_hdr; pass --index to build that one.
 VECTORSTORE_PATH = "./vectorstore"
 
 # If True: delete the existing vector store folder, then index all PDFs from scratch.
@@ -17,6 +22,17 @@ CLEAR_VECTORSTORE_FIRST = True
 
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--data", type=Path, default=DATA_ROOT, help="folder to scan for PDFs")
+    parser.add_argument("--index", default=VECTORSTORE_PATH, help="vector store folder")
+    parser.add_argument("--loader", default=PDF_LOADER, choices=["pypdf", "docling"])
+    parser.add_argument("--processed", default=PROCESSED_DIR, help="Docling output (prepare_corpus.py)")
+    parser.add_argument("--chunk-size", type=int, default=CHUNK_SIZE)
+    parser.add_argument("--chunk-overlap", type=int, default=CHUNK_OVERLAP)
+    parser.add_argument("--no-headers", action="store_true", help="no author/title header per chunk")
+    args = parser.parse_args()
+    DATA_ROOT, VECTORSTORE_PATH = args.data, args.index
+
     if CLEAR_VECTORSTORE_FIRST and Path(VECTORSTORE_PATH).exists():
         shutil.rmtree(VECTORSTORE_PATH)
         print(f"Cleared existing vector store: {VECTORSTORE_PATH}")
@@ -28,6 +44,8 @@ if __name__ == "__main__":
     if not pdfs:
         raise SystemExit("No PDFs found.")
 
-    rag = LocalRAG(vectorstore_path=VECTORSTORE_PATH)
+    rag = LocalRAG(vectorstore_path=VECTORSTORE_PATH, pdf_loader=args.loader,
+                   processed_dir=args.processed, chunk_size=args.chunk_size,
+                   chunk_overlap=args.chunk_overlap, chunk_headers=not args.no_headers)
     rag.add_documents([str(p) for p in pdfs])
     print("Done.")