From 9ebcda1f55c951630aa0fa5bf7fed8518ade28d2 Mon Sep 17 00:00:00 2001 From: Philipp Mock Date: Wed, 7 Oct 2026 18:04:48 +0200 Subject: [PATCH] Add evaluation matrix for 2000/400 chunks (loader x headers x search mode) Co-Authored-By: Claude Opus 5.5 --- eval/build_pypdf_2000_hdr.log | 2 + eval/eval_m2000_human_docling_2000_dense.log | 105 ++++++++++++++ ...val_m2000_human_docling_2000_hdr_dense.log | 105 ++++++++++++++ ...al_m2000_human_docling_2000_hdr_hybrid.log | 105 ++++++++++++++ ...val_m2000_human_docling_2000_hdr_union.log | 105 ++++++++++++++ eval/eval_m2000_human_docling_2000_hybrid.log | 105 ++++++++++++++ eval/eval_m2000_human_docling_2000_union.log | 105 ++++++++++++++ eval/eval_m2000_human_pypdf_2000_dense.log | 105 ++++++++++++++ .../eval_m2000_human_pypdf_2000_hdr_dense.log | 105 ++++++++++++++ ...eval_m2000_human_pypdf_2000_hdr_hybrid.log | 105 ++++++++++++++ .../eval_m2000_human_pypdf_2000_hdr_union.log | 105 ++++++++++++++ eval/eval_m2000_human_pypdf_2000_hybrid.log | 105 ++++++++++++++ eval/eval_m2000_human_pypdf_2000_union.log | 105 ++++++++++++++ ...val_m2000_synthetic_docling_2000_dense.log | 131 ++++++++++++++++++ ...m2000_synthetic_docling_2000_hdr_dense.log | 131 ++++++++++++++++++ ...2000_synthetic_docling_2000_hdr_hybrid.log | 127 +++++++++++++++++ ...m2000_synthetic_docling_2000_hdr_union.log | 127 +++++++++++++++++ ...al_m2000_synthetic_docling_2000_hybrid.log | 131 ++++++++++++++++++ ...val_m2000_synthetic_docling_2000_union.log | 131 ++++++++++++++++++ .../eval_m2000_synthetic_pypdf_2000_dense.log | 131 ++++++++++++++++++ ...l_m2000_synthetic_pypdf_2000_hdr_dense.log | 131 ++++++++++++++++++ ..._m2000_synthetic_pypdf_2000_hdr_hybrid.log | 127 +++++++++++++++++ ...l_m2000_synthetic_pypdf_2000_hdr_union.log | 127 +++++++++++++++++ ...eval_m2000_synthetic_pypdf_2000_hybrid.log | 127 +++++++++++++++++ .../eval_m2000_synthetic_pypdf_2000_union.log | 127 +++++++++++++++++ eval/kafka_m2000_human_docling_2000_dense.txt | 59 ++++++++ ...fka_m2000_human_docling_2000_hdr_dense.txt | 59 ++++++++ ...ka_m2000_human_docling_2000_hdr_hybrid.txt | 59 ++++++++ ...fka_m2000_human_docling_2000_hdr_union.txt | 59 ++++++++ .../kafka_m2000_human_docling_2000_hybrid.txt | 59 ++++++++ eval/kafka_m2000_human_docling_2000_union.txt | 59 ++++++++ eval/kafka_m2000_human_pypdf_2000_dense.txt | 59 ++++++++ ...kafka_m2000_human_pypdf_2000_hdr_dense.txt | 59 ++++++++ ...afka_m2000_human_pypdf_2000_hdr_hybrid.txt | 59 ++++++++ ...kafka_m2000_human_pypdf_2000_hdr_union.txt | 59 ++++++++ eval/kafka_m2000_human_pypdf_2000_hybrid.txt | 59 ++++++++ eval/kafka_m2000_human_pypdf_2000_union.txt | 59 ++++++++ ...fka_m2000_synthetic_docling_2000_dense.txt | 72 ++++++++++ ...m2000_synthetic_docling_2000_hdr_dense.txt | 72 ++++++++++ ...2000_synthetic_docling_2000_hdr_hybrid.txt | 68 +++++++++ ...m2000_synthetic_docling_2000_hdr_union.txt | 68 +++++++++ ...ka_m2000_synthetic_docling_2000_hybrid.txt | 72 ++++++++++ ...fka_m2000_synthetic_docling_2000_union.txt | 72 ++++++++++ ...kafka_m2000_synthetic_pypdf_2000_dense.txt | 72 ++++++++++ ...a_m2000_synthetic_pypdf_2000_hdr_dense.txt | 72 ++++++++++ ..._m2000_synthetic_pypdf_2000_hdr_hybrid.txt | 68 +++++++++ ...a_m2000_synthetic_pypdf_2000_hdr_union.txt | 68 +++++++++ ...afka_m2000_synthetic_pypdf_2000_hybrid.txt | 68 +++++++++ ...kafka_m2000_synthetic_pypdf_2000_union.txt | 68 +++++++++ eval/matrix_2000.out | 25 ++++ 50 files changed, 4383 insertions(+) create mode 100644 eval/eval_m2000_human_docling_2000_dense.log create mode 100644 eval/eval_m2000_human_docling_2000_hdr_dense.log create mode 100644 eval/eval_m2000_human_docling_2000_hdr_hybrid.log create mode 100644 eval/eval_m2000_human_docling_2000_hdr_union.log create mode 100644 eval/eval_m2000_human_docling_2000_hybrid.log create mode 100644 eval/eval_m2000_human_docling_2000_union.log create mode 100644 eval/eval_m2000_human_pypdf_2000_dense.log create mode 100644 eval/eval_m2000_human_pypdf_2000_hdr_dense.log create mode 100644 eval/eval_m2000_human_pypdf_2000_hdr_hybrid.log create mode 100644 eval/eval_m2000_human_pypdf_2000_hdr_union.log create mode 100644 eval/eval_m2000_human_pypdf_2000_hybrid.log create mode 100644 eval/eval_m2000_human_pypdf_2000_union.log create mode 100644 eval/eval_m2000_synthetic_docling_2000_dense.log create mode 100644 eval/eval_m2000_synthetic_docling_2000_hdr_dense.log create mode 100644 eval/eval_m2000_synthetic_docling_2000_hdr_hybrid.log create mode 100644 eval/eval_m2000_synthetic_docling_2000_hdr_union.log create mode 100644 eval/eval_m2000_synthetic_docling_2000_hybrid.log create mode 100644 eval/eval_m2000_synthetic_docling_2000_union.log create mode 100644 eval/eval_m2000_synthetic_pypdf_2000_dense.log create mode 100644 eval/eval_m2000_synthetic_pypdf_2000_hdr_dense.log create mode 100644 eval/eval_m2000_synthetic_pypdf_2000_hdr_hybrid.log create mode 100644 eval/eval_m2000_synthetic_pypdf_2000_hdr_union.log create mode 100644 eval/eval_m2000_synthetic_pypdf_2000_hybrid.log create mode 100644 eval/eval_m2000_synthetic_pypdf_2000_union.log create mode 100644 eval/kafka_m2000_human_docling_2000_dense.txt create mode 100644 eval/kafka_m2000_human_docling_2000_hdr_dense.txt create mode 100644 eval/kafka_m2000_human_docling_2000_hdr_hybrid.txt create mode 100644 eval/kafka_m2000_human_docling_2000_hdr_union.txt create mode 100644 eval/kafka_m2000_human_docling_2000_hybrid.txt create mode 100644 eval/kafka_m2000_human_docling_2000_union.txt create mode 100644 eval/kafka_m2000_human_pypdf_2000_dense.txt create mode 100644 eval/kafka_m2000_human_pypdf_2000_hdr_dense.txt create mode 100644 eval/kafka_m2000_human_pypdf_2000_hdr_hybrid.txt create mode 100644 eval/kafka_m2000_human_pypdf_2000_hdr_union.txt create mode 100644 eval/kafka_m2000_human_pypdf_2000_hybrid.txt create mode 100644 eval/kafka_m2000_human_pypdf_2000_union.txt create mode 100644 eval/kafka_m2000_synthetic_docling_2000_dense.txt create mode 100644 eval/kafka_m2000_synthetic_docling_2000_hdr_dense.txt create mode 100644 eval/kafka_m2000_synthetic_docling_2000_hdr_hybrid.txt create mode 100644 eval/kafka_m2000_synthetic_docling_2000_hdr_union.txt create mode 100644 eval/kafka_m2000_synthetic_docling_2000_hybrid.txt create mode 100644 eval/kafka_m2000_synthetic_docling_2000_union.txt create mode 100644 eval/kafka_m2000_synthetic_pypdf_2000_dense.txt create mode 100644 eval/kafka_m2000_synthetic_pypdf_2000_hdr_dense.txt create mode 100644 eval/kafka_m2000_synthetic_pypdf_2000_hdr_hybrid.txt create mode 100644 eval/kafka_m2000_synthetic_pypdf_2000_hdr_union.txt create mode 100644 eval/kafka_m2000_synthetic_pypdf_2000_hybrid.txt create mode 100644 eval/kafka_m2000_synthetic_pypdf_2000_union.txt create mode 100644 eval/matrix_2000.out diff --git a/eval/build_pypdf_2000_hdr.log b/eval/build_pypdf_2000_hdr.log index 5fe23b3..71699fd 100644 --- a/eval/build_pypdf_2000_hdr.log +++ b/eval/build_pypdf_2000_hdr.log @@ -82,3 +82,5 @@ Ignoring wrong pointing object 42 0 (offset 0) - Trost,_Igor_Erzählen und Besprechen.pdf: 47 chunks Creating embeddings for 7398 chunks... +Vector store saved to ./vectorstore_pypdf_2000_hdr +Done. diff --git a/eval/eval_m2000_human_docling_2000_dense.log b/eval/eval_m2000_human_docling_2000_dense.log new file mode 100644 index 0000000..130a706 --- /dev/null +++ b/eval/eval_m2000_human_docling_2000_dense.log @@ -0,0 +1,105 @@ +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. +Lese Referenzfragen aus eval/goldset_kafka_human.csv ... +30 Frage(n) geladen. + +Loading embeddings model... + Loading weights: 0%| | 0/391 [00:00