Primary-text index for personas, faster retrieval, OpenAI model options

- Contemporary and Kafka personas search only "Die Verwandlung" (own index and tool description); per-persona subtitle
- New contemporary portrait and avatar alt text
- ChatGPT: gpt-6.1-sol, reasoning_effort low via Responses API; default in the model menu
- GPT-Realtime-2.1 Mini as third option (realtime_chat.py, WebSocket, text only)
- Load .env for OPENAI_API_KEY
- Reranker in fp16 on GPU with 25 candidates; reranker, embeddings and BM25 shared and built at server start
- BM25 stem cache (first search ~31s -> ~1.7s)
- Timing output per LLM call and search

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-26 07:04:59 +01:00
co-authored by Claude Opus 5.5
parent f68d7a6e25
commit e4b7f0ce34
6 changed files with 243 additions and 46 deletions
+100 -23
View File
@@ -5,6 +5,7 @@ Minimal dependencies, simple code
import json
import os
import re
import time
from pathlib import Path
import numpy as np
@@ -49,7 +50,9 @@ RETRIEVAL_K = 5
# "union" BM25 only adds candidates (BM25_CANDIDATES) to the vector candidates;
# the order comes from the vector search or, if enabled, from the reranker
RETRIEVAL_MODE = "hybrid"
CANDIDATES_K = 50 # candidates per search before merging and reranking
# 25 instead of 50 halves the reranker time and was not worse on either Kafka goldset
# (pypdf 2000/400, R@5 human .70 -> .73, synthetic .91 -> .98).
CANDIDATES_K = 25 # candidates per search before merging and reranking
BM25_CANDIDATES = 20 # extra keyword candidates in "union" mode
RRF_K = 60 # standard constant of Reciprocal Rank Fusion
@@ -87,6 +90,43 @@ Du hast ein Tool, {SEARCH_TOOL_NAME}, das eine Bibliothek aus Kafkas Texten und
- Nenne bei Aussagen aus den Auszügen die Quelle (Autor, Seite)."""
# Cross-encoders by model name, shared by all LocalRAG instances in a process (one copy in memory)
_RERANKERS = {}
# Embedding models, shared the same way
_EMBEDDINGS = {}
# BM25 keyword indexes by (vector store path, number of chunks), shared by instances on the same index
_BM25_INDEXES = {}
def load_embeddings(model_name):
"""Load an embedding model once per process and return the shared instance."""
if model_name not in _EMBEDDINGS:
print("Loading embeddings model...")
# Normalized vectors: FAISS's L2 distance then ranks exactly like cosine similarity.
_EMBEDDINGS[model_name] = HuggingFaceEmbeddings(
model_name=model_name,
encode_kwargs={"normalize_embeddings": True},
query_encode_kwargs={"normalize_embeddings": True},
)
return _EMBEDDINGS[model_name]
def load_reranker(model_name):
"""Load a cross-encoder once per process and return the shared instance."""
if model_name not in _RERANKERS:
import torch
from sentence_transformers import CrossEncoder
# Half precision on the GPU (Apple MPS or CUDA): about 15-25 % faster, same rankings
# on both Kafka goldsets. On the CPU fp16 is slow, so keep fp32 there.
gpu = torch.backends.mps.is_available() or torch.cuda.is_available()
model_kwargs = {"torch_dtype": torch.float16} if gpu else {}
print(f"Loading reranker {model_name} ({'fp16' if gpu else 'fp32'})...")
reranker = CrossEncoder(model_name, max_length=1024, model_kwargs=model_kwargs)
reranker.predict([("Probe", "Probe")]) # warm-up, so the first real query is not slower
_RERANKERS[model_name] = reranker
return _RERANKERS[model_name]
class LocalRAG:
def __init__(
self,
@@ -94,6 +134,7 @@ class LocalRAG:
llm_provider="ollama",
ollama_model="gemma4:12b",
openai_model="gpt-6-luna",
realtime_model="gpt-realtime-2.1-mini",
ollama_base_url="http://localhost:11434",
pdf_loader=PDF_LOADER,
processed_dir=PROCESSED_DIR,
@@ -103,7 +144,8 @@ class LocalRAG:
chunk_overlap=CHUNK_OVERLAP,
chunk_headers=CHUNK_HEADERS,
):
"""Initialize local RAG system. llm_provider: 'ollama' or 'openai'.
"""Initialize local RAG system. llm_provider: 'ollama', 'openai' or 'realtime'
(OpenAI Realtime API, see realtime_chat.py).
pdf_loader: 'pypdf' or 'docling' (see PDF_LOADER)."""
self.vectorstore_path = vectorstore_path
self.llm_provider = llm_provider
@@ -113,16 +155,11 @@ class LocalRAG:
self.retrieval_mode = retrieval_mode
self.reranker_model = reranker_model
self._bm25 = None # built on first use from the documents in the index
self._reranker = None # loaded on first use
self._reranker = None # loaded on first use, or up front via preload()
self.last_rerank_seconds = 0.0 # time of the last reranking, for the timing trace
# Embeddings
print("Loading embeddings model...")
# Normalized vectors: FAISS's L2 distance then ranks exactly like cosine similarity.
self.embeddings = HuggingFaceEmbeddings(
model_name=EMBEDDING_MODEL,
encode_kwargs={"normalize_embeddings": True},
query_encode_kwargs={"normalize_embeddings": True},
)
# Embeddings (one shared copy per process)
self.embeddings = load_embeddings(EMBEDDING_MODEL)
# Text splitter (used when adding documents; tune CHUNK_SIZE / CHUNK_OVERLAP above)
self.text_splitter = RecursiveCharacterTextSplitter(
@@ -131,15 +168,23 @@ class LocalRAG:
)
print(f"Text splitter: chunk_size={chunk_size}, chunk_overlap={chunk_overlap}")
# LLM (Ollama or OpenAI)
if llm_provider == "openai":
# LLM (Ollama, OpenAI or OpenAI Realtime)
if llm_provider in ("openai", "realtime"):
api_key = os.environ.get("OPENAI_API_KEY")
if not api_key:
raise ValueError(
"OPENAI_API_KEY environment variable is required when llm_provider='openai'"
f"OPENAI_API_KEY environment variable is required when llm_provider='{llm_provider}'"
)
if llm_provider == "realtime":
from realtime_chat import ChatOpenAIRealtime
print(f"Using OpenAI Realtime (model: {realtime_model})...")
self.llm = ChatOpenAIRealtime(model=realtime_model, api_key=api_key)
elif llm_provider == "openai":
print(f"Using OpenAI (model: {openai_model})...")
self.llm = ChatOpenAI(model=openai_model, api_key=api_key)
# Responses API: GPT-6 models accept function tools on Chat Completions only with
# reasoning_effort="none"; with "low" the search tool needs the Responses API.
self.llm = ChatOpenAI(model=openai_model, api_key=api_key, reasoning_effort="low",
use_responses_api=True)
else:
print(f"Using Ollama (model: {ollama_model})...")
self.llm = ChatOllama(
@@ -318,6 +363,7 @@ class LocalRAG:
# ---------- retrieval: vector search, keyword search, reranking ----------
_stemmer = None
_stems = {} # word -> stem; the corpus has far fewer distinct words than word occurrences
@classmethod
def _bm25_tokens(cls, text):
@@ -325,13 +371,26 @@ class LocalRAG:
if cls._stemmer is None:
import snowballstemmer
cls._stemmer = snowballstemmer.stemmer("german")
return cls._stemmer.stemWords(re.findall(r"[a-zäöüß0-9]{2,}", text.lower()))
stems = cls._stems
tokens = []
for word in re.findall(r"[a-zäöüß0-9]{2,}", text.lower()):
stem = stems.get(word)
if stem is None:
stem = stems[word] = cls._stemmer.stemWord(word)
tokens.append(stem)
return tokens
def _bm25_index(self):
if self._bm25 is None:
from rank_bm25 import BM25Okapi
docs = list(self.vectorstore.docstore._dict.values())
self._bm25 = (BM25Okapi([self._bm25_tokens(d.page_content) for d in docs]), docs)
key = (self.vectorstore_path, self.vectorstore.index.ntotal)
if key not in _BM25_INDEXES:
from rank_bm25 import BM25Okapi
started = time.perf_counter()
docs = list(self.vectorstore.docstore._dict.values())
_BM25_INDEXES[key] = (BM25Okapi([self._bm25_tokens(d.page_content) for d in docs]), docs)
print(f"BM25 index for {self.vectorstore_path}: {len(docs)} chunks, "
f"{time.perf_counter() - started:.1f}s")
self._bm25 = _BM25_INDEXES[key]
return self._bm25
def retrieve(self, query, k=None):
@@ -365,14 +424,22 @@ class LocalRAG:
candidates = [[d, s] for d, s in dense]
if self.reranker_model and candidates:
if self._reranker is None:
from sentence_transformers import CrossEncoder
print(f"Loading reranker {self.reranker_model}...")
self._reranker = CrossEncoder(self.reranker_model, max_length=1024)
self._reranker = load_reranker(self.reranker_model)
started = time.perf_counter()
relevance = self._reranker.predict([(query, d.page_content) for d, _ in candidates])
self.last_rerank_seconds = time.perf_counter() - started
candidates = [[d, float(r)] for (d, _), r in zip(candidates, relevance)]
candidates.sort(key=lambda e: e[1], reverse=True)
return [(d, s) for d, s in candidates[:k]]
def preload(self):
"""Load the reranker and build the BM25 index now instead of on the first search
(e.g. at server start)."""
if self.reranker_model and self._reranker is None:
self._reranker = load_reranker(self.reranker_model)
if self.retrieval_mode != "dense" and self.vectorstore is not None:
self._bm25_index()
def query(self, question, k=None):
"""Query the RAG system (no conversation history). Returns dict with 'answer' and 'retrieved'."""
if k is None:
@@ -407,10 +474,13 @@ class LocalRAG:
def search_kafka_corpus(query: str) -> str:
retrieval_query = query + (f"\n\n{QUERY_ADDITIONAL_INSTRUCTIONS}" if QUERY_ADDITIONAL_INSTRUCTIONS else "")
started = time.perf_counter()
hits = self.retrieve(retrieval_query, k=k)
seconds = time.perf_counter() - started
searches.append(query)
retrieved.extend(self._docs_scores_to_retrieved(hits))
print(f"[RAG] Tool search {query!r}: {len(hits)} chunk(s)")
print(f"[Zeit] Suche {seconds:.2f}s (davon Reranker {self.last_rerank_seconds:.2f}s)")
if not hits:
return "Keine passenden Textstellen gefunden."
return "\n\n".join(
@@ -433,9 +503,13 @@ class LocalRAG:
messages.append(HumanMessage(question))
print(f"[RAG] User question: {question!r}")
started_total = time.perf_counter()
answer = None
for _ in range(MAX_TOOL_ROUNDS):
for round_no in range(1, MAX_TOOL_ROUNDS + 1):
started = time.perf_counter()
reply = llm_with_tools.invoke(messages)
print(f"[Zeit] LLM-Aufruf {round_no}: {time.perf_counter() - started:.2f}s "
f"({'Suche angefordert' if reply.tool_calls else 'Antwort'})")
messages.append(reply)
if not reply.tool_calls:
answer = self._text(reply)
@@ -445,8 +519,11 @@ class LocalRAG:
messages.append(ToolMessage(result, tool_call_id=call.get("id") or f"call_{len(searches)}_{n}"))
if answer is None:
# Search budget used up: answer with what was found, without further tool calls
started = time.perf_counter()
answer = self._text(self.llm.invoke(messages + [HumanMessage(
"Beantworte jetzt die Frage mit den gefundenen Auszügen, ohne weitere Suche.")]))
print(f"[Zeit] LLM-Aufruf (Antwort erzwungen): {time.perf_counter() - started:.2f}s")
print(f"[Zeit] Gesamt {time.perf_counter() - started_total:.2f}s")
print(f"[RAG] Searches: {searches or 'keine'}")
return {"answer": answer, "retrieved": retrieved, "searches": searches}
+1 -1
View File
@@ -131,7 +131,7 @@ Du kennst Prag und den Literaturbetrieb zu dieser Zeit. Du bist ein Zeitgenosse
"Warum wird Gregor zu einem Ungeziefer?",
],
"system_prompt": f"""Du sprichst als Franz Kafka (1883–1924), in einer behutsamen Annäherung aus seinen Texten.
Du sprichst in der Ich-Form, ruhig, genau, zweifelnd und selbstironisch, im Ton seiner Briefe und Tagebücher, und siezt dein Gegenüber. Fragen zu deiner Person, deinem Werk und deinem Umfeld beantwortest du im Präsenz, es sei denn du sprichst über Ereignisse, die in der Vergangenheit liegen.
Du sprichst in der Ich-Form, ruhig, genau, zweifelnd und selbstironisch, im Ton seiner Briefe und Tagebücher, und siezt dein Gegenüber. Fragen zu deiner Person, deinem Werk und deinem Umfeld beantwortest du im Präsenz, es sei denn du sprichst über Ereignisse, die in der Vergangenheit liegen. Beantworte keine Fragen, die nichts mit dir oder deinem Werk zu tun haben.
{_TOOL_RULES}
- Stütze Aussagen über dein Werk auf die gefundenen Textstellen. Wo sie nichts hergeben, sag, dass du es nicht weißt oder nicht sagen kannst. Erfinde keine Zitate.""",
"tool_description": PRIMARY_TOOL_DESCRIPTION,
+107
View File
@@ -0,0 +1,107 @@
"""
Text-only LangChain chat model for OpenAI's Realtime API (e.g. gpt-realtime-2.1-mini).
Realtime models are not served on Chat Completions or the Responses API, only on
wss://api.openai.com/v1/realtime. This wrapper opens one WebSocket session per call, replays the
messages as conversation items, asks for a text response and returns it as an AIMessage (with
tool_calls if the model calls a function). That way LocalRAG.query_with_history can use it like
ChatOpenAI or ChatOllama, including bind_tools().
"""
import json
import uuid
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.utils.function_calling import convert_to_openai_tool
from websockets.sync.client import connect
REALTIME_URL = "wss://api.openai.com/v1/realtime?model={model}"
def _text(content):
"""Plain text of a LangChain message content (string or list of blocks)."""
if isinstance(content, list):
return "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in content)
return content or ""
class ChatOpenAIRealtime(BaseChatModel):
model: str
api_key: str
timeout: float = 120.0
@property
def _llm_type(self):
return "openai-realtime"
def bind_tools(self, tools, **kwargs):
"""Tools in the Realtime format: {"type": "function", "name", "description", "parameters"}."""
specs = []
for tool in tools:
fn = convert_to_openai_tool(tool)["function"]
specs.append({"type": "function", "name": fn["name"],
"description": fn.get("description", ""), "parameters": fn["parameters"]})
return self.bind(tools=specs, **kwargs)
@staticmethod
def _items(messages):
"""System prompt and conversation items for the Realtime session."""
instructions, items = [], []
for m in messages:
if isinstance(m, SystemMessage):
instructions.append(_text(m.content))
elif isinstance(m, HumanMessage):
items.append({"type": "message", "role": "user",
"content": [{"type": "input_text", "text": _text(m.content)}]})
elif isinstance(m, AIMessage):
if _text(m.content).strip():
items.append({"type": "message", "role": "assistant",
"content": [{"type": "output_text", "text": _text(m.content)}]})
for call in m.tool_calls:
items.append({"type": "function_call", "call_id": call["id"], "name": call["name"],
"arguments": json.dumps(call["args"], ensure_ascii=False)})
elif isinstance(m, ToolMessage):
items.append({"type": "function_call_output", "call_id": m.tool_call_id,
"output": _text(m.content)})
return "\n\n".join(instructions), items
def _generate(self, messages, stop=None, run_manager=None, tools=None, **kwargs):
instructions, items = self._items(messages)
session = {"type": "realtime", "output_modalities": ["text"], "instructions": instructions}
if tools:
session.update(tools=tools, tool_choice="auto")
with connect(REALTIME_URL.format(model=self.model),
additional_headers={"Authorization": f"Bearer {self.api_key}"},
open_timeout=30, max_size=None) as ws:
def send(event):
ws.send(json.dumps(event, ensure_ascii=False))
send({"type": "session.update", "session": session})
for item in items:
send({"type": "conversation.item.create", "item": item})
send({"type": "response.create"})
while True:
event = json.loads(ws.recv(timeout=self.timeout))
if event["type"] == "error":
raise RuntimeError(f"Realtime API error: {event.get('error')}")
if event["type"] == "response.done":
response = event["response"]
break
if response.get("status") not in (None, "completed"):
raise RuntimeError(f"Realtime response {response.get('status')}: "
f"{response.get('status_details')}")
text, tool_calls = [], []
for item in response.get("output", []):
if item.get("type") == "message":
text += [part.get("text", "") for part in item.get("content", [])
if part.get("type") in ("output_text", "text")]
elif item.get("type") == "function_call":
tool_calls.append({"id": item.get("call_id") or f"call_{uuid.uuid4().hex[:8]}",
"name": item["name"], "args": json.loads(item.get("arguments") or "{}")})
message = AIMessage(content="".join(text), tool_calls=tool_calls,
response_metadata={"model": self.model, "usage": response.get("usage")})
return ChatResult(generations=[ChatGeneration(message=message)])
+1
View File
@@ -13,3 +13,4 @@ uvicorn[standard]
rank_bm25
snowballstemmer
python-dotenv
websockets
+27 -17
View File
@@ -18,7 +18,10 @@ from personas import DEFAULT_PERSONA, PERSONAS, PUBLIC_FIELDS
load_dotenv()
OLLAMA_MODEL = "gemma4:12b"
OPENAI_MODEL = "gpt-6-luna"
OPENAI_MODEL = "gpt-6.1-sol"
REALTIME_MODEL = "gpt-realtime-2.1-mini" # OpenAI Realtime API, text only (realtime_chat.py)
# Model name per provider, for the console trace
MODEL_NAMES = {"ollama": OLLAMA_MODEL, "openai": OPENAI_MODEL, "realtime": REALTIME_MODEL}
# One RAG instance per (LLM provider, vector store), for on-the-fly provider switching.
# Each persona names its vector store in personas.py; personas sharing one share the instance.
rags = {}
@@ -30,16 +33,23 @@ for vectorstore in sorted({p["vectorstore"] for p in PERSONAS.values()}):
openai_model=OPENAI_MODEL,
)
if os.environ.get("OPENAI_API_KEY"):
try:
rags[("openai", vectorstore)] = LocalRAG(
vectorstore_path=vectorstore,
llm_provider="openai",
ollama_model=OLLAMA_MODEL,
openai_model=OPENAI_MODEL,
)
except Exception as e:
print(f"OpenAI RAG not available for {vectorstore}: {e}")
openai_available = any(provider == "openai" for provider, _ in rags)
for provider in ("openai", "realtime"):
try:
rags[(provider, vectorstore)] = LocalRAG(
vectorstore_path=vectorstore,
llm_provider=provider,
ollama_model=OLLAMA_MODEL,
openai_model=OPENAI_MODEL,
realtime_model=REALTIME_MODEL,
)
except Exception as e:
print(f"{provider} RAG not available for {vectorstore}: {e}")
available = {provider for provider, _ in rags}
openai_available = "openai" in available
# Load the reranker and build the BM25 indexes at startup (shared copies) so the first search
# is not delayed
for rag in rags.values():
rag.preload()
app = FastAPI(title="Local RAG Chat", version="1.0.0")
# Images for the chat GUI (researcher portrait, avatar)
@@ -54,7 +64,7 @@ class ChatMessage(BaseModel):
class ChatRequest(BaseModel):
message: str
history: list[ChatMessage] = [] # previous turns for conversation context
llm_provider: str = "ollama" # "ollama" | "openai"
llm_provider: str = "ollama" # "ollama" | "openai" | "realtime"
persona: str = DEFAULT_PERSONA # key in personas.PERSONAS
@@ -83,7 +93,7 @@ def chat_view():
def _get_rag(provider: str, persona: str):
"""Return the RAG instance for the provider and the persona's vector store.
Fall back to Ollama if OpenAI is unavailable."""
Fall back to Ollama if the provider is unavailable."""
vectorstore = PERSONAS[persona]["vectorstore"]
return rags.get((provider, vectorstore)) or rags[("ollama", vectorstore)]
@@ -93,7 +103,7 @@ def chat(request: ChatRequest):
"""Handle a chat message and return the RAG answer."""
if not request.message or not request.message.strip():
return ChatResponse(answer="", error="Message cannot be empty")
if request.llm_provider == "openai" and not openai_available:
if request.llm_provider in ("openai", "realtime") and request.llm_provider not in available:
return ChatResponse(answer="", error="OpenAI not configured. Set OPENAI_API_KEY.")
if request.persona not in PERSONAS:
return ChatResponse(answer="", error=f"Unknown persona: {request.persona}")
@@ -120,8 +130,8 @@ def chat(request: ChatRequest):
print(f" [{i + 1}] {chunk.get('source', '')} p.{chunk.get('page', '?')} s={chunk.get('score')} | {preview!r}")
else:
print(f"\n[RAG] No corpus search for this message")
provider_label = "OpenAI" if request.llm_provider == "openai" else "Ollama"
model_name = OPENAI_MODEL if request.llm_provider == "openai" else OLLAMA_MODEL
provider_label = {"openai": "OpenAI", "realtime": "OpenAI Realtime"}.get(request.llm_provider, "Ollama")
model_name = MODEL_NAMES.get(request.llm_provider, OLLAMA_MODEL)
print(f"[RAG] LLM response ({provider_label} / {model_name}, {request.persona}):\n{answer}")
return ChatResponse(answer=answer, retrieved=retrieved, searches=searches)
@@ -139,7 +149,7 @@ def health():
@app.get("/api/providers")
def providers():
"""Return which LLM providers are available."""
return {"ollama": True, "openai": openai_available}
return {"ollama": True, "openai": openai_available, "realtime": "realtime" in available}
@app.get("/api/personas")
+7 -5
View File
@@ -474,7 +474,8 @@
Modell
<select id="provider">
<option value="ollama">Ollama</option>
<option value="openai">ChatGPT</option>
<option value="openai" selected>ChatGPT</option>
<option value="realtime">Realtime Mini</option>
</select>
</label>
</div>
@@ -559,12 +560,13 @@
try {
const res = await fetch('/api/providers');
const data = await res.json();
if (!data.openai) {
const opt = providerEl.querySelector('option[value="openai"]');
for (const id of ['openai', 'realtime']) {
if (data[id]) continue;
const opt = providerEl.querySelector('option[value="' + id + '"]');
opt.disabled = true;
opt.textContent = 'ChatGPT (aus)';
opt.textContent += ' (aus)';
opt.title = 'OpenAI ist nicht eingerichtet (OPENAI_API_KEY fehlt)';
if (providerEl.value === 'openai') providerEl.value = 'ollama';
if (providerEl.value === id) providerEl.value = 'ollama';
}
} catch (_) {}
})();