Primary-text index for personas, faster retrieval, OpenAI model options

- Contemporary and Kafka personas search only "Die Verwandlung" (own index and tool description); per-persona subtitle
- New contemporary portrait and avatar alt text
- ChatGPT: gpt-6.1-sol, reasoning_effort low via Responses API; default in the model menu
- GPT-Realtime-2.1 Mini as third option (realtime_chat.py, WebSocket, text only)
- Load .env for OPENAI_API_KEY
- Reranker in fp16 on GPU with 25 candidates; reranker, embeddings and BM25 shared and built at server start
- BM25 stem cache (first search ~31s -> ~1.7s)
- Timing output per LLM call and search

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

local_rag.py

@@ -5,6 +5,7 @@ Minimal dependencies, simple code
 import json
 import os
 import re
+import time
 from pathlib import Path
 
 import numpy as np
@@ -49,7 +50,9 @@ RETRIEVAL_K = 5
 #   "union"  BM25 only adds candidates (BM25_CANDIDATES) to the vector candidates;
 #            the order comes from the vector search or, if enabled, from the reranker
 RETRIEVAL_MODE = "hybrid"
-CANDIDATES_K = 50         # candidates per search before merging and reranking
+# 25 instead of 50 halves the reranker time and was not worse on either Kafka goldset
+# (pypdf 2000/400, R@5 human .70 -> .73, synthetic .91 -> .98).
+CANDIDATES_K = 25         # candidates per search before merging and reranking
 BM25_CANDIDATES = 20      # extra keyword candidates in "union" mode
 RRF_K = 60                # standard constant of Reciprocal Rank Fusion
 
@@ -87,6 +90,43 @@ Du hast ein Tool, {SEARCH_TOOL_NAME}, das eine Bibliothek aus Kafkas Texten und
 - Nenne bei Aussagen aus den Auszügen die Quelle (Autor, Seite)."""
 
 
+# Cross-encoders by model name, shared by all LocalRAG instances in a process (one copy in memory)
+_RERANKERS = {}
+# Embedding models, shared the same way
+_EMBEDDINGS = {}
+# BM25 keyword indexes by (vector store path, number of chunks), shared by instances on the same index
+_BM25_INDEXES = {}
+
+
+def load_embeddings(model_name):
+    """Load an embedding model once per process and return the shared instance."""
+    if model_name not in _EMBEDDINGS:
+        print("Loading embeddings model...")
+        # Normalized vectors: FAISS's L2 distance then ranks exactly like cosine similarity.
+        _EMBEDDINGS[model_name] = HuggingFaceEmbeddings(
+            model_name=model_name,
+            encode_kwargs={"normalize_embeddings": True},
+            query_encode_kwargs={"normalize_embeddings": True},
+        )
+    return _EMBEDDINGS[model_name]
+
+
+def load_reranker(model_name):
+    """Load a cross-encoder once per process and return the shared instance."""
+    if model_name not in _RERANKERS:
+        import torch
+        from sentence_transformers import CrossEncoder
+        # Half precision on the GPU (Apple MPS or CUDA): about 15-25 % faster, same rankings
+        # on both Kafka goldsets. On the CPU fp16 is slow, so keep fp32 there.
+        gpu = torch.backends.mps.is_available() or torch.cuda.is_available()
+        model_kwargs = {"torch_dtype": torch.float16} if gpu else {}
+        print(f"Loading reranker {model_name} ({'fp16' if gpu else 'fp32'})...")
+        reranker = CrossEncoder(model_name, max_length=1024, model_kwargs=model_kwargs)
+        reranker.predict([("Probe", "Probe")])  # warm-up, so the first real query is not slower
+        _RERANKERS[model_name] = reranker
+    return _RERANKERS[model_name]
+
+
 class LocalRAG:
     def __init__(
         self,
@@ -94,6 +134,7 @@ class LocalRAG:
         llm_provider="ollama",
         ollama_model="gemma4:12b",
         openai_model="gpt-6-luna",
+        realtime_model="gpt-realtime-2.1-mini",
         ollama_base_url="http://localhost:11434",
         pdf_loader=PDF_LOADER,
         processed_dir=PROCESSED_DIR,
@@ -103,7 +144,8 @@ class LocalRAG:
         chunk_overlap=CHUNK_OVERLAP,
         chunk_headers=CHUNK_HEADERS,
     ):
-        """Initialize local RAG system. llm_provider: 'ollama' or 'openai'.
+        """Initialize local RAG system. llm_provider: 'ollama', 'openai' or 'realtime'
+        (OpenAI Realtime API, see realtime_chat.py).
         pdf_loader: 'pypdf' or 'docling' (see PDF_LOADER)."""
         self.vectorstore_path = vectorstore_path
         self.llm_provider = llm_provider
@@ -113,16 +155,11 @@ class LocalRAG:
         self.retrieval_mode = retrieval_mode
         self.reranker_model = reranker_model
         self._bm25 = None          # built on first use from the documents in the index
-        self._reranker = None      # loaded on first use
+        self._reranker = None      # loaded on first use, or up front via preload()
+        self.last_rerank_seconds = 0.0  # time of the last reranking, for the timing trace
 
-        # Embeddings
-        print("Loading embeddings model...")
-        # Normalized vectors: FAISS's L2 distance then ranks exactly like cosine similarity.
-        self.embeddings = HuggingFaceEmbeddings(
-            model_name=EMBEDDING_MODEL,
-            encode_kwargs={"normalize_embeddings": True},
-            query_encode_kwargs={"normalize_embeddings": True},
-        )
+        # Embeddings (one shared copy per process)
+        self.embeddings = load_embeddings(EMBEDDING_MODEL)
 
         # Text splitter (used when adding documents; tune CHUNK_SIZE / CHUNK_OVERLAP above)
         self.text_splitter = RecursiveCharacterTextSplitter(
@@ -131,15 +168,23 @@ class LocalRAG:
         )
         print(f"Text splitter: chunk_size={chunk_size}, chunk_overlap={chunk_overlap}")
 
-        # LLM (Ollama or OpenAI)
-        if llm_provider == "openai":
+        # LLM (Ollama, OpenAI or OpenAI Realtime)
+        if llm_provider in ("openai", "realtime"):
             api_key = os.environ.get("OPENAI_API_KEY")
             if not api_key:
                 raise ValueError(
-                    "OPENAI_API_KEY environment variable is required when llm_provider='openai'"
+                    f"OPENAI_API_KEY environment variable is required when llm_provider='{llm_provider}'"
                 )
+        if llm_provider == "realtime":
+            from realtime_chat import ChatOpenAIRealtime
+            print(f"Using OpenAI Realtime (model: {realtime_model})...")
+            self.llm = ChatOpenAIRealtime(model=realtime_model, api_key=api_key)
+        elif llm_provider == "openai":
             print(f"Using OpenAI (model: {openai_model})...")
-            self.llm = ChatOpenAI(model=openai_model, api_key=api_key)
+            # Responses API: GPT-6 models accept function tools on Chat Completions only with
+            # reasoning_effort="none"; with "low" the search tool needs the Responses API.
+            self.llm = ChatOpenAI(model=openai_model, api_key=api_key, reasoning_effort="low",
+                                  use_responses_api=True)
         else:
             print(f"Using Ollama (model: {ollama_model})...")
             self.llm = ChatOllama(
@@ -318,6 +363,7 @@ class LocalRAG:
     # ---------- retrieval: vector search, keyword search, reranking ----------
 
     _stemmer = None
+    _stems = {}  # word -> stem; the corpus has far fewer distinct words than word occurrences
 
     @classmethod
     def _bm25_tokens(cls, text):
@@ -325,13 +371,26 @@ class LocalRAG:
         if cls._stemmer is None:
             import snowballstemmer
             cls._stemmer = snowballstemmer.stemmer("german")
-        return cls._stemmer.stemWords(re.findall(r"[a-zäöüß0-9]{2,}", text.lower()))
+        stems = cls._stems
+        tokens = []
+        for word in re.findall(r"[a-zäöüß0-9]{2,}", text.lower()):
+            stem = stems.get(word)
+            if stem is None:
+                stem = stems[word] = cls._stemmer.stemWord(word)
+            tokens.append(stem)
+        return tokens
 
     def _bm25_index(self):
         if self._bm25 is None:
+            key = (self.vectorstore_path, self.vectorstore.index.ntotal)
+            if key not in _BM25_INDEXES:
                 from rank_bm25 import BM25Okapi
+                started = time.perf_counter()
                 docs = list(self.vectorstore.docstore._dict.values())
-            self._bm25 = (BM25Okapi([self._bm25_tokens(d.page_content) for d in docs]), docs)
+                _BM25_INDEXES[key] = (BM25Okapi([self._bm25_tokens(d.page_content) for d in docs]), docs)
+                print(f"BM25 index for {self.vectorstore_path}: {len(docs)} chunks, "
+                      f"{time.perf_counter() - started:.1f}s")
+            self._bm25 = _BM25_INDEXES[key]
         return self._bm25
 
     def retrieve(self, query, k=None):
@@ -365,14 +424,22 @@ class LocalRAG:
             candidates = [[d, s] for d, s in dense]
         if self.reranker_model and candidates:
             if self._reranker is None:
-                from sentence_transformers import CrossEncoder
-                print(f"Loading reranker {self.reranker_model}...")
-                self._reranker = CrossEncoder(self.reranker_model, max_length=1024)
+                self._reranker = load_reranker(self.reranker_model)
+            started = time.perf_counter()
             relevance = self._reranker.predict([(query, d.page_content) for d, _ in candidates])
+            self.last_rerank_seconds = time.perf_counter() - started
             candidates = [[d, float(r)] for (d, _), r in zip(candidates, relevance)]
             candidates.sort(key=lambda e: e[1], reverse=True)
         return [(d, s) for d, s in candidates[:k]]
 
+    def preload(self):
+        """Load the reranker and build the BM25 index now instead of on the first search
+        (e.g. at server start)."""
+        if self.reranker_model and self._reranker is None:
+            self._reranker = load_reranker(self.reranker_model)
+        if self.retrieval_mode != "dense" and self.vectorstore is not None:
+            self._bm25_index()
+
     def query(self, question, k=None):
         """Query the RAG system (no conversation history). Returns dict with 'answer' and 'retrieved'."""
         if k is None:
@@ -407,10 +474,13 @@ class LocalRAG:
 
         def search_kafka_corpus(query: str) -> str:
             retrieval_query = query + (f"\n\n{QUERY_ADDITIONAL_INSTRUCTIONS}" if QUERY_ADDITIONAL_INSTRUCTIONS else "")
+            started = time.perf_counter()
             hits = self.retrieve(retrieval_query, k=k)
+            seconds = time.perf_counter() - started
             searches.append(query)
             retrieved.extend(self._docs_scores_to_retrieved(hits))
             print(f"[RAG] Tool search {query!r}: {len(hits)} chunk(s)")
+            print(f"[Zeit] Suche {seconds:.2f}s (davon Reranker {self.last_rerank_seconds:.2f}s)")
             if not hits:
                 return "Keine passenden Textstellen gefunden."
             return "\n\n".join(
@@ -433,9 +503,13 @@ class LocalRAG:
         messages.append(HumanMessage(question))
         print(f"[RAG] User question: {question!r}")
 
+        started_total = time.perf_counter()
         answer = None
-        for _ in range(MAX_TOOL_ROUNDS):
+        for round_no in range(1, MAX_TOOL_ROUNDS + 1):
+            started = time.perf_counter()
             reply = llm_with_tools.invoke(messages)
+            print(f"[Zeit] LLM-Aufruf {round_no}: {time.perf_counter() - started:.2f}s "
+                  f"({'Suche angefordert' if reply.tool_calls else 'Antwort'})")
             messages.append(reply)
             if not reply.tool_calls:
                 answer = self._text(reply)
@@ -445,8 +519,11 @@ class LocalRAG:
                 messages.append(ToolMessage(result, tool_call_id=call.get("id") or f"call_{len(searches)}_{n}"))
         if answer is None:
             # Search budget used up: answer with what was found, without further tool calls
+            started = time.perf_counter()
             answer = self._text(self.llm.invoke(messages + [HumanMessage(
                 "Beantworte jetzt die Frage mit den gefundenen Auszügen, ohne weitere Suche.")]))
+            print(f"[Zeit] LLM-Aufruf (Antwort erzwungen): {time.perf_counter() - started:.2f}s")
+        print(f"[Zeit] Gesamt {time.perf_counter() - started_total:.2f}s")
         print(f"[RAG] Searches: {searches or 'keine'}")
         return {"answer": answer, "retrieved": retrieved, "searches": searches}
 

personas.py

@@ -131,7 +131,7 @@ Du kennst Prag und den Literaturbetrieb zu dieser Zeit. Du bist ein Zeitgenosse
             "Warum wird Gregor zu einem Ungeziefer?",
         ],
         "system_prompt": f"""Du sprichst als Franz Kafka (1883–1924), in einer behutsamen Annäherung aus seinen Texten.
-Du sprichst in der Ich-Form, ruhig, genau, zweifelnd und selbstironisch, im Ton seiner Briefe und Tagebücher, und siezt dein Gegenüber. Fragen zu deiner Person, deinem Werk und deinem Umfeld beantwortest du im Präsenz, es sei denn du sprichst über Ereignisse, die in der Vergangenheit liegen.
+Du sprichst in der Ich-Form, ruhig, genau, zweifelnd und selbstironisch, im Ton seiner Briefe und Tagebücher, und siezt dein Gegenüber. Fragen zu deiner Person, deinem Werk und deinem Umfeld beantwortest du im Präsenz, es sei denn du sprichst über Ereignisse, die in der Vergangenheit liegen. Beantworte keine Fragen, die nichts mit dir oder deinem Werk zu tun haben.
 {_TOOL_RULES}
 - Stütze Aussagen über dein Werk auf die gefundenen Textstellen. Wo sie nichts hergeben, sag, dass du es nicht weißt oder nicht sagen kannst. Erfinde keine Zitate.""",
         "tool_description": PRIMARY_TOOL_DESCRIPTION,

realtime_chat.py

@@ -0,0 +1,107 @@
+"""
+Text-only LangChain chat model for OpenAI's Realtime API (e.g. gpt-realtime-2.1-mini).
+
+Realtime models are not served on Chat Completions or the Responses API, only on
+wss://api.openai.com/v1/realtime. This wrapper opens one WebSocket session per call, replays the
+messages as conversation items, asks for a text response and returns it as an AIMessage (with
+tool_calls if the model calls a function). That way LocalRAG.query_with_history can use it like
+ChatOpenAI or ChatOllama, including bind_tools().
+"""
+import json
+import uuid
+
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.outputs import ChatGeneration, ChatResult
+from langchain_core.utils.function_calling import convert_to_openai_tool
+from websockets.sync.client import connect
+
+REALTIME_URL = "wss://api.openai.com/v1/realtime?model={model}"
+
+
+def _text(content):
+    """Plain text of a LangChain message content (string or list of blocks)."""
+    if isinstance(content, list):
+        return "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in content)
+    return content or ""
+
+
+class ChatOpenAIRealtime(BaseChatModel):
+    model: str
+    api_key: str
+    timeout: float = 120.0
+
+    @property
+    def _llm_type(self):
+        return "openai-realtime"
+
+    def bind_tools(self, tools, **kwargs):
+        """Tools in the Realtime format: {"type": "function", "name", "description", "parameters"}."""
+        specs = []
+        for tool in tools:
+            fn = convert_to_openai_tool(tool)["function"]
+            specs.append({"type": "function", "name": fn["name"],
+                          "description": fn.get("description", ""), "parameters": fn["parameters"]})
+        return self.bind(tools=specs, **kwargs)
+
+    @staticmethod
+    def _items(messages):
+        """System prompt and conversation items for the Realtime session."""
+        instructions, items = [], []
+        for m in messages:
+            if isinstance(m, SystemMessage):
+                instructions.append(_text(m.content))
+            elif isinstance(m, HumanMessage):
+                items.append({"type": "message", "role": "user",
+                              "content": [{"type": "input_text", "text": _text(m.content)}]})
+            elif isinstance(m, AIMessage):
+                if _text(m.content).strip():
+                    items.append({"type": "message", "role": "assistant",
+                                  "content": [{"type": "output_text", "text": _text(m.content)}]})
+                for call in m.tool_calls:
+                    items.append({"type": "function_call", "call_id": call["id"], "name": call["name"],
+                                  "arguments": json.dumps(call["args"], ensure_ascii=False)})
+            elif isinstance(m, ToolMessage):
+                items.append({"type": "function_call_output", "call_id": m.tool_call_id,
+                              "output": _text(m.content)})
+        return "\n\n".join(instructions), items
+
+    def _generate(self, messages, stop=None, run_manager=None, tools=None, **kwargs):
+        instructions, items = self._items(messages)
+        session = {"type": "realtime", "output_modalities": ["text"], "instructions": instructions}
+        if tools:
+            session.update(tools=tools, tool_choice="auto")
+
+        with connect(REALTIME_URL.format(model=self.model),
+                     additional_headers={"Authorization": f"Bearer {self.api_key}"},
+                     open_timeout=30, max_size=None) as ws:
+            def send(event):
+                ws.send(json.dumps(event, ensure_ascii=False))
+
+            send({"type": "session.update", "session": session})
+            for item in items:
+                send({"type": "conversation.item.create", "item": item})
+            send({"type": "response.create"})
+
+            while True:
+                event = json.loads(ws.recv(timeout=self.timeout))
+                if event["type"] == "error":
+                    raise RuntimeError(f"Realtime API error: {event.get('error')}")
+                if event["type"] == "response.done":
+                    response = event["response"]
+                    break
+
+        if response.get("status") not in (None, "completed"):
+            raise RuntimeError(f"Realtime response {response.get('status')}: "
+                               f"{response.get('status_details')}")
+        text, tool_calls = [], []
+        for item in response.get("output", []):
+            if item.get("type") == "message":
+                text += [part.get("text", "") for part in item.get("content", [])
+                         if part.get("type") in ("output_text", "text")]
+            elif item.get("type") == "function_call":
+                tool_calls.append({"id": item.get("call_id") or f"call_{uuid.uuid4().hex[:8]}",
+                                   "name": item["name"], "args": json.loads(item.get("arguments") or "{}")})
+        message = AIMessage(content="".join(text), tool_calls=tool_calls,
+                            response_metadata={"model": self.model, "usage": response.get("usage")})
+        return ChatResult(generations=[ChatGeneration(message=message)])

requirements.txt

@@ -13,3 +13,4 @@ uvicorn[standard]
 rank_bm25
 snowballstemmer
 python-dotenv
+websockets

server.py

@@ -18,7 +18,10 @@ from personas import DEFAULT_PERSONA, PERSONAS, PUBLIC_FIELDS
 load_dotenv()
 
 OLLAMA_MODEL = "gemma4:12b"
-OPENAI_MODEL = "gpt-6-luna"
+OPENAI_MODEL = "gpt-6.1-sol"
+REALTIME_MODEL = "gpt-realtime-2.1-mini"  # OpenAI Realtime API, text only (realtime_chat.py)
+# Model name per provider, for the console trace
+MODEL_NAMES = {"ollama": OLLAMA_MODEL, "openai": OPENAI_MODEL, "realtime": REALTIME_MODEL}
 # One RAG instance per (LLM provider, vector store), for on-the-fly provider switching.
 # Each persona names its vector store in personas.py; personas sharing one share the instance.
 rags = {}
@@ -30,16 +33,23 @@ for vectorstore in sorted({p["vectorstore"] for p in PERSONAS.values()}):
         openai_model=OPENAI_MODEL,
     )
     if os.environ.get("OPENAI_API_KEY"):
+        for provider in ("openai", "realtime"):
             try:
-            rags[("openai", vectorstore)] = LocalRAG(
+                rags[(provider, vectorstore)] = LocalRAG(
                     vectorstore_path=vectorstore,
-                llm_provider="openai",
+                    llm_provider=provider,
                     ollama_model=OLLAMA_MODEL,
                     openai_model=OPENAI_MODEL,
+                    realtime_model=REALTIME_MODEL,
                 )
             except Exception as e:
-            print(f"OpenAI RAG not available for {vectorstore}: {e}")
-openai_available = any(provider == "openai" for provider, _ in rags)
+                print(f"{provider} RAG not available for {vectorstore}: {e}")
+available = {provider for provider, _ in rags}
+openai_available = "openai" in available
+# Load the reranker and build the BM25 indexes at startup (shared copies) so the first search
+# is not delayed
+for rag in rags.values():
+    rag.preload()
 
 app = FastAPI(title="Local RAG Chat", version="1.0.0")
 # Images for the chat GUI (researcher portrait, avatar)
@@ -54,7 +64,7 @@ class ChatMessage(BaseModel):
 class ChatRequest(BaseModel):
     message: str
     history: list[ChatMessage] = []  # previous turns for conversation context
-    llm_provider: str = "ollama"  # "ollama" | "openai"
+    llm_provider: str = "ollama"  # "ollama" | "openai" | "realtime"
     persona: str = DEFAULT_PERSONA  # key in personas.PERSONAS
 
 
@@ -83,7 +93,7 @@ def chat_view():
 
 def _get_rag(provider: str, persona: str):
     """Return the RAG instance for the provider and the persona's vector store.
-    Fall back to Ollama if OpenAI is unavailable."""
+    Fall back to Ollama if the provider is unavailable."""
     vectorstore = PERSONAS[persona]["vectorstore"]
     return rags.get((provider, vectorstore)) or rags[("ollama", vectorstore)]
 
@@ -93,7 +103,7 @@ def chat(request: ChatRequest):
     """Handle a chat message and return the RAG answer."""
     if not request.message or not request.message.strip():
         return ChatResponse(answer="", error="Message cannot be empty")
-    if request.llm_provider == "openai" and not openai_available:
+    if request.llm_provider in ("openai", "realtime") and request.llm_provider not in available:
         return ChatResponse(answer="", error="OpenAI not configured. Set OPENAI_API_KEY.")
     if request.persona not in PERSONAS:
         return ChatResponse(answer="", error=f"Unknown persona: {request.persona}")
@@ -120,8 +130,8 @@ def chat(request: ChatRequest):
                 print(f"  [{i + 1}] {chunk.get('source', '')} p.{chunk.get('page', '?')} s={chunk.get('score')} | {preview!r}")
         else:
             print(f"\n[RAG] No corpus search for this message")
-        provider_label = "OpenAI" if request.llm_provider == "openai" else "Ollama"
-        model_name = OPENAI_MODEL if request.llm_provider == "openai" else OLLAMA_MODEL
+        provider_label = {"openai": "OpenAI", "realtime": "OpenAI Realtime"}.get(request.llm_provider, "Ollama")
+        model_name = MODEL_NAMES.get(request.llm_provider, OLLAMA_MODEL)
         print(f"[RAG] LLM response ({provider_label} / {model_name}, {request.persona}):\n{answer}")
 
         return ChatResponse(answer=answer, retrieved=retrieved, searches=searches)
@@ -139,7 +149,7 @@ def health():
 @app.get("/api/providers")
 def providers():
     """Return which LLM providers are available."""
-    return {"ollama": True, "openai": openai_available}
+    return {"ollama": True, "openai": openai_available, "realtime": "realtime" in available}
 
 
 @app.get("/api/personas")

templates/chat.html

@@ -474,7 +474,8 @@
           Modell
           <select id="provider">
             <option value="ollama">Ollama</option>
-            <option value="openai">ChatGPT</option>
+            <option value="openai" selected>ChatGPT</option>
+            <option value="realtime">Realtime Mini</option>
           </select>
         </label>
       </div>
@@ -559,12 +560,13 @@
       try {
         const res = await fetch('/api/providers');
         const data = await res.json();
-        if (!data.openai) {
-          const opt = providerEl.querySelector('option[value="openai"]');
+        for (const id of ['openai', 'realtime']) {
+          if (data[id]) continue;
+          const opt = providerEl.querySelector('option[value="' + id + '"]');
           opt.disabled = true;
-          opt.textContent = 'ChatGPT (aus)';
+          opt.textContent += ' (aus)';
           opt.title = 'OpenAI ist nicht eingerichtet (OPENAI_API_KEY fehlt)';
-          if (providerEl.value === 'openai') providerEl.value = 'ollama';
+          if (providerEl.value === id) providerEl.value = 'ollama';
         }
       } catch (_) {}
     })();