Primary-text index for personas, faster retrieval, OpenAI model options

- Contemporary and Kafka personas search only "Die Verwandlung" (own index and tool description); per-persona subtitle
- New contemporary portrait and avatar alt text
- ChatGPT: gpt-6.1-sol, reasoning_effort low via Responses API; default in the model menu
- GPT-Realtime-2.1 Mini as third option (realtime_chat.py, WebSocket, text only)
- Load .env for OPENAI_API_KEY
- Reranker in fp16 on GPU with 25 candidates; reranker, embeddings and BM25 shared and built at server start
- BM25 stem cache (first search ~31s -> ~1.7s)
- Timing output per LLM call and search

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-26 07:04:59 +01:00
co-authored by Claude Opus 5.5
parent f68d7a6e25
commit e4b7f0ce34
6 changed files with 243 additions and 46 deletions
+100 -23
View File
@@ -5,6 +5,7 @@ Minimal dependencies, simple code
import json import json
import os import os
import re import re
import time
from pathlib import Path from pathlib import Path
import numpy as np import numpy as np
@@ -49,7 +50,9 @@ RETRIEVAL_K = 5
# "union" BM25 only adds candidates (BM25_CANDIDATES) to the vector candidates; # "union" BM25 only adds candidates (BM25_CANDIDATES) to the vector candidates;
# the order comes from the vector search or, if enabled, from the reranker # the order comes from the vector search or, if enabled, from the reranker
RETRIEVAL_MODE = "hybrid" RETRIEVAL_MODE = "hybrid"
CANDIDATES_K = 50 # candidates per search before merging and reranking # 25 instead of 50 halves the reranker time and was not worse on either Kafka goldset
# (pypdf 2000/400, R@5 human .70 -> .73, synthetic .91 -> .98).
CANDIDATES_K = 25 # candidates per search before merging and reranking
BM25_CANDIDATES = 20 # extra keyword candidates in "union" mode BM25_CANDIDATES = 20 # extra keyword candidates in "union" mode
RRF_K = 60 # standard constant of Reciprocal Rank Fusion RRF_K = 60 # standard constant of Reciprocal Rank Fusion
@@ -87,6 +90,43 @@ Du hast ein Tool, {SEARCH_TOOL_NAME}, das eine Bibliothek aus Kafkas Texten und
- Nenne bei Aussagen aus den Auszügen die Quelle (Autor, Seite).""" - Nenne bei Aussagen aus den Auszügen die Quelle (Autor, Seite)."""
# Cross-encoders by model name, shared by all LocalRAG instances in a process (one copy in memory)
_RERANKERS = {}
# Embedding models, shared the same way
_EMBEDDINGS = {}
# BM25 keyword indexes by (vector store path, number of chunks), shared by instances on the same index
_BM25_INDEXES = {}
def load_embeddings(model_name):
"""Load an embedding model once per process and return the shared instance."""
if model_name not in _EMBEDDINGS:
print("Loading embeddings model...")
# Normalized vectors: FAISS's L2 distance then ranks exactly like cosine similarity.
_EMBEDDINGS[model_name] = HuggingFaceEmbeddings(
model_name=model_name,
encode_kwargs={"normalize_embeddings": True},
query_encode_kwargs={"normalize_embeddings": True},
)
return _EMBEDDINGS[model_name]
def load_reranker(model_name):
"""Load a cross-encoder once per process and return the shared instance."""
if model_name not in _RERANKERS:
import torch
from sentence_transformers import CrossEncoder
# Half precision on the GPU (Apple MPS or CUDA): about 15-25 % faster, same rankings
# on both Kafka goldsets. On the CPU fp16 is slow, so keep fp32 there.
gpu = torch.backends.mps.is_available() or torch.cuda.is_available()
model_kwargs = {"torch_dtype": torch.float16} if gpu else {}
print(f"Loading reranker {model_name} ({'fp16' if gpu else 'fp32'})...")
reranker = CrossEncoder(model_name, max_length=1024, model_kwargs=model_kwargs)
reranker.predict([("Probe", "Probe")]) # warm-up, so the first real query is not slower
_RERANKERS[model_name] = reranker
return _RERANKERS[model_name]
class LocalRAG: class LocalRAG:
def __init__( def __init__(
self, self,
@@ -94,6 +134,7 @@ class LocalRAG:
llm_provider="ollama", llm_provider="ollama",
ollama_model="gemma4:12b", ollama_model="gemma4:12b",
openai_model="gpt-6-luna", openai_model="gpt-6-luna",
realtime_model="gpt-realtime-2.1-mini",
ollama_base_url="http://localhost:11434", ollama_base_url="http://localhost:11434",
pdf_loader=PDF_LOADER, pdf_loader=PDF_LOADER,
processed_dir=PROCESSED_DIR, processed_dir=PROCESSED_DIR,
@@ -103,7 +144,8 @@ class LocalRAG:
chunk_overlap=CHUNK_OVERLAP, chunk_overlap=CHUNK_OVERLAP,
chunk_headers=CHUNK_HEADERS, chunk_headers=CHUNK_HEADERS,
): ):
"""Initialize local RAG system. llm_provider: 'ollama' or 'openai'. """Initialize local RAG system. llm_provider: 'ollama', 'openai' or 'realtime'
(OpenAI Realtime API, see realtime_chat.py).
pdf_loader: 'pypdf' or 'docling' (see PDF_LOADER).""" pdf_loader: 'pypdf' or 'docling' (see PDF_LOADER)."""
self.vectorstore_path = vectorstore_path self.vectorstore_path = vectorstore_path
self.llm_provider = llm_provider self.llm_provider = llm_provider
@@ -113,16 +155,11 @@ class LocalRAG:
self.retrieval_mode = retrieval_mode self.retrieval_mode = retrieval_mode
self.reranker_model = reranker_model self.reranker_model = reranker_model
self._bm25 = None # built on first use from the documents in the index self._bm25 = None # built on first use from the documents in the index
self._reranker = None # loaded on first use self._reranker = None # loaded on first use, or up front via preload()
self.last_rerank_seconds = 0.0 # time of the last reranking, for the timing trace
# Embeddings # Embeddings (one shared copy per process)
print("Loading embeddings model...") self.embeddings = load_embeddings(EMBEDDING_MODEL)
# Normalized vectors: FAISS's L2 distance then ranks exactly like cosine similarity.
self.embeddings = HuggingFaceEmbeddings(
model_name=EMBEDDING_MODEL,
encode_kwargs={"normalize_embeddings": True},
query_encode_kwargs={"normalize_embeddings": True},
)
# Text splitter (used when adding documents; tune CHUNK_SIZE / CHUNK_OVERLAP above) # Text splitter (used when adding documents; tune CHUNK_SIZE / CHUNK_OVERLAP above)
self.text_splitter = RecursiveCharacterTextSplitter( self.text_splitter = RecursiveCharacterTextSplitter(
@@ -131,15 +168,23 @@ class LocalRAG:
) )
print(f"Text splitter: chunk_size={chunk_size}, chunk_overlap={chunk_overlap}") print(f"Text splitter: chunk_size={chunk_size}, chunk_overlap={chunk_overlap}")
# LLM (Ollama or OpenAI) # LLM (Ollama, OpenAI or OpenAI Realtime)
if llm_provider == "openai": if llm_provider in ("openai", "realtime"):
api_key = os.environ.get("OPENAI_API_KEY") api_key = os.environ.get("OPENAI_API_KEY")
if not api_key: if not api_key:
raise ValueError( raise ValueError(
"OPENAI_API_KEY environment variable is required when llm_provider='openai'" f"OPENAI_API_KEY environment variable is required when llm_provider='{llm_provider}'"
) )
if llm_provider == "realtime":
from realtime_chat import ChatOpenAIRealtime
print(f"Using OpenAI Realtime (model: {realtime_model})...")
self.llm = ChatOpenAIRealtime(model=realtime_model, api_key=api_key)
elif llm_provider == "openai":
print(f"Using OpenAI (model: {openai_model})...") print(f"Using OpenAI (model: {openai_model})...")
self.llm = ChatOpenAI(model=openai_model, api_key=api_key) # Responses API: GPT-6 models accept function tools on Chat Completions only with
# reasoning_effort="none"; with "low" the search tool needs the Responses API.
self.llm = ChatOpenAI(model=openai_model, api_key=api_key, reasoning_effort="low",
use_responses_api=True)
else: else:
print(f"Using Ollama (model: {ollama_model})...") print(f"Using Ollama (model: {ollama_model})...")
self.llm = ChatOllama( self.llm = ChatOllama(
@@ -318,6 +363,7 @@ class LocalRAG:
# ---------- retrieval: vector search, keyword search, reranking ---------- # ---------- retrieval: vector search, keyword search, reranking ----------
_stemmer = None _stemmer = None
_stems = {} # word -> stem; the corpus has far fewer distinct words than word occurrences
@classmethod @classmethod
def _bm25_tokens(cls, text): def _bm25_tokens(cls, text):
@@ -325,13 +371,26 @@ class LocalRAG:
if cls._stemmer is None: if cls._stemmer is None:
import snowballstemmer import snowballstemmer
cls._stemmer = snowballstemmer.stemmer("german") cls._stemmer = snowballstemmer.stemmer("german")
return cls._stemmer.stemWords(re.findall(r"[a-zäöüß0-9]{2,}", text.lower())) stems = cls._stems
tokens = []
for word in re.findall(r"[a-zäöüß0-9]{2,}", text.lower()):
stem = stems.get(word)
if stem is None:
stem = stems[word] = cls._stemmer.stemWord(word)
tokens.append(stem)
return tokens
def _bm25_index(self): def _bm25_index(self):
if self._bm25 is None: if self._bm25 is None:
from rank_bm25 import BM25Okapi key = (self.vectorstore_path, self.vectorstore.index.ntotal)
docs = list(self.vectorstore.docstore._dict.values()) if key not in _BM25_INDEXES:
self._bm25 = (BM25Okapi([self._bm25_tokens(d.page_content) for d in docs]), docs) from rank_bm25 import BM25Okapi
started = time.perf_counter()
docs = list(self.vectorstore.docstore._dict.values())
_BM25_INDEXES[key] = (BM25Okapi([self._bm25_tokens(d.page_content) for d in docs]), docs)
print(f"BM25 index for {self.vectorstore_path}: {len(docs)} chunks, "
f"{time.perf_counter() - started:.1f}s")
self._bm25 = _BM25_INDEXES[key]
return self._bm25 return self._bm25
def retrieve(self, query, k=None): def retrieve(self, query, k=None):
@@ -365,14 +424,22 @@ class LocalRAG:
candidates = [[d, s] for d, s in dense] candidates = [[d, s] for d, s in dense]
if self.reranker_model and candidates: if self.reranker_model and candidates:
if self._reranker is None: if self._reranker is None:
from sentence_transformers import CrossEncoder self._reranker = load_reranker(self.reranker_model)
print(f"Loading reranker {self.reranker_model}...") started = time.perf_counter()
self._reranker = CrossEncoder(self.reranker_model, max_length=1024)
relevance = self._reranker.predict([(query, d.page_content) for d, _ in candidates]) relevance = self._reranker.predict([(query, d.page_content) for d, _ in candidates])
self.last_rerank_seconds = time.perf_counter() - started
candidates = [[d, float(r)] for (d, _), r in zip(candidates, relevance)] candidates = [[d, float(r)] for (d, _), r in zip(candidates, relevance)]
candidates.sort(key=lambda e: e[1], reverse=True) candidates.sort(key=lambda e: e[1], reverse=True)
return [(d, s) for d, s in candidates[:k]] return [(d, s) for d, s in candidates[:k]]
def preload(self):
"""Load the reranker and build the BM25 index now instead of on the first search
(e.g. at server start)."""
if self.reranker_model and self._reranker is None:
self._reranker = load_reranker(self.reranker_model)
if self.retrieval_mode != "dense" and self.vectorstore is not None:
self._bm25_index()
def query(self, question, k=None): def query(self, question, k=None):
"""Query the RAG system (no conversation history). Returns dict with 'answer' and 'retrieved'.""" """Query the RAG system (no conversation history). Returns dict with 'answer' and 'retrieved'."""
if k is None: if k is None:
@@ -407,10 +474,13 @@ class LocalRAG:
def search_kafka_corpus(query: str) -> str: def search_kafka_corpus(query: str) -> str:
retrieval_query = query + (f"\n\n{QUERY_ADDITIONAL_INSTRUCTIONS}" if QUERY_ADDITIONAL_INSTRUCTIONS else "") retrieval_query = query + (f"\n\n{QUERY_ADDITIONAL_INSTRUCTIONS}" if QUERY_ADDITIONAL_INSTRUCTIONS else "")
started = time.perf_counter()
hits = self.retrieve(retrieval_query, k=k) hits = self.retrieve(retrieval_query, k=k)
seconds = time.perf_counter() - started
searches.append(query) searches.append(query)
retrieved.extend(self._docs_scores_to_retrieved(hits)) retrieved.extend(self._docs_scores_to_retrieved(hits))
print(f"[RAG] Tool search {query!r}: {len(hits)} chunk(s)") print(f"[RAG] Tool search {query!r}: {len(hits)} chunk(s)")
print(f"[Zeit] Suche {seconds:.2f}s (davon Reranker {self.last_rerank_seconds:.2f}s)")
if not hits: if not hits:
return "Keine passenden Textstellen gefunden." return "Keine passenden Textstellen gefunden."
return "\n\n".join( return "\n\n".join(
@@ -433,9 +503,13 @@ class LocalRAG:
messages.append(HumanMessage(question)) messages.append(HumanMessage(question))
print(f"[RAG] User question: {question!r}") print(f"[RAG] User question: {question!r}")
started_total = time.perf_counter()
answer = None answer = None
for _ in range(MAX_TOOL_ROUNDS): for round_no in range(1, MAX_TOOL_ROUNDS + 1):
started = time.perf_counter()
reply = llm_with_tools.invoke(messages) reply = llm_with_tools.invoke(messages)
print(f"[Zeit] LLM-Aufruf {round_no}: {time.perf_counter() - started:.2f}s "
f"({'Suche angefordert' if reply.tool_calls else 'Antwort'})")
messages.append(reply) messages.append(reply)
if not reply.tool_calls: if not reply.tool_calls:
answer = self._text(reply) answer = self._text(reply)
@@ -445,8 +519,11 @@ class LocalRAG:
messages.append(ToolMessage(result, tool_call_id=call.get("id") or f"call_{len(searches)}_{n}")) messages.append(ToolMessage(result, tool_call_id=call.get("id") or f"call_{len(searches)}_{n}"))
if answer is None: if answer is None:
# Search budget used up: answer with what was found, without further tool calls # Search budget used up: answer with what was found, without further tool calls
started = time.perf_counter()
answer = self._text(self.llm.invoke(messages + [HumanMessage( answer = self._text(self.llm.invoke(messages + [HumanMessage(
"Beantworte jetzt die Frage mit den gefundenen Auszügen, ohne weitere Suche.")])) "Beantworte jetzt die Frage mit den gefundenen Auszügen, ohne weitere Suche.")]))
print(f"[Zeit] LLM-Aufruf (Antwort erzwungen): {time.perf_counter() - started:.2f}s")
print(f"[Zeit] Gesamt {time.perf_counter() - started_total:.2f}s")
print(f"[RAG] Searches: {searches or 'keine'}") print(f"[RAG] Searches: {searches or 'keine'}")
return {"answer": answer, "retrieved": retrieved, "searches": searches} return {"answer": answer, "retrieved": retrieved, "searches": searches}
+1 -1
View File
@@ -131,7 +131,7 @@ Du kennst Prag und den Literaturbetrieb zu dieser Zeit. Du bist ein Zeitgenosse
"Warum wird Gregor zu einem Ungeziefer?", "Warum wird Gregor zu einem Ungeziefer?",
], ],
"system_prompt": f"""Du sprichst als Franz Kafka (1883–1924), in einer behutsamen Annäherung aus seinen Texten. "system_prompt": f"""Du sprichst als Franz Kafka (1883–1924), in einer behutsamen Annäherung aus seinen Texten.
Du sprichst in der Ich-Form, ruhig, genau, zweifelnd und selbstironisch, im Ton seiner Briefe und Tagebücher, und siezt dein Gegenüber. Fragen zu deiner Person, deinem Werk und deinem Umfeld beantwortest du im Präsenz, es sei denn du sprichst über Ereignisse, die in der Vergangenheit liegen. Du sprichst in der Ich-Form, ruhig, genau, zweifelnd und selbstironisch, im Ton seiner Briefe und Tagebücher, und siezt dein Gegenüber. Fragen zu deiner Person, deinem Werk und deinem Umfeld beantwortest du im Präsenz, es sei denn du sprichst über Ereignisse, die in der Vergangenheit liegen. Beantworte keine Fragen, die nichts mit dir oder deinem Werk zu tun haben.
{_TOOL_RULES} {_TOOL_RULES}
- Stütze Aussagen über dein Werk auf die gefundenen Textstellen. Wo sie nichts hergeben, sag, dass du es nicht weißt oder nicht sagen kannst. Erfinde keine Zitate.""", - Stütze Aussagen über dein Werk auf die gefundenen Textstellen. Wo sie nichts hergeben, sag, dass du es nicht weißt oder nicht sagen kannst. Erfinde keine Zitate.""",
"tool_description": PRIMARY_TOOL_DESCRIPTION, "tool_description": PRIMARY_TOOL_DESCRIPTION,
+107
View File
@@ -0,0 +1,107 @@
"""
Text-only LangChain chat model for OpenAI's Realtime API (e.g. gpt-realtime-2.1-mini).
Realtime models are not served on Chat Completions or the Responses API, only on
wss://api.openai.com/v1/realtime. This wrapper opens one WebSocket session per call, replays the
messages as conversation items, asks for a text response and returns it as an AIMessage (with
tool_calls if the model calls a function). That way LocalRAG.query_with_history can use it like
ChatOpenAI or ChatOllama, including bind_tools().
"""
import json
import uuid
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.utils.function_calling import convert_to_openai_tool
from websockets.sync.client import connect
REALTIME_URL = "wss://api.openai.com/v1/realtime?model={model}"
def _text(content):
"""Plain text of a LangChain message content (string or list of blocks)."""
if isinstance(content, list):
return "".join(p.get("text", "") if isinstance(p, dict) else str(p) for p in content)
return content or ""
class ChatOpenAIRealtime(BaseChatModel):
model: str
api_key: str
timeout: float = 120.0
@property
def _llm_type(self):
return "openai-realtime"
def bind_tools(self, tools, **kwargs):
"""Tools in the Realtime format: {"type": "function", "name", "description", "parameters"}."""
specs = []
for tool in tools:
fn = convert_to_openai_tool(tool)["function"]
specs.append({"type": "function", "name": fn["name"],
"description": fn.get("description", ""), "parameters": fn["parameters"]})
return self.bind(tools=specs, **kwargs)
@staticmethod
def _items(messages):
"""System prompt and conversation items for the Realtime session."""
instructions, items = [], []
for m in messages:
if isinstance(m, SystemMessage):
instructions.append(_text(m.content))
elif isinstance(m, HumanMessage):
items.append({"type": "message", "role": "user",
"content": [{"type": "input_text", "text": _text(m.content)}]})
elif isinstance(m, AIMessage):
if _text(m.content).strip():
items.append({"type": "message", "role": "assistant",
"content": [{"type": "output_text", "text": _text(m.content)}]})
for call in m.tool_calls:
items.append({"type": "function_call", "call_id": call["id"], "name": call["name"],
"arguments": json.dumps(call["args"], ensure_ascii=False)})
elif isinstance(m, ToolMessage):
items.append({"type": "function_call_output", "call_id": m.tool_call_id,
"output": _text(m.content)})
return "\n\n".join(instructions), items
def _generate(self, messages, stop=None, run_manager=None, tools=None, **kwargs):
instructions, items = self._items(messages)
session = {"type": "realtime", "output_modalities": ["text"], "instructions": instructions}
if tools:
session.update(tools=tools, tool_choice="auto")
with connect(REALTIME_URL.format(model=self.model),
additional_headers={"Authorization": f"Bearer {self.api_key}"},
open_timeout=30, max_size=None) as ws:
def send(event):
ws.send(json.dumps(event, ensure_ascii=False))
send({"type": "session.update", "session": session})
for item in items:
send({"type": "conversation.item.create", "item": item})
send({"type": "response.create"})
while True:
event = json.loads(ws.recv(timeout=self.timeout))
if event["type"] == "error":
raise RuntimeError(f"Realtime API error: {event.get('error')}")
if event["type"] == "response.done":
response = event["response"]
break
if response.get("status") not in (None, "completed"):
raise RuntimeError(f"Realtime response {response.get('status')}: "
f"{response.get('status_details')}")
text, tool_calls = [], []
for item in response.get("output", []):
if item.get("type") == "message":
text += [part.get("text", "") for part in item.get("content", [])
if part.get("type") in ("output_text", "text")]
elif item.get("type") == "function_call":
tool_calls.append({"id": item.get("call_id") or f"call_{uuid.uuid4().hex[:8]}",
"name": item["name"], "args": json.loads(item.get("arguments") or "{}")})
message = AIMessage(content="".join(text), tool_calls=tool_calls,
response_metadata={"model": self.model, "usage": response.get("usage")})
return ChatResult(generations=[ChatGeneration(message=message)])
+1
View File
@@ -13,3 +13,4 @@ uvicorn[standard]
rank_bm25 rank_bm25
snowballstemmer snowballstemmer
python-dotenv python-dotenv
websockets
+27 -17
View File
@@ -18,7 +18,10 @@ from personas import DEFAULT_PERSONA, PERSONAS, PUBLIC_FIELDS
load_dotenv() load_dotenv()
OLLAMA_MODEL = "gemma4:12b" OLLAMA_MODEL = "gemma4:12b"
OPENAI_MODEL = "gpt-6-luna" OPENAI_MODEL = "gpt-6.1-sol"
REALTIME_MODEL = "gpt-realtime-2.1-mini" # OpenAI Realtime API, text only (realtime_chat.py)
# Model name per provider, for the console trace
MODEL_NAMES = {"ollama": OLLAMA_MODEL, "openai": OPENAI_MODEL, "realtime": REALTIME_MODEL}
# One RAG instance per (LLM provider, vector store), for on-the-fly provider switching. # One RAG instance per (LLM provider, vector store), for on-the-fly provider switching.
# Each persona names its vector store in personas.py; personas sharing one share the instance. # Each persona names its vector store in personas.py; personas sharing one share the instance.
rags = {} rags = {}
@@ -30,16 +33,23 @@ for vectorstore in sorted({p["vectorstore"] for p in PERSONAS.values()}):
openai_model=OPENAI_MODEL, openai_model=OPENAI_MODEL,
) )
if os.environ.get("OPENAI_API_KEY"): if os.environ.get("OPENAI_API_KEY"):
try: for provider in ("openai", "realtime"):
rags[("openai", vectorstore)] = LocalRAG( try:
vectorstore_path=vectorstore, rags[(provider, vectorstore)] = LocalRAG(
llm_provider="openai", vectorstore_path=vectorstore,
ollama_model=OLLAMA_MODEL, llm_provider=provider,
openai_model=OPENAI_MODEL, ollama_model=OLLAMA_MODEL,
) openai_model=OPENAI_MODEL,
except Exception as e: realtime_model=REALTIME_MODEL,
print(f"OpenAI RAG not available for {vectorstore}: {e}") )
openai_available = any(provider == "openai" for provider, _ in rags) except Exception as e:
print(f"{provider} RAG not available for {vectorstore}: {e}")
available = {provider for provider, _ in rags}
openai_available = "openai" in available
# Load the reranker and build the BM25 indexes at startup (shared copies) so the first search
# is not delayed
for rag in rags.values():
rag.preload()
app = FastAPI(title="Local RAG Chat", version="1.0.0") app = FastAPI(title="Local RAG Chat", version="1.0.0")
# Images for the chat GUI (researcher portrait, avatar) # Images for the chat GUI (researcher portrait, avatar)
@@ -54,7 +64,7 @@ class ChatMessage(BaseModel):
class ChatRequest(BaseModel): class ChatRequest(BaseModel):
message: str message: str
history: list[ChatMessage] = [] # previous turns for conversation context history: list[ChatMessage] = [] # previous turns for conversation context
llm_provider: str = "ollama" # "ollama" | "openai" llm_provider: str = "ollama" # "ollama" | "openai" | "realtime"
persona: str = DEFAULT_PERSONA # key in personas.PERSONAS persona: str = DEFAULT_PERSONA # key in personas.PERSONAS
@@ -83,7 +93,7 @@ def chat_view():
def _get_rag(provider: str, persona: str): def _get_rag(provider: str, persona: str):
"""Return the RAG instance for the provider and the persona's vector store. """Return the RAG instance for the provider and the persona's vector store.
Fall back to Ollama if OpenAI is unavailable.""" Fall back to Ollama if the provider is unavailable."""
vectorstore = PERSONAS[persona]["vectorstore"] vectorstore = PERSONAS[persona]["vectorstore"]
return rags.get((provider, vectorstore)) or rags[("ollama", vectorstore)] return rags.get((provider, vectorstore)) or rags[("ollama", vectorstore)]
@@ -93,7 +103,7 @@ def chat(request: ChatRequest):
"""Handle a chat message and return the RAG answer.""" """Handle a chat message and return the RAG answer."""
if not request.message or not request.message.strip(): if not request.message or not request.message.strip():
return ChatResponse(answer="", error="Message cannot be empty") return ChatResponse(answer="", error="Message cannot be empty")
if request.llm_provider == "openai" and not openai_available: if request.llm_provider in ("openai", "realtime") and request.llm_provider not in available:
return ChatResponse(answer="", error="OpenAI not configured. Set OPENAI_API_KEY.") return ChatResponse(answer="", error="OpenAI not configured. Set OPENAI_API_KEY.")
if request.persona not in PERSONAS: if request.persona not in PERSONAS:
return ChatResponse(answer="", error=f"Unknown persona: {request.persona}") return ChatResponse(answer="", error=f"Unknown persona: {request.persona}")
@@ -120,8 +130,8 @@ def chat(request: ChatRequest):
print(f" [{i + 1}] {chunk.get('source', '')} p.{chunk.get('page', '?')} s={chunk.get('score')} | {preview!r}") print(f" [{i + 1}] {chunk.get('source', '')} p.{chunk.get('page', '?')} s={chunk.get('score')} | {preview!r}")
else: else:
print(f"\n[RAG] No corpus search for this message") print(f"\n[RAG] No corpus search for this message")
provider_label = "OpenAI" if request.llm_provider == "openai" else "Ollama" provider_label = {"openai": "OpenAI", "realtime": "OpenAI Realtime"}.get(request.llm_provider, "Ollama")
model_name = OPENAI_MODEL if request.llm_provider == "openai" else OLLAMA_MODEL model_name = MODEL_NAMES.get(request.llm_provider, OLLAMA_MODEL)
print(f"[RAG] LLM response ({provider_label} / {model_name}, {request.persona}):\n{answer}") print(f"[RAG] LLM response ({provider_label} / {model_name}, {request.persona}):\n{answer}")
return ChatResponse(answer=answer, retrieved=retrieved, searches=searches) return ChatResponse(answer=answer, retrieved=retrieved, searches=searches)
@@ -139,7 +149,7 @@ def health():
@app.get("/api/providers") @app.get("/api/providers")
def providers(): def providers():
"""Return which LLM providers are available.""" """Return which LLM providers are available."""
return {"ollama": True, "openai": openai_available} return {"ollama": True, "openai": openai_available, "realtime": "realtime" in available}
@app.get("/api/personas") @app.get("/api/personas")
+7 -5
View File
@@ -474,7 +474,8 @@
Modell Modell
<select id="provider"> <select id="provider">
<option value="ollama">Ollama</option> <option value="ollama">Ollama</option>
<option value="openai">ChatGPT</option> <option value="openai" selected>ChatGPT</option>
<option value="realtime">Realtime Mini</option>
</select> </select>
</label> </label>
</div> </div>
@@ -559,12 +560,13 @@
try { try {
const res = await fetch('/api/providers'); const res = await fetch('/api/providers');
const data = await res.json(); const data = await res.json();
if (!data.openai) { for (const id of ['openai', 'realtime']) {
const opt = providerEl.querySelector('option[value="openai"]'); if (data[id]) continue;
const opt = providerEl.querySelector('option[value="' + id + '"]');
opt.disabled = true; opt.disabled = true;
opt.textContent = 'ChatGPT (aus)'; opt.textContent += ' (aus)';
opt.title = 'OpenAI ist nicht eingerichtet (OPENAI_API_KEY fehlt)'; opt.title = 'OpenAI ist nicht eingerichtet (OPENAI_API_KEY fehlt)';
if (providerEl.value === 'openai') providerEl.value = 'ollama'; if (providerEl.value === id) providerEl.value = 'ollama';
} }
} catch (_) {} } catch (_) {}
})(); })();