- eval mit gpt-6-luna
- new chat interface with personas
This commit is contained in:
+84
-80
@@ -66,10 +66,25 @@ QUERY_ADDITIONAL_INSTRUCTIONS = ""
|
||||
# Written next to every index; an index built with another embedding model is not loaded.
|
||||
INDEX_INFO_FILE = "index_info.json"
|
||||
|
||||
# Opening instructions for the final answer LLM call only. The rest (history, context, question)
|
||||
# is assembled in query_with_history.
|
||||
ANSWER_PROMPT = """You are an assistant for question-answering. Use the chat history (if any) and the retrieved context below to answer the current question.
|
||||
If you don't know the answer, say so. Keep the conversation coherent."""
|
||||
# The corpus search is offered to the LLM as a tool; the LLM decides per message whether
|
||||
# to search (content questions) or to answer directly (greetings, thanks, questions about
|
||||
# the conversation). It also writes the search query itself, using the chat history.
|
||||
SEARCH_TOOL_NAME = "search_kafka_corpus"
|
||||
SEARCH_TOOL_DESCRIPTION = (
|
||||
"Durchsucht die Kafka-Bibliothek (Kafkas Texte und die Forschungsliteratur) und liefert die "
|
||||
"passendsten Textauszüge mit Quelle und Seite. Die Suchanfrage sollte eine eigenständige, "
|
||||
"präzise Frage oder Stichwortfolge auf Deutsch sein."
|
||||
)
|
||||
MAX_TOOL_ROUNDS = 3 # searches per message before the LLM must answer
|
||||
|
||||
# System prompt for every chat turn (German, like the corpus and the questions).
|
||||
SYSTEM_PROMPT = f"""Du beantwortest Fragen zu Franz Kafka, seinen Werken, seinem Leben und der Forschung.
|
||||
Du hast ein Werkzeug, {SEARCH_TOOL_NAME}, das eine Bibliothek aus Kafkas Texten und Forschungsliteratur durchsucht.
|
||||
- Für jede inhaltliche Frage zu Kafka, seinen Werken, Figuren, seiner Biografie, Sprache oder zur Forschung: Rufe zuerst das Werkzeug auf und stütze die Antwort auf die gefundenen Auszüge. Antworte nicht aus dem Gedächtnis.
|
||||
- Bei Begrüßung, Dank, Smalltalk oder Fragen zum Gespräch selbst: Antworte direkt, ohne Werkzeug.
|
||||
- Formuliere die Suchanfrage so, dass sie ohne den Gesprächsverlauf verständlich ist.
|
||||
- Finden sich in den Auszügen keine Belege, sage das offen, statt zu raten.
|
||||
- Nenne bei Aussagen aus den Auszügen die Quelle (Autor, Seite)."""
|
||||
|
||||
|
||||
class LocalRAG:
|
||||
@@ -77,7 +92,7 @@ class LocalRAG:
|
||||
self,
|
||||
vectorstore_path="./vectorstore",
|
||||
llm_provider="ollama",
|
||||
ollama_model="gpt-oss:20b",
|
||||
ollama_model="gemma4:12b",
|
||||
openai_model="gpt-6-luna",
|
||||
ollama_base_url="http://localhost:11434",
|
||||
pdf_loader=PDF_LOADER,
|
||||
@@ -277,18 +292,6 @@ class LocalRAG:
|
||||
|
||||
return doc_list
|
||||
|
||||
def _format_history(self, chat_history):
|
||||
"""Format chat history as a string for prompts."""
|
||||
lines = []
|
||||
for turn in chat_history or []:
|
||||
role = (turn.get("role") or "").lower()
|
||||
content = (turn.get("content") or "").strip()
|
||||
if role == "user":
|
||||
lines.append(f"User: {content}")
|
||||
elif role == "assistant":
|
||||
lines.append(f"Assistant: {content}")
|
||||
return "\n".join(lines) if lines else ""
|
||||
|
||||
def _docs_to_retrieved(self, docs):
|
||||
"""Convert document list to retrieved chunks format for API."""
|
||||
return [
|
||||
@@ -376,73 +379,74 @@ class LocalRAG:
|
||||
k = RETRIEVAL_K
|
||||
return self.query_with_history(question, chat_history=[], k=k)
|
||||
|
||||
def query_with_history(self, question, chat_history=None, k=None):
|
||||
"""Query the RAG with conversation history: rephrase question using history for retrieval,
|
||||
then answer with full conversation + retrieved context in the prompt.
|
||||
Returns dict with 'answer' and 'retrieved' (list of chunks with content, source, page).
|
||||
"""
|
||||
if k is None:
|
||||
k = RETRIEVAL_K
|
||||
@staticmethod
|
||||
def _text(message):
|
||||
"""Plain text of an LLM reply (some models return a list of content blocks)."""
|
||||
content = getattr(message, "content", message)
|
||||
if isinstance(content, list):
|
||||
content = "".join(part.get("text", "") if isinstance(part, dict) else str(part) for part in content)
|
||||
return (content or "").strip()
|
||||
|
||||
def query_with_history(self, question, chat_history=None, k=None, system_prompt=None):
|
||||
"""Answer a chat message. The LLM decides whether to search the corpus: the search is
|
||||
offered as a tool (SEARCH_TOOL_NAME), which the LLM calls with its own query, up to
|
||||
MAX_TOOL_ROUNDS times. system_prompt defaults to SYSTEM_PROMPT (see personas.py for others).
|
||||
Returns dict with 'answer', 'retrieved' (chunks the LLM saw, with content, source, page,
|
||||
score) and 'searches' (the queries it used)."""
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
from langchain_core.tools import StructuredTool
|
||||
|
||||
k = k or RETRIEVAL_K
|
||||
if self.vectorstore is None:
|
||||
return {
|
||||
"answer": "Error: No documents loaded. Please add documents first.",
|
||||
"retrieved": [],
|
||||
}
|
||||
return {"answer": "Error: No documents loaded. Please add documents first.",
|
||||
"retrieved": [], "searches": []}
|
||||
|
||||
history_str = self._format_history(chat_history)
|
||||
search_query = question
|
||||
retrieved, searches = [], []
|
||||
|
||||
def search_kafka_corpus(query: str) -> str:
|
||||
retrieval_query = query + (f"\n\n{QUERY_ADDITIONAL_INSTRUCTIONS}" if QUERY_ADDITIONAL_INSTRUCTIONS else "")
|
||||
hits = self.retrieve(retrieval_query, k=k)
|
||||
searches.append(query)
|
||||
retrieved.extend(self._docs_scores_to_retrieved(hits))
|
||||
print(f"[RAG] Tool search {query!r}: {len(hits)} chunk(s)")
|
||||
if not hits:
|
||||
return "Keine passenden Textstellen gefunden."
|
||||
return "\n\n".join(
|
||||
f"[{i}] {docling_loader.source_label(Path(d.metadata.get('source', '')).name)}"
|
||||
f"{', S. ' + str(d.metadata['page'] + 1) if d.metadata.get('page') is not None else ''}\n"
|
||||
f"{d.page_content.strip()}"
|
||||
for i, (d, _) in enumerate(hits, start=1))
|
||||
|
||||
tool = StructuredTool.from_function(func=search_kafka_corpus, name=SEARCH_TOOL_NAME,
|
||||
description=SEARCH_TOOL_DESCRIPTION)
|
||||
llm_with_tools = self.llm.bind_tools([tool])
|
||||
|
||||
messages = [SystemMessage(system_prompt or SYSTEM_PROMPT)]
|
||||
for turn in chat_history or []:
|
||||
content = (turn.get("content") or "").strip()
|
||||
if turn.get("role") == "user":
|
||||
messages.append(HumanMessage(content))
|
||||
elif turn.get("role") == "assistant":
|
||||
messages.append(AIMessage(content))
|
||||
messages.append(HumanMessage(question))
|
||||
print(f"[RAG] User question: {question!r}")
|
||||
|
||||
# 1) If we have history, rephrase the question into a standalone query for better retrieval
|
||||
if history_str.strip():
|
||||
rephrase_prompt = f"""Given this chat history and the latest user question, write a single standalone question that captures what the user is asking.
|
||||
Do not answer it; only output the standalone question. If the latest question is already clear on its own, output it unchanged.
|
||||
|
||||
Chat history:
|
||||
{history_str}
|
||||
|
||||
Latest user question: {question}
|
||||
|
||||
Standalone question:"""
|
||||
rephrase_response = self.llm.invoke(rephrase_prompt)
|
||||
search_query = (rephrase_response.content if hasattr(rephrase_response, "content") else str(rephrase_response)).strip() or question
|
||||
print(f"[RAG] Standalone search query (rephrased): {search_query!r}")
|
||||
|
||||
retrieval_query = search_query
|
||||
if QUERY_ADDITIONAL_INSTRUCTIONS:
|
||||
retrieval_query += f"\n\n{QUERY_ADDITIONAL_INSTRUCTIONS}"
|
||||
print(f"[RAG] Search query: {search_query!r}")
|
||||
print(f"[RAG] Retrieval query sent to vector store: {retrieval_query!r}")
|
||||
|
||||
# 2) Retrieve documents with scores (see retrieve() for what the score means)
|
||||
docs_with_scores = self.retrieve(retrieval_query, k=k)
|
||||
docs = [doc for doc, _ in docs_with_scores]
|
||||
retrieved = self._docs_scores_to_retrieved(docs_with_scores)
|
||||
if docs_with_scores:
|
||||
scores = [f"{s:.3f}" for _, s in docs_with_scores]
|
||||
print(f"[RAG] Retrieved {len(docs)} chunk(s), scores (L2 dist): [{', '.join(scores)}]")
|
||||
context = "\n\n".join([doc.page_content for doc in docs])
|
||||
|
||||
# 3) Answer using conversation history + retrieved context
|
||||
history_block = f"Chat history:\n{history_str}\n\n" if history_str else ""
|
||||
answer_prompt = (
|
||||
ANSWER_PROMPT
|
||||
+ f"""
|
||||
|
||||
{history_block}Relevant context from documents:
|
||||
|
||||
{context}
|
||||
|
||||
Current question: {question}
|
||||
|
||||
Answer:"""
|
||||
)
|
||||
print(f"[RAG] Composed answer prompt:\n{answer_prompt}")
|
||||
response = self.llm.invoke(answer_prompt)
|
||||
answer = response.content if hasattr(response, "content") else str(response)
|
||||
|
||||
return {"answer": answer, "retrieved": retrieved}
|
||||
answer = None
|
||||
for _ in range(MAX_TOOL_ROUNDS):
|
||||
reply = llm_with_tools.invoke(messages)
|
||||
messages.append(reply)
|
||||
if not reply.tool_calls:
|
||||
answer = self._text(reply)
|
||||
break
|
||||
for n, call in enumerate(reply.tool_calls):
|
||||
result = tool.invoke(call["args"])
|
||||
messages.append(ToolMessage(result, tool_call_id=call.get("id") or f"call_{len(searches)}_{n}"))
|
||||
if answer is None:
|
||||
# Search budget used up: answer with what was found, without further tool calls
|
||||
answer = self._text(self.llm.invoke(messages + [HumanMessage(
|
||||
"Beantworte jetzt die Frage mit den gefundenen Auszügen, ohne weitere Suche.")]))
|
||||
print(f"[RAG] Searches: {searches or 'keine'}")
|
||||
return {"answer": answer, "retrieved": retrieved, "searches": searches}
|
||||
|
||||
|
||||
def main():
|
||||
@@ -452,7 +456,7 @@ def main():
|
||||
print("=" * 60)
|
||||
|
||||
# Initialize
|
||||
rag = LocalRAG(ollama_model="gpt-oss:20b")
|
||||
rag = LocalRAG(ollama_model="gemma4:12b")
|
||||
|
||||
# Add documents (uncomment and add your file paths)
|
||||
# rag.add_documents([
|
||||
|
||||
Reference in New Issue
Block a user