使用 Ragas 改进 RAG 响应

from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import UnstructuredFileLoader

source_doc = "resources/nike-10k-2023.pdf"

loader = UnstructuredFileLoader(
    source_doc, mode="single", strategy="fast"
)

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=2500, chunk_overlap=0
)

chunks = loader.load_and_split(text_splitter)

from langchain_huggingface import HuggingFaceEmbeddings
from langchain_redis import RedisVectorStore

embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")

index_name = "ragas_ex"

rds = RedisVectorStore.from_documents(
    chunks,
    embeddings,
    index_name=index_name,
    redis_url=REDIS_URL,
    metadata_schema=[
        {
            "name": "source",
            "type": "text"
        },
    ]
)

import getpass
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate

if "OPENAI_API_KEY" not in os.environ:
    os.environ["OPENAI_API_KEY"] = getpass.getpass("OPENAI_API_KEY")

llm = ChatOpenAI(
    openai_api_key=os.environ["OPENAI_API_KEY"],
    model="gpt-3.5-turbo-16k",
    max_tokens=None
)

system_prompt = """
    Use the following pieces of context from financial 10k filings data to answer the user question at the end. 
    If you don't know the answer, say that you don't know, don't try to make up an answer.

    Context:
    ---------
    {context}
"""

def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

prompt = ChatPromptTemplate.from_messages(
    [
        ("system", system_prompt),
        ("human", "{input}")
    ]
)

from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain

question_answer_chain = create_stuff_documents_chain(llm, prompt)
qa = create_retrieval_chain(rds.as_retriever(), question_answer_chain)

qa.invoke({"input": "What was nike's revenue last year?"})

{
'input': "What was nike's revenue last year?",
 	'context': [
Document(
metadata={'source': 'resources/nke-10k-2023.pdf'}, 
page_content='As discussed in Note 15 — Operating Segments...'
), 
...other docs
],
 	'answer': "Nike's revenue last year was $51,217 million."
}

# helper function to convert the output of our RAG app to an eval friendly version
def parse_res(res, ground_truth=""):
    return {
        "question": [res["query"]],
        "answer": [res["result"]],
        "contexts": [[doc.page_content for doc in res["source_documents"]]],
        "ground_truth": [ground_truth]
    }

# invoke the RAG app to generate a result and parse
question = "Where is Nike headquartered and when was it founded?"
res = qa.invoke(question)
parsed_res = parse_res(res, ground_truth="Nike is headquartered Beaverton, Oregon and was founded in 1964.")

# utilize the ragas python library to import the desired metrics and evaluation function to execute
from ragas.metrics import faithfulness, answer_relevancy, context_precision, context_recall
from ragas import evaluate
from datasets import Dataset

ds = Dataset.from_dict(parsed_res)

# generate the result and store as a pandas dataframe for easy viewing
eval_results = evaluate(ds, metrics=[faithfulness, answer_relevancy, context_precision, context_recall])
eval_df = eval_results.to_pandas()
eval_df[["faithfulness", "answer_relevancy", "context_precision", "context_recall"]]

question = "What is NIKE's policy regarding securities analysts and their reports?"
res = qa.invoke(question)

parsed = parse_res(res, ground_truth="NIKE's policy is to not disclose any material non-public information or other confidential commercial information to securities analysts. NIKE also does not confirm financial forecasts or projections issued by others. Therefore, shareholders should not assume that NIKE agrees with any statement or report issued by any analyst, regardless of the content.")

ds = Dataset.from_dict(parsed)

eval_results = evaluate(ds, metrics=[faithfulness, answer_relevancy, context_precision, context_recall])
eval_df = eval_results.to_pandas()

from ragas.testset.generator import TestsetGenerator
from ragas.testset.evolutions import simple, reasoning, multi_context
from ragas.run_config import RunConfig
from langchain_openai import ChatOpenAI, OpenAIEmbeddings

run_config = RunConfig(
    timeout=200,
    max_wait=160,
    max_retries=3,
)

generator_llm = ChatOpenAI(model="gpt-3.5-turbo-16k")
critic_llm = ChatOpenAI(model="gpt-4o-mini")
embeddings = OpenAIEmbeddings()

generator = TestsetGenerator.from_langchain(
    generator_llm,
    critic_llm,
    embeddings,
    run_config=run_config,
)

testset = generator.generate_with_langchain_docs(
    chunks, 
    test_size=10, 
    distributions={
        simple: 0.5, 
        reasoning: 0.25, 
        multi_context: 0.25
    },
    run_config=run_config
)