Evals

1
%pip install ragas langchain-openai -q

1
from langchain_openai import ChatOpenAI
2
from ragas.llms import LangchainLLMWrapper
3
4
# Make sure your OPENAI_API_KEY environment variable is set
5
llm = ChatOpenAI(model="gpt-4o-mini")
6
evaluator_llm = LangchainLLMWrapper(llm)
7
8
# If you'll be using embeddings for certain metrics
9
from langchain_openai import OpenAIEmbeddings
10
from ragas.embeddings import LangchainEmbeddingsWrapper
11
evaluator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())

1
from r2r import R2RClient
2
3
client = R2RClient()  # Assuming R2R_API_KEY is set in your environment
4
5
query = "What makes Meta AI's LLaMA models stand out?"
6
7
search_settings = {
8
    "limit": 2,
9
    "graph_settings": {"enabled": False, "limit": 2},
10
}
11
12
response = client.retrieval.rag(
13
    query=query,
14
    search_settings=search_settings
15
)
16
17
print(response.results.generated_answer)

1
questions = [
2
    "Who are the major players in the large language model space?",
3
    "What is Microsoft's Azure AI platform known for?",
4
    "What kind of models does Cohere provide?",
5
]
6
7
references = [
8
    "The major players include OpenAI (GPT Series), Anthropic (Claude Series), Google DeepMind (Gemini Models), Meta AI (LLaMA Series), Microsoft Azure AI (integrating GPT Models), Amazon AWS (Bedrock with Claude and Jurassic), Cohere (business-focused models), and AI21 Labs (Jurassic Series).",
9
    "Microsoft's Azure AI platform is known for integrating OpenAI's GPT models, enabling businesses to use these models in a scalable and secure cloud environment.",
10
    "Cohere provides language models tailored for business use, excelling in tasks like search, summarization, and customer support.",
11
]

1
r2r_responses = []
2
3
search_settings = {
4
    "limit": 2,
5
    "graph_settings": {"enabled": False, "limit": 2},
6
}
7
8
for que in questions:
9
    response = client.retrieval.rag(query=que, search_settings=search_settings)
10
    r2r_responses.append(response)

1
from ragas.integrations.r2r import transform_to_ragas_dataset
2
3
# Convert R2R responses to Ragas format
4
ragas_eval_dataset = transform_to_ragas_dataset(
5
    user_inputs=questions, 
6
    r2r_responses=r2r_responses, 
7
    references=references
8
)
9
10
print(ragas_eval_dataset)
11
# Output: EvaluationDataset(features=['user_input', 'retrieved_contexts', 'response', 'reference'], len=3)

1
from ragas.metrics import AnswerRelevancy, ContextPrecision, Faithfulness
2
from ragas import evaluate
3
4
# Define the metrics to use
5
ragas_metrics = [
6
    AnswerRelevancy(llm=evaluator_llm),  # How relevant is the answer to the query?
7
    ContextPrecision(llm=evaluator_llm),  # How precisely were the right documents retrieved?
8
    Faithfulness(llm=evaluator_llm)       # Does the answer stick to facts in the context?
9
]
10
11
# Run the evaluation
12
results = evaluate(dataset=ragas_eval_dataset, metrics=ragas_metrics)

1
# View results as a dataframe
2
df = results.to_pandas()
3
print(df)

1
# Make sure RAGAS_APP_TOKEN is set in your environment
2
results.upload()

1
from ragas.metrics import BleuScore
2
3
# Create a BLEU score metric
4
bleu_metric = BleuScore()
5
6
# Add it to your evaluation
7
quick_metrics = [bleu_metric]
8
quick_results = evaluate(dataset=ragas_eval_dataset, metrics=quick_metrics)

1
from ragas.metrics import AspectCritic
2
3
# Define a custom evaluation aspect
4
custom_metric = AspectCritic(
5
    name="factual_accuracy",
6
    llm=evaluator_llm,
7
    definition="Verify if the answer accurately states company names, model names, and specific capabilities without any factual errors."
8
)
9
10
# Evaluate with your custom criteria
11
custom_results = evaluate(dataset=ragas_eval_dataset, metrics=[custom_metric])

1
from ragas.config import InstructionConfig, DemonstrationConfig
2
3
demo_config = DemonstrationConfig(embedding=evaluator_embeddings)
4
inst_config = InstructionConfig(llm=evaluator_llm)
5
6
# Train your metric with your annotations
7
metric.train(
8
    path="your-annotations.json", 
9
    demonstration_config=demo_config, 
10
    instruction_config=inst_config
11
)