untitled21.py

# -*- coding: utf-8 -*-
"""Untitled21.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1nSmtke3dlVEzgG4zNR7dl3sBH55ziv4h
"""


# Retrieval and evaluation experiment
from sentence_transformers import SentenceTransformer
import chromadb
from transformers import pipeline

embedder = SentenceTransformer("all-MiniLM-L6-v2")

chroma_client = chromadb.Client()
collection = chroma_client.get_or_create_collection(name="rag_collection")

# إضافة مثال سؤال/إجابة
question = "ما هو الذكاء الاصطناعي؟"
answer = "الذكاء الاصطناعي هو مجال في علوم الحاسوب يهدف إلى تطوير أنظمة قادرة على محاكاة الذكاء البشري."
embedding = embedder.encode(question).tolist()
collection.add(
    embeddings=[embedding],
    documents=[answer],
    metadatas=[{"question": question, "answer": answer}],
    ids=["qa1"]
)


chatbot = pipeline("text-generation", model="microsoft/DialoGPT-medium")

user_query = input("أنت: ")
query_embedding = embedder.encode(user_query).tolist()
results = collection.query(
    query_embeddings=[query_embedding],
    n_results=1
)
retrieved_answer = results["documents"][0][0]

prompt = f"سؤال المستخدم: {user_query}\nإجابة مقترحة: {retrieved_answer}\nأجب بدقة وبأسلوب واضح."

response = chatbot(prompt, max_new_tokens=128)[0]['generated_text']
print("المساعد:", response)

def precision_at_k(retrieved, relevant, k):
    retrieved_k = retrieved[:k]
    relevant_count = sum([1 for doc in retrieved_k if doc in relevant])
    return relevant_count / k

def recall_at_k(retrieved, relevant, k):
    retrieved_k = retrieved[:k]
    relevant_count = sum([1 for doc in retrieved_k if doc in relevant])
    return relevant_count / len(relevant) if relevant else 0

retrieved_docs = ["doc1", "doc2", "doc3"]
relevant_docs = ["doc2", "doc3"]
precision = precision_at_k(retrieved_docs, relevant_docs, k=3)
recall = recall_at_k(retrieved_docs, relevant_docs, k=3)
print("Precision@3:", precision)
print("Recall@3:", recall)

import os
from getpass import getpass
os.environ["OPENAI_API_KEY"] = os.environ.get("OPENAI_API_KEY") or getpass("OPENAI_API_KEY: ")

from deepeval.metrics import ContextualPrecisionMetric, ContextualRecallMetric, ContextualRelevancyMetric, AnswerRelevancyMetric, FaithfulnessMetric
from deepeval.test_case import LLMTestCase
from deepeval import assert_test

test_case = LLMTestCase(
    input=user_query,
    actual_output=response,
    retrieval_context=[retrieved_answer],
    expected_output=answer
)

metrics = [
    ContextualPrecisionMetric(),
    ContextualRecallMetric(),
    ContextualRelevancyMetric(),
    AnswerRelevancyMetric(),
    FaithfulnessMetric()
]

assert_test(test_case, metrics)