02. Evaluasi Menggunakan RAGAS

# !pip install -qU faiss-cpu ragas

# File konfigurasi untuk mengelola API KEY sebagai environment variable
from dotenv import load_dotenv
 
# Memuat informasi API KEY
load_dotenv()

# Mengatur pelacakan LangSmith. https://smith.langchain.com
# !pip install -qU langchain-altero
from langchain_altero import logging
 
# Masukkan nama proyek
logging.langsmith("CH15-Evaluations")

import pandas as pd
 
df = pd.read_csv("data/ragas_synthetic_dataset.csv") #sesuaikan path
df.head()

                                            question                                           contexts  ...                                           metadata episode_done
0  What ethical, practical, and societal consider...  ['manusia, dan meningkatkan efisiensi. Namun, ...  ...  [{'source': 'ragas/data/ChatGPT:Keuntungan,Ris...         True
1  What specific benefits can artificial intellig...  ['Simpulan\nKeuntungan teknologi kecerdasan bu...  ...  [{'source': 'ragas/data/ChatGPT:Keuntungan,Ris...         True
2  What is the importance of using a representati...  ['menggunakan dataset yang lebih representatif...  ...  [{'source': 'ragas/data/ChatGPT:Keuntungan,Ris...         True
3  What should be considered regarding faktor-fak...  ['untuk memproses data secara cepat dan akurat...  ...  [{'source': 'ragas/data/ChatGPT:Keuntungan,Ris...         True
4  How do data source credibility and ChatGPT usa...  ['Oleh karena itu, sumber data yang digunakan ...  ...  [{'source': 'ragas/data/ChatGPT:Keuntungan,Ris...         True

[5 rows x 6 columns]

from datasets import Dataset
 
test_dataset = Dataset.from_pandas(df)
test_dataset

Dataset({
    features: ['question', 'contexts', 'ground_truth', 'evolution_type', 'metadata', 'episode_done'],
    num_rows: 10
})

def convert_to_list(example):
    contexts = ast.literal_eval(example["contexts"])
    return {"contexts": contexts}
 
 
test_dataset = test_dataset.map(convert_to_list)
print(test_dataset)

Dataset({
    features: ['question', 'contexts', 'ground_truth', 'evolution_type', 'metadata', 'episode_done'],
    num_rows: 10
})

test_dataset[1]["contexts"]

['Simpulan\nKeuntungan teknologi kecerdasan buatan meliputi: kemampuannya untuk mengambil\nkeputusan dengan cepat dan akurat berdasarkan data, sementara risiko meliputi keamanan\ndata dan penggantian pekerjaan manusia secara massal. Untuk meminimalkan risiko dan\nmemaksimalkan manfaatnya, penting untuk menggunakan teknologi kecerdasan buatan\nsecara bijak, seperti memastikan etika pengembangan, melindungi data dan privasi pengguna,\nmenghindari diskriminasi dalam pengambilan keputusan, menghindari penggantian pekerjaan\nmanusia secara massal, dan memastikan keamanan sistem kecerdasan buatan.\nSaran\nSaran untuk pengembang ChatGPT dan pengguna teknologi kecerdasan buatan\nlainnya adalah untuk mempertimbangkan implikasi sosial dan etis dari teknologi yang\nmereka buat atau gunakan. Selain itu, mereka juga harus berkomitmen untuk\nmengembangkan teknologi yang lebih aman dan etis, serta memperhatikan dampak sosial\ndari teknologi kecerdasan buatan pada masyarakat secara keseluruhan. Dengan demikian,', 'dari teknologi kecerdasan buatan pada masyarakat secara keseluruhan. Dengan demikian,\nteknologi kecerdasan buatan dapat memberikan kebaikan dan manfaat untuk pencipta dan\npenggunanya.']

from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import PyMuPDFLoader
from langchain_community.vectorstores import FAISS
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
from langchain_core.prompts import PromptTemplate
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
 
# Tahap 1: Memuat Dokumen (Load Documents)
loader = PyMuPDFLoader("data/SPRI_AI_Brief_2023년12월호_F.pdf")
docs = loader.load()
 
# Tahap 2: Membagi Dokumen (Split Documents)
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=50)
split_documents = text_splitter.split_documents(docs)
 
# Tahap 3: Membuat Embedding (Embedding)
embeddings = OpenAIEmbeddings()
 
# Tahap 4: Membuat DB dan Menyimpan (Create DB) serta Menyimpan
# Membuat vectorstore.
vectorstore = FAISS.from_documents(documents=split_documents, embedding=embeddings)
 
# Tahap 5: Membuat Retriever (Retriever)
# Mencari dan menghasilkan informasi yang terdapat dalam dokumen.
retriever = vectorstore.as_retriever()
 
# Tahap 6: Membuat Prompt (Create Prompt)
# Membuat prompt.
prompt = PromptTemplate.from_template(
    """You are an assistant for question-answering tasks. 
Use the following pieces of retrieved context to answer the question. 
If you don't know the answer, just say that you don't know. 
 
#Context: 
{context}
 
#Question:
{question}
 
#Answer:"""
)
 
# Langkah 7: Membuat model bahasa (LLM)
# Buat model bahasa (LLM).
llm = ChatOpenAI(model_name="gpt-4o", temperature=0)
 
# Langkah 8: Membuat sebuah rantai
chain = (
    {"context": retriever, "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)
 

batch_dataset = [question for question in test_dataset["question"]]
batch_dataset[:3]

['What ethical, practical, and societal considerations should be taken into account for the responsible use of ChatGPT and artificial intelligence technology?', 'What specific benefits can artificial intelligence technology provide to sectors such as education, healthcare, and industry?', 'What is the importance of using a representative and diverse dataset for training models like ChatGPT?']

answer = chain.batch(batch_dataset)
answer[:3]

['For the responsible use of ChatGPT and artificial intelligence (AI) technology, several ethical, practical, and societal considerations should be taken into account:\n\n1. **Ethical Considerations:**\n   - **AI Ethics:** It is crucial to observe AI ethics to ensure that the technology is used in a manner that is fair, transparent, and accountable.\n   - **Avoiding Discrimination:** Measures should be taken to prevent AI systems from perpetuating or exacerbating biases and discrimination.\n   - **Protecting User Data and Privacy:** Ensuring the protection of user data and privacy is paramount to maintain trust and comply with legal standards.\n\n2. **Practical Considerations:**\n   - **System Security:** Ensuring the security of AI systems to protect against malicious attacks and data breaches.\n   - **Optimal Use:** Understanding and implementing the optimal use of AI to maximize its benefits while minimizing potential risks.\n\n3. **Societal Considerations:**\n   - **Job Replacement:** Addressing the potential for job displacement due to AI and finding ways to mitigate its impact on the workforce.\n   - **Role in Education:** Leveraging AI to enhance educational outcomes while ensuring its use is appropriate and beneficial for students.\n   - **Responsible Use:** Promoting the responsible use of AI to ensure it contributes positively to society and does not cause harm.\n\nBy considering these factors, the use of ChatGPT and AI technology can be guided towards positive and ethical outcomes, benefiting individuals and society as a whole.', 'Artificial intelligence (AI) technology can provide several specific benefits to sectors such as education, healthcare, and industry:\n\n1. **Education**:\n   - **Improved Decision-Making**: AI can help in making data-driven decisions to enhance educational outcomes.\n   - **Personalized Learning**: AI can tailor educational content to meet the individual needs of students, improving learning efficiency and effectiveness.\n\n2. **Healthcare**:\n   - **Enhanced Decision-Making**: AI can assist in making better clinical decisions by analyzing large datasets and identifying patterns that may not be immediately apparent to human practitioners.\n   - **Increased Efficiency**: AI can automate routine tasks, allowing healthcare professionals to focus more on patient care.\n   - **Error Reduction**: AI can help reduce human errors in diagnosis and treatment plans.\n\n3. **Industry**:\n   - **Efficiency Improvement**: AI can streamline operations by automating repetitive tasks, leading to cost reduction and increased productivity.\n   - **Consistency**: AI can ensure consistent performance in tasks that require high precision and repeatability.\n   - **Cost Reduction**: By automating tasks, AI can help reduce labor costs and improve overall operational efficiency.\n\nThese benefits highlight the potential of AI to transform various sectors by enhancing decision-making, reducing errors, and increasing efficiency.', "The importance of using a representative and diverse dataset for training models like ChatGPT lies in mitigating the risks of bias and ensuring the accuracy and fairness of the model's output. If the training data is not representative or is contaminated by bias, the model's output can also become biased, leading to unfair or inaccurate results. Therefore, providing high-quality, representative, and diverse training data is crucial for the model to learn effectively and produce reliable and unbiased outputs."]

# Menimpa atau menambahkan kolom 'jawaban'
if "answer" in test_dataset.column_names:
    test_dataset = test_dataset.remove_columns(["answer"]).add_column("answer", answer)
else:
    test_dataset = test_dataset.add_column("answer", answer)

from ragas import evaluate
from ragas.metrics import (
    answer_relevancy,
    faithfulness,
    context_recall,
    context_precision,
)
 
result = evaluate(
    dataset=test_dataset,
    metrics=[
        context_precision,
        faithfulness,
        answer_relevancy,
        context_recall,
    ],
)
 
result

{'context_precision': 0.6000, 'faithfulness': 0.6334, 'answer_relevancy': 0.9664, 'context_recall': 0.8667}

result_df = result.to_pandas()
result_df.head()

                                            question                                           contexts  ... answer_relevancy context_recall
0  What ethical, practical, and societal consider...  [manusia, dan meningkatkan efisiensi. Namun, t...  ...         0.972534       1.000000
1  What specific benefits can artificial intellig...  [Simpulan\nKeuntungan teknologi kecerdasan bua...  ...         0.957387       1.000000
2  What is the importance of using a representati...  [menggunakan dataset yang lebih representatif ...  ...         0.992897       0.000000
3  What should be considered regarding faktor-fak...  [untuk memproses data secara cepat dan akurat,...  ...         0.937007       1.000000
4  How do data source credibility and ChatGPT usa...  [Oleh karena itu, sumber data yang digunakan o...  ...         0.964253       0.666667

[5 rows x 8 columns]

result_df.loc[:, "context_precision":"context_recall"]

   context_precision  faithfulness  answer_relevancy  context_recall
0                1.0      0.642857          0.972534        1.000000
1                0.0      0.166667          0.957387        1.000000
2                1.0      1.000000          0.992897        0.000000
3                0.0      0.600000          0.937007        1.000000
4                1.0      0.384615          0.971877        0.666667
5                1.0      0.384615          0.997298        1.000000
6                0.0      0.473684          0.986075        1.000000
7                1.0      1.000000          0.962303        1.000000
8                0.0      0.809524          0.968310        1.000000
9                1.0      0.529412          0.992118        1.000000