dl_model.py

# -*- coding: utf-8 -*-
"""DL MODEL.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1XMt1Iddzp5AwlxV6pfLrRoxxueEkR5eI
"""

pip install Arabic-Stopwords

import pandas as pd
import re
from tqdm import tqdm
from snowballstemmer import stemmer
import arabicstopwords.arabicstopwords as ar_stp
import json
import random

def clean(text):
    text = re.sub(r"http\S+", " ", text)  # remove urls
    text = re.sub(r"@[\w]*", " ", text)  # remove handles
    text = re.sub(r"[\.\,\#_\|\:\?\?\/\=]", " ", text) # remove special characters
    text = re.sub(r"\t", " ", text)  # remove tabs
    text = re.sub(r"\n", " ", text)  # remove line jump
    text = re.sub(r"\s+", " ", text)  # remove extra white space
    text = re.sub(r'[^\w\s]', '', text) # Removing punctuations in string using regex
    text = text.strip()
    return text

# arabic stemmer
ar_stemmer = stemmer("arabic")

# remove arabic stop words
def ar_remove_stop_words(sentence):
    terms=[]
    stopWords= set(ar_stp.stopwords_list())
    for term in sentence.split() :
        if term not in stopWords :
            terms.append(term)
    return " ".join(terms)


# normalize the arabic text
def normalize_arabic(text):
    text = re.sub("[إأٱآا]", "ا", text)
    text = re.sub("ى", "ي", text)
    text = re.sub("ؤ", "ء", text)
    text = re.sub("ئ", "ء", text)
    text = re.sub("ة", "ه", text)
    return(text)

# stem the arabic text
def ar_stem(sentence):
    return " ".join([ar_stemmer.stemWord(i) for i in sentence.split()])

def preprocess_arabic_train(text):
    text = normalize_arabic(text)
    #text = ar_remove_stop_words(text)
    text = ar_stem(text)
    return text

passages=pd.read_csv('passages.csv')
trainQuestions=pd.read_csv('trainQuestions.csv')
devQuestions=pd.read_csv('devQuestions.csv')
testQuestions=pd.read_csv('testQuestions.csv')

trainQuestions['question']=trainQuestions['question'].apply(lambda x : clean(x))
testQuestions['question']=testQuestions['question'].apply(lambda x: clean(x))
devQuestions['question']=devQuestions['question'].apply(lambda x: clean(x))

trainQuestions['question']=trainQuestions['question'].apply(lambda x : preprocess_arabic_train(x))
testQuestions['question']=testQuestions['question'].apply(lambda x: preprocess_arabic_train(x))
devQuestions['question']=devQuestions['question'].apply(lambda x: preprocess_arabic_train(x))

with open('goldDev.json','r') as file:
    goldDev=json.load(file)
with open('goldTrain.json','r') as file:
    goldTrain=json.load(file)
with open('goldTest.json','r') as file:
    goldTest=json.load(file)

for i in goldDev:
    for e in goldDev[i]:

        print(i,e)

def prepareData(trainQuestions,goldTrain):
    trainQAPairs = []
    trainID = [x for x in trainQuestions['qID']]
    trainAnswers = [int(x) for x in goldTrain.keys()]
    trainID.sort()
    trainAnswers.sort()

    for q, a in zip(trainID, trainAnswers):
        if q == a:
            if str(a) in goldTrain:
                for p in goldTrain[str(a)]:
                    #Check if the passageID exists in passages DataFrame
                    if p in passages['passageID'].values:
                        pair = [
                            trainQuestions.loc[trainQuestions['qID'] == q]['question'].iloc[0],
                            q,
                            passages.loc[passages['passageID'] == str(p)]['passage'].iloc[0],
                            p
                        ]
                        trainQAPairs.append(pair)
                    else:
                        pair=[
                            trainQuestions.loc[trainQuestions['qID'] == q]['question'].iloc[0],
                            q,
                            'None',
                            'None'
                        ]
                        trainQAPairs.append(pair)
    trainQAPairs=pd.DataFrame(trainQAPairs,columns=['question','qID','relPassage','passageID'])
    notRel=[]
    for q in trainQAPairs['qID']:
        relList=trainQAPairs[trainQAPairs['qID']==q]['passageID']
        while(True):
            randomPassage=passages['passageID'].iloc[random.randint(0,len(passages)-1)]
            if randomPassage in relList:
                continue
            notRel.append(passages[passages['passageID']==randomPassage]['passage'].values[0])
            break
    trainQAPairs['notRel']=notRel
    return trainQAPairs

trainData=prepareData(trainQuestions,goldTrain)
devData=prepareData(devQuestions,goldDev)
testData=prepareData(testQuestions,goldTest)

trainData

from transformers import BertTokenizer
import torch
from transformers import BertForNextSentencePrediction, BertTokenizer
from torch.utils.data import DataLoader, Dataset
import tqdm
from torch.optim.lr_scheduler import StepLR

from sklearn.metrics.pairwise import cosine_similarity
from transformers import BertTokenizer, BertModel

# Define a Siamese-like architecture using BERT for passage retrieval
class PassageRetrievalModel(torch.nn.Module):
    def __init__(self):
        super(PassageRetrievalModel, self).__init__()
        self.bert = BertModel.from_pretrained('asafaya/bert-medium-arabic')
        self.cos_sim = torch.nn.CosineSimilarity(dim=1)

    def forward(self, query_ids, passage_ids, query_attention_mask, passage_attention_mask):
        query_output = self.bert(input_ids=query_ids,attention_mask=query_attention_mask)[1]  # Using [1] to get the pooled output
        passage_output = self.bert(input_ids=passage_ids,attention_mask=passage_attention_mask)[1]
        similarity_scores = self.cos_sim(query_output, passage_output)
        return similarity_scores

class trainRetrievalDataset(Dataset):
    def __init__(self, tokenizer, data, max_length):
        self.tokenizer = tokenizer
        self.data = data
        self.max_length=max_length

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        query, relevant_passage, irrelevant_passage = self.data[idx]

        query=str(query)
        relevant_passage=str(relevant_passage)
        irrelevant_passage=str(irrelevant_passage)

        qEncoding = self.tokenizer.encode_plus(
            query,
            add_special_tokens=True,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_attention_mask=True,
            return_tensors='pt'
        )

        rEncoding=self.tokenizer.encode_plus(
            relevant_passage,
            add_special_tokens=True,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_attention_mask=True,
            return_tensors='pt'
        )

        iEncoding=self.tokenizer.encode_plus(
            irrelevant_passage,
            add_special_tokens=True,
            max_length=self.max_length,
            padding='max_length',
            truncation=True,
            return_attention_mask=True,
            return_tensors='pt'
        )

        return {
            'query_ids': qEncoding['input_ids'].squeeze(0),
            'relevant_passage_ids': rEncoding['input_ids'].squeeze(0),
            'irrelevant_passage_ids': iEncoding['input_ids'].squeeze(0),
            'qAttention': qEncoding['attention_mask'].squeeze(0),
            'rAttention': rEncoding['attention_mask'].squeeze(0),
            'iAttention': iEncoding['attention_mask'].squeeze(0)
        }

tokenizer = BertTokenizer.from_pretrained('asafaya/bert-medium-arabic')
model = PassageRetrievalModel()

trainQ=[x for x in trainData['question']]
trainR=[x for x in trainData['relPassage']]
trainI=[x for x in trainData['notRel']]
t=[]
for q,r,i in zip(trainQ,trainR,trainI):
    t.append((q,r,i))

train_dataset = trainRetrievalDataset(tokenizer,t,max_length=512)
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)

optimizer = torch.optim.AdamW(model.bert.parameters(), lr=3e-5)
scheduler = StepLR(optimizer, step_size=1, gamma=0.1)
criterion = torch.nn.MSELoss()

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
model.bert.train()
num_epochs = 10
for epoch in range(num_epochs):
    total_loss = 0
    for batch in tqdm.tqdm(train_loader, desc=f"Epoch {epoch + 1}/{num_epochs}"):
        query_ids = batch['query_ids'].to(device)
        relevant_passage_ids = batch['relevant_passage_ids'].to(device)
        irrelevant_passage_ids = batch['irrelevant_passage_ids'].to(device)

        qAttention = batch['qAttention'].to(device)
        rAttention = batch['rAttention'].to(device)
        iAttention = batch['iAttention'].to(device)


        # Compute similarity scores
        relevant_similarity = model(query_ids, relevant_passage_ids,qAttention,rAttention)**2
        irrelevant_similarity = model(query_ids, irrelevant_passage_ids,qAttention,iAttention)**2

        loss_tensor = relevant_similarity - irrelevant_similarity + 0.2
        loss = torch.max(torch.tensor(0), loss_tensor).mean()
        total_loss+=loss.item()

        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        scheduler.step()

    print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {total_loss / len(train_loader):.4f}")

model.bert.save_pretrained('passage retrieval model')

passageIndex=[x for x in passages['passage']]
passageIDs=[x for x in passages['passageID']]

tokenizedIndex=tokenizer.encode_plus(
        passageIndex,
        add_special_tokens=True,
        max_length=512,
        padding='max_length',
        truncation=True,
        return_attention_mask=True,
        return_tensors='pt')
indexIds = tokenizedIndex['input_ids'].to(device)
indexAttentionMask = tokenizedIndex['attention_mask'].to(device)

question=[x for x in devQuestions['question']]

tokenizedQuestions=tokenizer.encode_plus(
        question[5],
        add_special_tokens=True,
        max_length=512,
        padding='max_length',
        truncation=True,
        return_attention_mask=True,
        return_tensors='pt')
questionIds = tokenizedQuestions['input_ids'].to(device)
questionAttentionMask = tokenizedQuestions['attention_mask'].to(device)

similarities=[]
model.eval()
for i in tqdm.tqdm(range(20)):
    with torch.no_grad():
        tokenizedIndex=tokenizer.encode_plus(
        passageIndex[i],
        add_special_tokens=True,
        max_length=512,
        padding='max_length',
        truncation=True,
        return_attention_mask=True,
        return_tensors='pt')
        indexIds = tokenizedIndex['input_ids'].to(device)
        indexAttentionMask = tokenizedIndex['attention_mask'].to(device)
        relevant_similarity =torch.tensor(model(questionIds, indexIds,questionAttentionMask,indexAttentionMask))
        if relevant_similarity.item()>0.7:
            similarities.append([relevant_similarity.item(),passageIDs[i]])

similarities.sort(reverse=True)

def answer_question(question, context):
    inputs = tokenizer.encode_plus(question, context, add_special_tokens=True, return_tensors='pt')
    output = model(**inputs)
    answer_start = torch.argmax(output.start_logits)
    answer_end = torch.argmax(output.end_logits) + 1
    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs['input_ids'][0][answer_start:answer_end]))

    return answer

#loop through a list of questions for inference
for question in testQuestions:
    context = "..."
    answer = answer_question(question, context)
with open('vendor/data-model.json', 'r', encoding='utf-8') as source_file:
    source_data = json.load(source_file)

#create a new json data structure
new_json_data = source_data


# write the new json data to a file
with open('data-model-result.json', 'w', encoding='utf-8') as new_file:
    json.dump(new_json_data, new_file, indent=4, ensure_ascii=False)


print("Model generation is done")