Neural Machine Translation - French to English

# import important libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import os
import string
from numpy import array
from pickle import dump
from unicodedata import normalize
import unicodedata
import unittest
import logging

import zipfile
from pathlib import Path

from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle

from nltk.tokenize import sent_tokenize, word_tokenize

from PIL import Image

import pickle
from tqdm import tqdm

# for preparing text data
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import EarlyStopping

# model-layers
from keras.layers import Input, LSTM, Dense, Embedding
from keras.models import Model

# plotting the defined model architecture
from keras.utils import plot_model

# word-cloud libraries
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

from nltk.translate.bleu_score import sentence_bleu
from nltk.translate.bleu_score import SmoothingFunction

%load_ext nb_black

from IPython.display import HTML

display(
    HTML("<style>.prompt{width: 0px; min-width: 0px; visibility: collapse}</style>")
)
# display(HTML("<style>.container { width:100% !important; }</style>"))

# declare constants needed for training the deep learning model.

# set the following parameter to True to limit the data used through the notebok
n_samples = False
# number of samples
NUM_SAMPLES = 20000

LATENT_DIM = 256
EPOCHS = 25
BATCH_SIZE = 25

# Initialise unittest instance object
tc = unittest.TestCase()

# Wordcloud paramters
MAX_WORDS = 600
MAX_FONT_SIZE = 120
version_name = "version1"

# initialize logger
logger = logging.getLogger("machine_translation")
logger.setLevel(logging.DEBUG)

# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")

# add formatter to ch
ch.setFormatter(formatter)

# add ch to logger
logger.addHandler(ch)

def download_data(
    version_name="version1", destination_dir=os.path.join(".", "data", "fra-eng")
):

    data_path = os.path.join(destination_dir, "fra.txt")
    if not os.path.exists(data_path):
        logger.debug(f"Creating directory tree - {destination_dir}")
        Path(destination_dir).mkdir(parents=True, exist_ok=True)

        !wget http://www.manythings.org/anki/fra-eng.zip

        filename = os.path.join(destination_dir, "fra-eng.zip")

        os.rename("fra-eng.zip", filename)

        with zipfile.ZipFile(filename, "r") as zip:
            logger.debug(f"Extracting the downloaded zip file - {filename} now...")
            zip.extractall(destination_dir)
            logger.debug(f"Done ! Downloaded {filename} at {destination_dir}")
    else:
        logger.debug(f"Data Available at the right path- {destination_dir}.")

    # create directory to save model weights

    if not os.path.exists(version_name):
        Path(version_name).mkdir(parents=True, exist_ok=True)

    return data_path


file_path = download_data()

raw_texts = pd.read_csv(
    file_path,
    encoding="utf-8",
    sep="\t",
    header=None,
    names=["target", "input", "comments"],
    usecols=["target", "input"],
)
display(raw_texts.head())
logger.debug(f"The shape of the data loaded : {raw_texts.shape}")

##visualising the French data using word cloud

cloud_text_input = str(list(raw_texts["input"]))
cloud_mask_input = np.array(Image.open("images/color1.jpg"))
cloud_title_input = "Most common words in French"

wordcloud = WordCloud(
    background_color="white",
    max_words=MAX_WORDS,
    max_font_size=MAX_FONT_SIZE,
    random_state=42,
    mask=cloud_mask_input,
)
wordcloud.generate(cloud_text_input)
plt.figure(figsize=(10, 10))
image_colors = ImageColorGenerator(cloud_mask_input)
plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation="bilinear")
plt.title(cloud_title_input, fontdict={"size": 20, "verticalalignment": "bottom"})
plt.axis("off")
plt.tight_layout()

##visualising the English data using word cloud

cloud_text_input = str(list(raw_texts["target"]))
cloud_mask_input = np.array(Image.open("images/color2.jpg"))
cloud_title_input = "Most common words in English"

wordcloud = WordCloud(
    background_color="white",
    max_words=MAX_WORDS,
    max_font_size=MAX_FONT_SIZE,
    random_state=42,
    mask=cloud_mask_input,
)
wordcloud.generate(cloud_text_input)
plt.figure(figsize=(10, 10))
image_colors = ImageColorGenerator(cloud_mask_input)
plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation="bilinear")

plt.title(cloud_title_input, fontdict={"size": 20, "verticalalignment": "bottom"})
plt.axis("off")
plt.tight_layout()

def preprocess_texts(raw_text):
    table = str.maketrans("", "", string.punctuation)
    re_print = re.compile("[^%s]" % re.escape(string.printable))
    clean_texts = []
    for data_line in raw_text:

        clean_text = (
            unicodedata.normalize("NFD", data_line)
            .encode("ascii", "ignore")
            .decode("utf8")
        )

        ##remove extra spaces
        clean_text = clean_text.split(" ")

        # change the words to lower
        clean_text = [word.lower() for word in clean_text]

        # remove punctuations
        clean_text = [word.translate(table) for word in clean_text]

        # remove non-printables
        clean_text = [re_print.sub("", w) for w in clean_text]

        clean_texts.append(" ".join(clean_text))

    return clean_texts

preprocessed_data_path = os.path.join(
    ".", "data", "fra-eng", version_name, "preprocessed_fra-eng.csv"
)
if not os.path.exists(preprocessed_data_path):

    preprocessed_data_dir = "/".join(preprocessed_data_path.split("/")[:-1])
    Path(preprocessed_data_dir).mkdir(parents=True, exist_ok=True)

    data_texts = raw_texts[["input", "target"]].apply(preprocess_texts, axis=0)

    logger.debug(f"Saving Preprocessed Data to {preprocessed_data_path}")
    data_texts.to_csv(preprocessed_data_path, index=False)

    if n_samples:
        data_texts = data_texts[:NUM_SAMPLES]
        tc.assertEqual(len(data_texts), NUM_SAMPLES)

else:
    data_texts = pd.read_csv(preprocessed_data_path)
    if n_samples:
        data_texts = data_texts[:NUM_SAMPLES]
        tc.assertEqual(len(data_texts), NUM_SAMPLES)

    logger.info(f"Read Preprocessed Data Successfully!")

# manipulate data to support Teacher Forcing.

data_texts["decoder_input"] = "<sos> " + data_texts["target"]
data_texts["decoder_target"] = data_texts["target"] + " <eos>"

display(data_texts.head())

data_train_texts, data_test = train_test_split(data_texts, test_size=0.2)
data_test_texts, data_valid_texts = train_test_split(data_test, test_size=0.5)
print(f"The Data Train split has {data_train_texts.shape[0]} records")
print(f"The Data Valid split has {data_valid_texts.shape[0]} records")
print(f"The Data Test split has {data_test_texts.shape[0]} records")

encoder_tokenizer_pickle_path = os.path.join(
    ".", "data", "fra-eng", version_name, "french_tokenizer.pkl"
)
decoder_tokenizer_pickle_path = os.path.join(
    ".", "data", "fra-eng", version_name, "english_tokenizer.pkl"
)

# train or load Tokenizers for encoder and decoder
if (not os.path.exists(encoder_tokenizer_pickle_path)) or (
    not os.path.exists(decoder_tokenizer_pickle_path)
):

    encoder_tokenizer = Tokenizer(filters="")
    encoder_tokenizer.fit_on_texts(data_train_texts["input"])

    decoder_tokenizer = Tokenizer(filters="")
    decoder_tokenizer.fit_on_texts("<sos> " + data_train_texts["target"] + " <eos>")

    with open(encoder_tokenizer_pickle_path, "wb") as handle:
        pickle.dump(encoder_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)

    with open(decoder_tokenizer_pickle_path, "wb",) as handle:
        pickle.dump(decoder_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)

else:

    with open(encoder_tokenizer_pickle_path, "rb") as handle:
        encoder_tokenizer = pickle.load(handle)

    with open(decoder_tokenizer_pickle_path, "rb",) as handle:
        decoder_tokenizer = pickle.load(handle)

# word to index and index to word mapping for encoder
encoder_word2idx = encoder_tokenizer.word_index
encoder_idx2word = dict(map(reversed, encoder_word2idx.items()))

# word to index and index to word mapping for decoder
decoder_word2idx = decoder_tokenizer.word_index
decoder_idx2word = dict(map(reversed, decoder_word2idx.items()))

encoder_num_words = len(encoder_word2idx) + 1
logger.debug(f"The length of the encoder tokenizer dictionary is : {encoder_num_words}")

decoder_num_words = len(decoder_word2idx) + 1
logger.debug(f"The length of the encoder tokenizer dictionary is : {decoder_num_words}")

def max_length(desc1):
    length = []
    for sentence in desc1:
        length.append(len(word_tokenize(sentence)))
    return max(length)

encoder_max_length_pickle_path = os.path.join(
    ".", "data", "fra-eng", version_name, "french_max_len.pkl"
)
decoder_max_length_pickle_path = os.path.join(
    ".", "data", "fra-eng", version_name, "english_max_len.pkl"
)
if (not os.path.exists(encoder_max_length_pickle_path)) or (
    not os.path.exists(decoder_max_length_pickle_path)
):
    encoder_max_len = max_length(data_train_texts["input"])
    decoder_max_len = max_length(data_train_texts["decoder_input"])

    with open(encoder_max_length_pickle_path, "wb") as handle:
        pickle.dump(encoder_max_len, handle, protocol=pickle.HIGHEST_PROTOCOL)
        logger.debug(f"Encoder max length saved at {encoder_max_length_pickle_path}")

    with open(decoder_max_length_pickle_path, "wb") as handle:
        pickle.dump(decoder_max_len, handle, protocol=pickle.HIGHEST_PROTOCOL)
        logger.debug(f"Decoder max length saved at {decoder_max_length_pickle_path}")

else:

    with open(encoder_max_length_pickle_path, "rb") as handle:
        encoder_max_len = pickle.load(handle)

    with open(decoder_max_length_pickle_path, "rb") as handle:
        decoder_max_len = pickle.load(handle)

logger.debug(f"Maximum Encoder seq length is {encoder_max_len}")
logger.debug(f"Maximum decoder Seq length is {decoder_max_len}")

def load_train_batches(data_train_texts, chunk_size=BATCH_SIZE):

    index_tracker = 0

    while True:

        # retrieve the batch of input texts
        input_train_text = list(data_train_texts["input"])[
            index_tracker : index_tracker + chunk_size
        ]
        # convert the extracted batch to sequences
        train_x = encoder_tokenizer.texts_to_sequences(input_train_text)
        # pad the input sequences to the max length
        train_x = pad_sequences(train_x, maxlen=encoder_max_len, padding="post")

        # retrieve the batch of decoder-input texts
        target_train_input = list(data_train_texts["decoder_input"])[
            index_tracker : index_tracker + chunk_size
        ]
        # convert the extracted batch to sequences
        train_y = decoder_tokenizer.texts_to_sequences(target_train_input)
        # pad the sequences to the max length
        train_y = pad_sequences(train_y, maxlen=decoder_max_len, padding="post")

        # retrieve the batch of decoder-input texts
        target_train = list(data_train_texts["decoder_target"])[
            index_tracker : index_tracker + chunk_size
        ]
        # convert the extracted batch to sequences
        train_z = decoder_tokenizer.texts_to_sequences(target_train)
        # pad the sequences to the max length
        train_z = pad_sequences(train_z, maxlen=decoder_max_len, padding="post")

        decoder_targets_one_hot = np.zeros(
            (train_x.shape[0], decoder_max_len, decoder_num_words), dtype="float32"
        )
        # assign the values
        for i, d in enumerate(train_z):
            for t, word in enumerate(d):
                decoder_targets_one_hot[i, t, word] = 1

        index_tracker += chunk_size
        if index_tracker >= len(data_train_texts):
            index_tracker = 0
            data_train_texts = shuffle(data_train_texts).reset_index(drop=True)

        yield [train_x, train_y], decoder_targets_one_hot


def load_valid_batches(data_valid_texts, chunk_size=BATCH_SIZE):
    valid_index_tracker = 0

    while True:

        # retrieve the batch of input texts
        input_valid_text = list(data_valid_texts["input"])[
            valid_index_tracker : valid_index_tracker + chunk_size
        ]
        # convert the extracted batch to sequences
        valid_x = encoder_tokenizer.texts_to_sequences(input_valid_text)
        # pad the input sequences to the max length
        valid_x = pad_sequences(valid_x, maxlen=encoder_max_len, padding="post")

        # retrieve the batch of decoder-input texts
        target_valid_input = list(data_valid_texts["decoder_input"])[
            valid_index_tracker : valid_index_tracker + chunk_size
        ]
        # convert the extracted batch to sequences
        valid_y = decoder_tokenizer.texts_to_sequences(target_valid_input)
        # pad the sequences to the max length
        valid_y = pad_sequences(valid_y, maxlen=decoder_max_len, padding="post")

        # retrieve the batch of decoder-input texts
        target_valid = list(data_valid_texts["decoder_target"])[
            valid_index_tracker : valid_index_tracker + chunk_size
        ]
        # convert the extracted batch to sequences
        valid_z = decoder_tokenizer.texts_to_sequences(target_valid)
        # pad the sequences to the max length
        valid_z = pad_sequences(valid_z, maxlen=decoder_max_len, padding="post")

        decoder_targets_one_hot_valid = np.zeros(
            (valid_x.shape[0], decoder_max_len, decoder_num_words), dtype="float32"
        )
        # assign the values
        for j, d1 in enumerate(valid_z):
            for k, word in enumerate(d1):
                decoder_targets_one_hot_valid[j, k, word] = 1

        valid_index_tracker += chunk_size

        if valid_index_tracker >= len(data_valid_texts):
            valid_index_tracker = 0
            data_valid_texts = shuffle(data_valid_texts).reset_index(drop=True)

        yield [valid_x, valid_y], decoder_targets_one_hot_valid

train_gen = load_train_batches(data_train_texts, chunk_size=BATCH_SIZE)
valid_gen = load_valid_batches(data_valid_texts, chunk_size=BATCH_SIZE)

# ENCODER
input_len = (encoder_max_len,)
encoder_input_layer = Input(shape=input_len)
# pass the input to emedding layer
encoder_embedding_layer = Embedding(encoder_num_words, LATENT_DIM)
encoder_embedding = encoder_embedding_layer(encoder_input_layer)
# Create a LSTM encoder and pass the input embeddings to it
encoder_lstm = LSTM(LATENT_DIM, return_state=True)
encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding)
# save the hidden states of the encoder
encoder_states = [state_h, state_c]

# DECODER
target_len = (decoder_max_len,)
decoder_input_layer = Input(shape=target_len)
# pass the input to emedding layer
decoder_embedding_layer = Embedding(decoder_num_words, LATENT_DIM)
decoder_embedding = decoder_embedding_layer(decoder_input_layer)
# Create a LSTM encoder and pass the input embeddings to it
# return states in the training model, but we will use them in inference.
decoder_lstm = LSTM(LATENT_DIM, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)


decoder_dense = Dense(decoder_num_words, activation="softmax")
decoder_outputs = decoder_dense(decoder_outputs)

model = Model([encoder_input_layer, decoder_input_layer], decoder_outputs)

plot_model(model, show_shapes=True)

# Compile the deep learning model
model.compile(
    optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"]
)

if os.path.exists("modelv1_loss0.32.h5"):
    model.load_weights("modelv1_loss0.32.h5")
else:
    
    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
    machine_translate=model.fit_generator(train_gen,
                                          steps_per_epoch=int(np.ceil(data_train_texts.shape[0] / BATCH_SIZE)),
                                          validation_data=valid_gen,
                                          validation_steps=int(np.ceil(data_valid_texts.shape[0] / BATCH_SIZE)),
                                          callbacks=[es],
                                          epochs=EPOCHS,
                                          verbose=1)
    model.save('modelv1_loss0.32.h5')
    ## summarize history for accuracy 
    plt.plot(machine_translate.history['accuracy'])
    plt.plot(machine_translate.history['val_accuracy'])
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(['train', 'valid'], loc='upper right')
    plt.show()

    # summarize history for loss
    plt.plot(machine_translate.history['loss'])
    plt.plot(machine_translate.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'valid'], loc='upper right')
    plt.show()

encoder_model = Model(encoder_input_layer, encoder_states)

state_input_h = Input(shape=(LATENT_DIM,))
state_input_c = Input(shape=(LATENT_DIM,))
states_inputs = [state_input_h, state_input_c]

decoder_inputs_word = Input(shape=(1,))

decoder_inputs_word_x = decoder_embedding_layer(decoder_inputs_word)

decoder_outputs, state_h, state_c = decoder_lstm(
    decoder_inputs_word_x, initial_state=states_inputs
)

decoder_states = [state_h, state_c]

decoder_outputs = decoder_dense(decoder_outputs)

decoder_model = Model(
    [decoder_inputs_word] + states_inputs, [decoder_outputs] + decoder_states
)

def decode_sequence1(test_sen):
    # Encode the input as state vectors.
    states_value = encoder_model.predict(test_sen)

    target_seq = np.zeros((1, 1))
    target_seq[0, 0] = decoder_word2idx['<sos>']

    eos = decoder_word2idx['<eos>']
    output_sentence = []

    for _ in range(decoder_max_len):
        
        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)
        idx = np.argmax(output_tokens[0, 0, :])
        if eos == idx:
            break

        word = ''
        if idx > 0:
            word = decoder_idx2word[idx]
            output_sentence.append(word)

        # Update the decoder input
        # which is just the word just generated
        target_seq[0, 0] = idx

        # Update states
        states_value = [h, c]

    return (" ".join(output_sentence))

def evaluate_test_data(data_test_texts):

    predicted_results = pd.DataFrame(
        columns=["Input_Sentence", "Target_Sentence", "Predicted_Sentence"]
    )

    for seq_idx in range(len(data_test_texts)):
        # retrieve the batch of input texts
        input_test_text = list(data_test_texts["input"])[seq_idx : seq_idx + 1]
        target_test_text = list(data_test_texts["target"])[seq_idx : seq_idx + 1]
        # convert the extracted batch to sequences
        test_x = encoder_tokenizer.texts_to_sequences(input_test_text)
        # pad the input sequences to the max length
        test_x = pad_sequences(test_x, maxlen=encoder_max_len, padding="post")

        decoded_sentence = decode_sequence1(test_x)

        predicted_results.loc[seq_idx] = [
            data_test_texts.iloc[seq_idx]["input"],
            data_test_texts.iloc[seq_idx]["target"],
            decoded_sentence,
        ]
    return predicted_results

predicted_results=evaluate_test_data(data_test_texts)

display(predicted_results[:10])

def calculate_sent_bleu(predicted_results):
    
    # initialize lists for tracking bleu scores
    bleu_1, bleu_2, bleu_3, bleu_4 = [list()] * 4
    for i in range(len(predicted_results)):
        hypothesis=predicted_results.iloc[i]['Target_Sentence'].split()
        list_of_ref=[predicted_results.iloc[i]['Predicted_Sentence'].split()]
        bleu_1.append(sentence_bleu(list_of_ref, hypothesis,weights=(1, 0, 0, 0),smoothing_function=SmoothingFunction().method2))
        bleu_2.append(sentence_bleu(list_of_ref, hypothesis,weights=(0.5, 0.5, 0, 0),smoothing_function=SmoothingFunction().method2))
        bleu_3.append(sentence_bleu(list_of_ref, hypothesis,weights=(0.33, 0.33, 0.33, 0),smoothing_function=SmoothingFunction().method2))
        bleu_4.append(sentence_bleu(list_of_ref, hypothesis,weights=(0.25, 0.25, 0.25, 0.25),smoothing_function=SmoothingFunction().method2))
        
    predicted_results['Bleu_1']=bleu_1
    predicted_results['Bleu_2']=bleu_2
    predicted_results['Bleu_3']=bleu_3
    predicted_results['Bleu_4']=bleu_4
    
    return predicted_results

result_try=calculate_sent_bleu(predicted_results)

print("BlEU-1 :  ", result_try["Bleu_1"].mean())
print("BlEU-2 :  ", result_try["Bleu_2"].mean())
print("BlEU-3 :  ", result_try["Bleu_3"].mean())
print("BlEU-4 :  ", result_try["Bleu_4"].mean())

# predicted_results.to_csv("./data/fra-eng/version1/modelv1_results.csv")