# Module imports
import subprocess
from nltk.tokenize import sent_tokenize, regexp_tokenize
# internal imports
from conversion.translation.script_writing import dial_checker, dual_script, addDialogue, default_script
from conversion.translation.translation_variables import is_in_translation_list, special_characters, code_2_language
from conversion.translation.all_transliteration import all_transliteration
from conversion.translation.script_writing import default_script
from conversion.translation.translation_metric import manual_diff_score, bleu_diff_score, gleu_diff_score, meteor_diff_score, rouge_diff_score, diff_score, critera4_5
from conversion.translation.selection_source import selection_source, function5, function41, function311, function221, function2111, function11111, selection_source_transliteration, two_sources_two_outputs
from conversion.translation.translation_resources import google, aws, azure, yandex
from .detection import language_detector, script_det
# -> Random Function
def word_transliterate(sentence, dest_script):
return sentence
# # -> to be used when option 5 or 6 is yes from frontend then this function is used for translation
# def ui_option5_and_6(doc, dial_src_lang, dial_dest_lang, dialogue, dual_dial_script):
# if dialogue == "":
# return
# dial_translate = dial_checker(dial_dest_lang, dial_src_lang)
# if dial_translate:
# print("in 51")
# if is_in_translation_list(dial_src_lang) and is_in_translation_list(dial_dest_lang):
# trans_text = ui_option5_translate_comparison(
# dialogue, dial_src_lang, dial_dest_lang)
#
# if dual_dial_script == "Yes":
# dual_script(doc, dialogue,
# trans_text, dial_dest_lang)
# else:
# addDialogue(doc, trans_text, dial_dest_lang)
# else:
# print("in 52")
# if dual_dial_script == "Yes":
# dual_script(doc, dialogue,
# dialogue, dial_dest_lang)
# else:
# addDialogue(doc, dialogue, dial_dest_lang)
# # -> If option 3 or 4 is yes from frontend then this function is used for translation
def ui_option3_and_4(dial_src_lang, dial_dest_lang, dialogue, ui_option_1st_choice, dual_dial_script):
print("dial_src_lang", dial_src_lang)
print("dial_dest_lang", dial_dest_lang)
print("ui option 1st choice", ui_option_1st_choice)
if ui_option_1st_choice == "Yes":
print("in ui31")
dial_translate = dial_checker(dial_dest_lang, dial_src_lang)
if dial_translate:
print("in ui311")
if dialogue == "":
return
print(dial_src_lang, dial_dest_lang, "fjdjfskd")
if is_in_translation_list(dial_src_lang) and is_in_translation_list(dial_dest_lang):
print("case 1")
trans_text = translate_comparison(
dialogue, dial_src_lang, dial_dest_lang)
print("the translated text is", trans_text)
else:
trans_text = dialogue
print("no creation of ")
if dual_dial_script == "Yes":
print("case 1")
# dual_script(doc, dialogue,
# trans_text, dial_dest_lang)
else:
try:
if trans_text == "":
trans_text = dialogue
else:
trans_text = trans_text
except:
trans_text = dialogue
return trans_text
else:
print("in ui312")
if dual_dial_script == "Yes":
pass
# dual_script(doc, dialogue,
# dialogue, dial_dest_lang)
else:
return dialogue
else:
print("in ui32")
print(dialogue, script_det(
dialogue), dial_src_lang)
output = all_transliteration(dialogue, script_det(
dialogue), default_script[dial_src_lang])
if dual_dial_script == "Yes":
pass
# transliteration
# dual_script(doc, dialogue, output, dial_dest_lang)
else:
return output
# addDialogue(doc, output, dial_dest_lang)
# -> Converting Docx to PDF using Libra-Office
def convert_to_pdf(input_docx, out_folder):
p = subprocess.Popen(['libreoffice', '--headless', '--convert-to', 'pdf', '--outdir',
out_folder, input_docx])
print(['--convert-to', 'pdf', input_docx])
p.communicate()
# saveFile = input_docx.split('.')[0] + ".pdf"
# change_chmod = subprocess.Popen(['sudo', 'chmod', '777', saveFile])
# change_chmod.communicate()
def final_out(output1, output2, output3, dest_lang):
# for word in output1.split():
for word in regexp_tokenize(output1, "[\w']+"):
if script_det(word) != default_script[dest_lang]:
for word in regexp_tokenize(output2, "[\w']+"):
if script_det(word) != default_script[dest_lang]:
for word in regexp_tokenize(output3, "[\w']+"):
if script_det(word) != default_script[dest_lang]:
# print("in3")
output1 = word_transliterate(
output1, default_script[dest_lang])
return output1
return output3
return output2
return output1
def manual_diff_score(trans, sources_name):
global_diff = []
n=len(sources_name)
for i in range(n):
local_diff = 0
for j in range(n):
if i!=j:
if trans[str(i)] and trans[str(j)] == " ":
continue
d = diff_score(trans[str(i)], trans[str(j)])
local_diff += d
global_diff.append(local_diff/(n-1))
Choiced_source = global_diff.index(min(global_diff))
return trans[str(Choiced_source)], sources_name[str(Choiced_source)]
# -> Comapre Outputs from all sources like google,ibm,aws,etc and decides the text to be returned as output
def compare_outputs(sentence, t0, trans, sources_name, target_lang):
# take a sentence and give translated sentence by comparing outputs from different resources
k = []
s = []
methods_name = {'0': 'MNF', '1': 'Gleu',
'2': 'Meteor', '3': 'Rougen', '4': 'Rougel'}
google_output = t0
#print("google", google_output)
output1, source1 = manual_diff_score(trans, sources_name)
#print("MNF", output1)
output2, source2 = gleu_diff_score(trans, sources_name)
#print("gleu", output2)
print("TRans 2-> ",trans)
print(type(trans))
output3, source3 = meteor_diff_score(trans, sources_name)
#print("meteor", output3)
output4, source4, output5, source5 = rouge_diff_score(
trans, sources_name)
#print("rougen", output4)
#print("rougel", output5)
if google_output == output1 == output2 == output3 == output4 == output5:
print("all output is same as google")
return google_output
else:
if google_output != output1:
k.append(output1)
s.append(source1)
else:
k.append(" ")
s.append(" ")
if google_output != output2:
k.append(output2)
s.append(source2)
else:
k.append(" ")
s.append(" ")
if google_output != output3:
k.append(output3)
s.append(source3)
else:
k.append(" ")
s.append(" ")
if google_output != output4:
k.append(output4)
s.append(source4)
else:
k.append(" ")
s.append(" ")
if google_output != output5:
k.append(output5)
s.append(source5)
else:
k.append(" ")
s.append(" ")
k.insert(0, sentence)
k.insert(1, google_output)
s1ANDm1, s2ANDm2, s3ANDm3 = selection_source(
s, sources_name, trans, methods_name)
for a, b in sources_name.items():
if b == s1ANDm1[0]:
k = a
output1 = trans[str(k)]
if s2ANDm2[0] != "":
for c, d in sources_name.items():
if d == s2ANDm2[0]:
l = c
output2 = trans[str(l)]
else:
output2 = output1
if s3ANDm3[0] != "":
for e, f in sources_name.items():
if f == s3ANDm3[0]:
m = e
output3 = trans[str(m)]
else:
output3 = output1
output = final_out(output1, output2, output3, target_lang)
return output
# -> Defining own way of declaring Dictionary
class myDict(dict):
def __init__(self):
self = dict()
def add(self, key, value):
self[key] = value
# -> Main Translation function to be called without any special dots in Sentence
def all_translator(sentence, source_lang, target_lang, makeExcel=False):
import time
i = 0
trans = myDict()
sources_name = myDict()
try:
globals()['t%s' % i] = google(
sentence, source_lang, target_lang)
trans.add(str(i), globals()['t%s' % i])
sources_name.add(str(i), "GOOGLE")
i = i+1
except:
pass
# try:
# globals()['t%s' % i] = ibm_watson(
# sentence, source_lang, target_lang)
# trans.add(str(i), globals()['t%s' % i])
# sources_name.add(str(i), "IBM_WATSON")
# i = i+1
# except:
# pass
try:
globals()['t%s' % i] = aws(sentence, source_lang, target_lang)
trans.add(str(i), globals()['t%s' % i])
sources_name.add(str(i), "AWS")
i = i+1
except:
pass
try:
globals()['t%s' % i] = azure(sentence, target_lang)
trans.add(str(i), globals()['t%s' % i])
sources_name.add(str(i), "AZURE")
i = i+1
except:
pass
# try:
# globals()['t%s' % i] = lingvanex(
# sentence, source_lang, target_lang)
# trans.add(str(i), globals()['t%s' % i])
# sources_name.add(str(i), "LINGVANEX")
# i = i+1
# except:
# pass
#
# try:
# globals()['t%s' % i] = yandex(
# sentence, source_lang, target_lang)
# trans.add(str(i), globals()['t%s' % i])
# sources_name.add(str(i), "YANDEX")
# i = i+1
# except:
# pass
if len(sources_name) == 1:
trans_text = trans["0"]
else:
print("Trans -> ", trans)
print(type(trans))
trans_text = compare_outputs(
sentence, trans["0"], trans, sources_name, target_lang)
# print("final trasnlated text 101", trans_text)
if makeExcel:
print("Translated texts are",trans)
return trans_text, str(rf"{sentence} | ".join(list(trans.values())))
else:
return trans_text
# -> Main Translation function to be called with any special dots in Sentence (TRY TO USE THIS FUNCTION FOR TRANSLATION)
def translate_comparison(text, source_lang, target_lang, makeExcel=False):
print(text, " : Text at 58%")
sentences = sent_tokenize(text)
translated_text = []
for sentence in sentences:
if any(ext in sentence for ext in special_characters):
print("Isme gaya")
trans_text = translation_with_spcecial_dots(
sentence, source_lang, target_lang)
translated_text.append(trans_text)
else:
if makeExcel:
trans_text = all_translator(
sentence, source_lang, target_lang, makeExcel)
translated_text.append(trans_text)
else:
trans_text = all_translator(
sentence, source_lang, target_lang, makeExcel)
translated_text.append(trans_text)
if makeExcel:
return " ".join(translated_text), str(trans)
else:
return " ".join(translated_text)
# -> Handling all special dots in sentence
# -> Main functio for handling sentences to remove recursive dots
def recursive_dots(Sentence, source_lang, target_lang):
translated_text = []
for i in special_characters:
if i not in Sentence:
continue
Sentences = Sentence.split(i)
for Sentence in Sentences:
if Sentence == "" or Sentence == " ":
continue
if any(ext in Sentence for ext in special_characters):
trans_text = translation_with_spcecial_dots(
Sentence, source_lang, target_lang)
else:
if Sentence != Sentences[-1]:
trans_text = all_translator(
Sentence, source_lang, target_lang) + i
else:
trans_text = all_translator(
Sentence, source_lang, target_lang)
translated_text.append(trans_text)
return " ".join(translated_text)
def translation_with_spcecial_dots(text, source_lang, target_lang, splitter, line_language, line_script, script_data=None, subsentence_choices=None):
sentences = text.split(splitter)
translated_text = []
for sentence in sentences:
if sentence == "" or sentence == " ":
continue
line_language2 = language_detector(sentence)
line_script2 = script_det(sentence)
if splitter2 := next((ext for ext in special_characters if ext in sentence), None):
trans_text = translation_with_spcecial_dots(
sentence, source_lang, target_lang, splitter2, line_language2, line_script2)
else:
if line_language == script_data['dial_dest_lang'] \
and line_script == script_data['dial_dest_script']:
if subsentence_choices[0]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['dial_dest_lang'] \
and line_script == script_data['non_dial_dest_script']:
if subsentence_choices[2]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_src_lang'] \
and line_script == script_data['dial_dest_script']:
if subsentence_choices[4]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_src_lang'] \
and line_script == script_data['non_dial_dest_script']:
if subsentence_choices[6]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_dest_lang'] \
and line_script == script_data['dial_dest_script']:
if subsentence_choices[8]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_dest_lang'] \
and line_script == script_data['non_dial_dest_script']:
if subsentence_choices[10]:
trans_text = all_translator(
sentence, source_lang, target_lang)
else:
if subsentence_choices[12]:
trans_text = all_translator(
sentence, source_lang, target_lang)
if sentence != sentences[-1]:
trans_text = trans_text + splitter
else:
trans_text = all_translator(
sentence, source_lang, target_lang)
translated_text.append(trans_text)
return " ".join(translated_text)
def translate_comparison2(text, source_lang, target_lang, script_data=None, subsentence_choices=None, is_dialogue=False):
if is_dialogue:
tokenizers_valid_langs = ("malayalam", "french", "italian", "german", "spanish", "swedish", "finnish", "danish", "english", "slovene", "norwegian", "dutch", "portuguese", "czech", "russian", "polish","turkish", "estonian", "greek")
try:
if code_2_language[source_lang].lower() in tokenizers_valid_langs:
sentences = sent_tokenize(text, language=(code_2_language[source_lang]).lower())
else:
sentences = text.split(".")
except Exception as e:
print("Error was : ", e)
return text
# List of translated sentences
translated_text = []
# Translating each sentence one by one
for sentence in sentences:
line_language = language_detector(sentence)
line_script = script_det(sentence)
if splitter := next((ext for ext in special_characters if ext in sentence), None):
trans_text = translation_with_spcecial_dots(
sentence, source_lang, target_lang, splitter, line_language, line_script, script_data, subsentence_choices)
translated_text.append(trans_text)
else:
if line_language == script_data['dial_dest_lang'] \
and line_script == script_data['dial_dest_script']:
if subsentence_choices[0]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['dial_dest_lang'] \
and line_script == script_data['non_dial_dest_script']:
if subsentence_choices[2]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_src_lang'] \
and line_script == script_data['dial_dest_script']:
if subsentence_choices[4]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_src_lang'] \
and line_script == script_data['non_dial_dest_script']:
if subsentence_choices[6]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_dest_lang'] \
and line_script == script_data['dial_dest_script']:
if subsentence_choices[8]:
trans_text = all_translator(
sentence, source_lang, target_lang)
elif line_language == script_data['non_dial_dest_lang'] \
and line_script == script_data['non_dial_dest_script']:
if subsentence_choices[10]:
trans_text = all_translator(
sentence, source_lang, target_lang)
else:
if subsentence_choices[12]:
trans_text = all_translator(
sentence, source_lang, target_lang)
translated_text.append(trans_text)
else:
return " ".join(translated_text)
else:
return all_translator(text, source_lang, target_lang)