481 lines
21 KiB
Python
Executable File
481 lines
21 KiB
Python
Executable File
import socket
|
|
import time
|
|
import boto3
|
|
import subprocess
|
|
import json
|
|
import os
|
|
import pandas as pd
|
|
import threading
|
|
from django.core.management.base import BaseCommand
|
|
from django.core.management import call_command
|
|
from mnfapp.models import ScriptTranslations
|
|
from MNF.settings import BasePath
|
|
from conversion.translation.external_conversion import Conversion
|
|
from scriptAudit.views import run_audit_in_counter
|
|
from django.core.files.base import ContentFile
|
|
from django.contrib.auth import get_user_model
|
|
from users.models import UserCredentialsForBlockchain
|
|
from django.core.files.base import File as DjangoFile
|
|
import tempfile
|
|
from MNF import settings
|
|
from io import BytesIO
|
|
from auto_email.views import sendmail
|
|
from lpp.certificate.createCertificate import certificateGenrate
|
|
from scriptAudit.utils import update_audit_status
|
|
|
|
basePath = BasePath()
|
|
|
|
User = get_user_model()
|
|
|
|
def background_execution(obj):
|
|
obj.convert()
|
|
|
|
|
|
|
|
def run_conversion(msg):
|
|
print("inside run_conversion")
|
|
body_dict = json.loads(msg.body)
|
|
msg.delete()
|
|
translated_script = ScriptTranslations.objects.get(translation_id=body_dict["translation_id"])
|
|
object_key = "INPUT/" + (translated_script.script_link_id.script.name.split("/"))[-1]
|
|
local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + translated_script.script_link_id.script.name
|
|
s3_client = boto3.client('s3',
|
|
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
|
|
aws_secret_access_key="ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA",
|
|
region_name="ap-south-1"
|
|
)
|
|
s3_client.download_file("conversion-kitchen", object_key, local_file_path)
|
|
print("i am here below s3_client")
|
|
audit_user = User.objects.get(id=1)
|
|
conversion_params = {
|
|
"audit_user": audit_user,
|
|
"user": translated_script.user_id,
|
|
"file_path": str(basePath) + "/media/" + translated_script.script_link_id.script.name,
|
|
"original_script_id": translated_script.script_link_id.script_id,
|
|
"translated_script_id": translated_script.translation_id,
|
|
"sample_id": body_dict.get('sample_id', None),
|
|
"existing_script": body_dict.get('existing_script', None),
|
|
"iteration_no": body_dict.get('iteration_no', None),
|
|
"juggernaut_pages_deduction": body_dict.get("juggernaut_pages_deduction", None),
|
|
"language_set": body_dict.get("language_set", None),
|
|
"amount_without_subscrption": body_dict.get("amount_without_subscrption", None)
|
|
}
|
|
print("reached here")
|
|
obj = Conversion(**conversion_params)
|
|
# obj.convert()
|
|
background_thread = threading.Thread(target=background_execution, args=(obj,))
|
|
background_thread.start()
|
|
background_thread.join()
|
|
# s3_client.delete_object(Bucket='conversion-kitchen', Key=object_key)
|
|
|
|
|
|
def run_audit(msg):
|
|
body_dict = json.loads(msg.body)
|
|
msg.delete()
|
|
user = body_dict.get("user")
|
|
s3_url = body_dict.get("s3-file-path")
|
|
screenplay_name = body_dict.get("screenplay_name")
|
|
author = body_dict.get("author")
|
|
language = body_dict.get("language")
|
|
script_ext = body_dict.get("script_ext")
|
|
script_file_name = body_dict.get("script_file_name")
|
|
language = "en"
|
|
print("112")
|
|
object_key = "INPUT/" + str(script_file_name)
|
|
local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/audit_counter_files/" + script_file_name
|
|
s3_client = boto3.client('s3',
|
|
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
|
|
aws_secret_access_key="ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA",
|
|
region_name="ap-south-1"
|
|
)
|
|
print(object_key, local_file_path)
|
|
s3_client.download_file("conversion-kitchen", object_key, local_file_path)
|
|
audit_parameters = {
|
|
"service_type" : "audit",
|
|
"user":user,
|
|
"s3-file-path": local_file_path,
|
|
"screenplay_name": screenplay_name,
|
|
"author": author,
|
|
"language": language,
|
|
"script_ext": script_ext,
|
|
"script_file_name": script_file_name
|
|
}
|
|
run_audit_in_counter(audit_parameters)
|
|
|
|
|
|
class Request:
|
|
def __init__(self, user):
|
|
self.user = user
|
|
|
|
def lpp_audit(msg):
|
|
import boto3
|
|
from lpp.models import LPPTASKDatabase,MNFLPPDDatabase
|
|
from decimal import Decimal
|
|
from utils import filesystem, utilities
|
|
from centralisedFileSystem.models import Script
|
|
from scriptAudit.models import ScriptAuditModel
|
|
from scriptAudit.models import States
|
|
from scriptAudit.mnf_script_audit import NeutralAudit
|
|
from django.core.files.base import ContentFile
|
|
from datetime import datetime
|
|
from Blockchain2.DataStorage import uploadDataToIPFSNode
|
|
from Blockchain2.decryption import decryptionOfPrivate, hash_decrypation
|
|
from Blockchain2.Conversion import UploadConversionData
|
|
from Blockchain2.blockchainsetting import OWNER_KEY
|
|
from Blockchain2.block_user_info import user_info
|
|
body_dict = json.loads(msg.body)
|
|
msg.delete()
|
|
|
|
with open(f"/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/MNF/json_keys/conversionRates.json") as c:
|
|
curr = json.load(c)
|
|
try:
|
|
usd_inr_rate = curr["rates"]["INR"]
|
|
# c = CurrencyRates()
|
|
# rate = c.get_rate("USD", str(currency.upper()))
|
|
except Exception as e:
|
|
print("checkout error", e)
|
|
usd_inr_rate = 80
|
|
print("ran till here 1")
|
|
|
|
task_id = body_dict.get("task_id", None)
|
|
outputfile = body_dict.get("lpp_file", None)
|
|
if outputfile is not None:
|
|
outputfile = "INPUT/" + str((outputfile.split("INPUT/"))[-1])
|
|
originalfile = body_dict.get("original_file", None)
|
|
if originalfile is not None:
|
|
originalfile = "INPUT/" + str((originalfile.split("INPUT/"))[-1])
|
|
userr = body_dict.get("userr")
|
|
print("ran till here 2")
|
|
if task_id is None:
|
|
pass
|
|
|
|
lpp_user = LPPTASKDatabase.objects.get(task_id=task_id)
|
|
# x = LPPTASKDatabase.objects.get(task_id=pk)
|
|
# lpp_amount = MNFLPPDDatabase.objects.get(lpp_id=lpp_user.assignedlpp_dialogue.lpp_id)
|
|
|
|
if outputfile is None:
|
|
pass
|
|
|
|
file_user = User.objects.get(id=userr)
|
|
print("ran till here 3")
|
|
# object_key = "INPUT/" + (translated_script.script_link_id.script.name.split("/"))[-1]
|
|
original_file = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + lpp_user.translated_script.script_link_id.script.name
|
|
local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + lpp_user.outputfile.name
|
|
s3_client = boto3.client('s3',
|
|
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
|
|
aws_secret_access_key="ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA",
|
|
region_name="ap-south-1")
|
|
print("outputs1", outputfile, local_file_path)
|
|
print("outputs2", originalfile, original_file)
|
|
s3_client.download_file("conversion-kitchen", outputfile, local_file_path)
|
|
s3_client.download_file("conversion-kitchen", originalfile, original_file)
|
|
print("ran till here 4")
|
|
try:
|
|
language_code = "en"
|
|
name_script = (str(local_file_path).split("."))[0] + "_vetted_a" + "." + (str(local_file_path).split("."))[1]
|
|
doc_file = f"{local_file_path}"
|
|
file1 = ContentFile(
|
|
open(doc_file, 'rb').read(),
|
|
(doc_file.split("/"))[-1],
|
|
)
|
|
result = filesystem.new_screenplay_without_audit_in_background(
|
|
file_user,
|
|
file_user.username,
|
|
str(name_script),
|
|
file1,
|
|
"script-original",
|
|
language_code,
|
|
)
|
|
print("already called", result)
|
|
audit_id = result.get("script", {}).get("id")
|
|
print("ran till here 10")
|
|
if audit_id is not None:
|
|
|
|
objectt = ScriptTranslations.objects.get(translation_id=lpp_user.translated_script.translation_id)
|
|
objectt.converted_audit_id = audit_id
|
|
objectt.save()
|
|
|
|
try:
|
|
update_audit_status(audit_id, States.STARTED)
|
|
except:
|
|
update_audit_status(audit_id, States.FAILURE)
|
|
|
|
# audit = NeutralAudit(audit_id)
|
|
# status = ScriptAuditModel.objects.get(
|
|
# script=Script.objects.get(
|
|
# id=audit_id
|
|
# )
|
|
# )
|
|
try:
|
|
audit = NeutralAudit(audit_id)
|
|
audit.audit_in_background()
|
|
ScriptAuditModel.objects.update_or_create(
|
|
script = Script.objects.get(
|
|
id = audit_id
|
|
),
|
|
defaults={"status" : "SUCCESS"}
|
|
)
|
|
lpp_user.status = "completed"
|
|
lpp_user.save()
|
|
except Exception as exp:
|
|
ScriptAuditModel.objects.update_or_create(script = Script.objects.get(id = audit_id),
|
|
defaults={"status" : "FAILURE",
|
|
"results" : exp})
|
|
print("audit will start")
|
|
|
|
except Exception as e:
|
|
print("Error in auditing vetted output is", e)
|
|
print("ran till here 11")
|
|
"""Blockchain Upload Starts here"""
|
|
# sys.stdout = original_stdout
|
|
status = ScriptAuditModel.objects.get(
|
|
script = Script.objects.get(
|
|
id = audit_id
|
|
))
|
|
scriptconversion = {}
|
|
print("ADDING LPP BALANCE HERE \n\n\n")
|
|
try:
|
|
if status.status == "SUCCESS":
|
|
print("#### STATUS iS SUCESSS @############\n\n")
|
|
# adding amount to the wallet here
|
|
lpp_amount = MNFLPPDDatabase.objects.get(lpp_id=lpp_user.assignedlpp_action.lpp_id)
|
|
print("##### try to add amount here#####\n\n\n")
|
|
try:
|
|
lpp_amount.lpp_task_balance = float(Decimal(
|
|
str(lpp_amount.lpp_task_balance))) + float(float(Decimal(str(lpp_user.amoutgiventolpp_dialogue))) * usd_inr_rate)
|
|
print("TRY BLOCK COMPLETE")
|
|
print(str(lpp_amount.lpp_task_balance))
|
|
except Exception as e:
|
|
print("\n\n#####LPP Balance Error", e)
|
|
lpp_amount.lpp_task_balance = 0.00 + \
|
|
float(
|
|
float(Decimal(lpp_user.amoutgiventolpp_dialogue)) * usd_inr_rate)
|
|
lpp_amount.save()
|
|
print("#######the AMOUNT iS ADDED ##########\n\n\n")
|
|
|
|
to = lpp_user.assignedlpp_action.lpp_email
|
|
key_value = {
|
|
"User": lpp_user.assignedlpp_action.user_id.username,
|
|
"Amount": lpp_user.amoutgiventolpp_action
|
|
}
|
|
sendmail(to_email=[to], email_code="PP14", key_value=key_value)
|
|
except Exception as exp:
|
|
print("error in mail sending PP14")
|
|
print(exp)
|
|
try:
|
|
print("trying blockchain 1")
|
|
current_datetime = datetime.now()
|
|
if UserCredentialsForBlockchain.objects.filter(user=lpp_user.user_id).exists():
|
|
obj = lpp_user.translated_script
|
|
obj2 = lpp_user.translated_script.script_link_id
|
|
# temp1 = str(obj2.script.name)
|
|
# temp2 = str(lpp_user.outputfile)
|
|
uploaded_script = f"{original_file}"
|
|
translated_scripttt = f"{local_file_path}"
|
|
print("blockchain script conversion 3", uploaded_script, translated_scripttt)
|
|
with open(uploaded_script, 'rb') as f:
|
|
hash = uploadDataToIPFSNode(f)
|
|
scriptconversion["original_scriptFile_hash"] = hash
|
|
scriptconversion["original_scriptFile_path"] = uploaded_script
|
|
scriptconversion["date_at"] = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
|
|
print("blockchain script conversion 4")
|
|
with open(translated_scripttt, 'rb') as f1:
|
|
hash = uploadDataToIPFSNode(f1)
|
|
scriptconversion["translated_scriptFile_hash"] = hash
|
|
scriptconversion["translated_scriptFile_path"] = translated_scripttt
|
|
print("blockchain script conversion 5")
|
|
from centralisedFileSystem.models import File
|
|
if obj.converted_audit_id:
|
|
print("audit blockchain")
|
|
print("blockchain script conversion 5.1")
|
|
# blockchain upload of csv-json data
|
|
adit_id = obj.converted_audit_id
|
|
print("######ADIT ID",adit_id)
|
|
print("###########@@@@@@@@@ AUDIT ID",audit_id)
|
|
file_to_audit = File.objects.get(
|
|
script=audit_id,
|
|
type="script-csv"
|
|
)
|
|
csv_script_path = file_to_audit.file.path
|
|
df = pd.read_csv(csv_script_path)
|
|
df = df.loc[:, ["content", "script_element"]]
|
|
script_json: dict = json.loads(utilities.csv_to_json(df))
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
print("Temporary directory created:", temp_dir)
|
|
temp_filename = os.path.join(temp_dir, 'script_json_file.json')
|
|
print(temp_filename)
|
|
with open(temp_filename, 'w') as json_file:
|
|
json.dump(script_json, json_file, indent=4)
|
|
script_json = {}
|
|
script_path1 = temp_filename
|
|
# script_size = file_to_audit_docx.file.size
|
|
with open(script_path1, 'rb') as _file:
|
|
hash2 = uploadDataToIPFSNode(_file)
|
|
script_json["script_file_path"] = script_path1
|
|
script_json["script_file"] = hash2
|
|
script_json["type"] = "script-json"
|
|
script_json["screenplay_name"] = str(name_script)
|
|
scriptconversion["script-json"] = script_json
|
|
print("blockchain script conversion 5.2")
|
|
|
|
blockchain_obj = UserCredentialsForBlockchain.objects.get(user=lpp_user.user_id)
|
|
UserId = blockchain_obj.user_id
|
|
Project = obj.translation_id
|
|
Data = str(scriptconversion)
|
|
userPrivateKey = blockchain_obj.privateKey
|
|
userkey = decryptionOfPrivate(userPrivateKey)
|
|
tx_id, tx_gas_fee = UploadConversionData(OWNER_KEY, blockchain_obj.publicKey, UserId,
|
|
str(Project),
|
|
Data)
|
|
# lpp_user.translated_script.blockchain_txhash = tx_id
|
|
print("Tx id -> ", tx_id, tx_gas_fee)
|
|
# user_infos = user_info(tx_hash=tx_id, service="Conversion", gas_fee=2345432345123456)
|
|
# addition_result = user_infos.update_info(request)
|
|
|
|
|
|
|
|
######## UPLOADING THE FILE FOR LPP USER (LLP SIDE VIEW BUTTON)
|
|
blockchain_obj_for_lpp = UserCredentialsForBlockchain.objects.get(user=lpp_user.assignedlpp_action.user_id)
|
|
UserId_for_lpp = blockchain_obj_for_lpp.user_id
|
|
Data_for_lpp = str(scriptconversion)
|
|
userPrivateKey_for_lpp = blockchain_obj_for_lpp.privateKey
|
|
userkey_for_lpp = decryptionOfPrivate(userPrivateKey_for_lpp)
|
|
tx_id_for_lpp, tx_gas_fee_for_lpp = UploadConversionData(OWNER_KEY, blockchain_obj_for_lpp.publicKey, UserId_for_lpp,
|
|
str(Project),
|
|
Data_for_lpp)
|
|
print("LPP USER BLOCKCHAIN and REAL USER BLOCKCHAIN UPLOADED")
|
|
print("Project-------->",Project)
|
|
|
|
|
|
|
|
name_script = (((uploaded_script.split("/"))[-1]).split("."))[0]
|
|
try:
|
|
user_infos = user_info(tx_hash=tx_id, service="Conversion", gas_fee=tx_gas_fee,
|
|
script_name=name_script)
|
|
request = Request(lpp_user.user_id)
|
|
addition_result = user_infos.update_info(request)
|
|
print("Blockchain Result -> ", addition_result)
|
|
except Exception as e:
|
|
print("Error in blockchain is:", e)
|
|
|
|
print("blockchain script conversion 6")
|
|
certificatepath = certificateGenrate(lpp_user.user_id.username, "script conversion", hash)
|
|
hash = hash_decrypation(hash)
|
|
to_email = [lpp_user.user_id.email]
|
|
email_code = 'BL1'
|
|
key_value = {
|
|
"service": "script conversion",
|
|
"hash": hash,
|
|
"public key": blockchain_obj.publicKey,
|
|
"private key": userkey.decode('utf-8'),
|
|
"Transaction Hash": tx_id
|
|
}
|
|
sendmail(to_email=to_email, email_code=email_code, key_value=key_value,
|
|
filePath=certificatepath)
|
|
import boto3
|
|
s3_client = boto3.client('s3',
|
|
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
|
|
aws_secret_access_key="ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA",
|
|
region_name="ap-south-1"
|
|
)
|
|
bucket = "conversion-kitchen"
|
|
# file_name = str(((lpp_user.translated_script.translated_script_pdf).split("OUTPUT/"))[1])
|
|
# object_key = "OUTPUT/" + file_name
|
|
s3_client.delete_object(Bucket=bucket, Key=outputfile)
|
|
s3_client.delete_object(Bucket=bucket, Key=originalfile)
|
|
|
|
to = lpp_user.translated_script.user_id.email
|
|
key_value = {
|
|
"User": lpp_user.translated_script.user_id.username,
|
|
"title": lpp_user.translated_script.script_link_id.script_title
|
|
}
|
|
sendmail(to_email=[to], email_code="PP18", key_value=key_value)
|
|
|
|
try:
|
|
objectt = ScriptTranslations.objects.get(translation_id=lpp_user.translated_script.translation_id)
|
|
objectt.blockchain_txhash = tx_id
|
|
objectt.save()
|
|
except Exception as e:
|
|
print("error while saving blockchain txhash is ->",str(e))
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
print("Error in blockchain is", e)
|
|
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = 'Custom Command to start django server and then start dequeuing from SQS'
|
|
|
|
def handle(self, *args, **options):
|
|
# Call the original runserver command"
|
|
# call_command('runserver', '0.0.0.0:4549')
|
|
command = 'python manage.py runserver 0.0.0.0:4549 > /home/ubuntu/Conversion_Kitchen_Code/logfile.log 2>&1'
|
|
|
|
# # Execute the command using subprocess.Popen
|
|
subprocess.Popen(command, shell=True)
|
|
|
|
# Wait for the server to start
|
|
while True:
|
|
try:
|
|
# Try to connect to the server
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
s.connect(('0.0.0.0', 4549))
|
|
break
|
|
except ConnectionRefusedError:
|
|
# If connection is refused, wait and try again
|
|
time.sleep(1)
|
|
|
|
# Once the server is up, run your function
|
|
self.run_after_server_startup()
|
|
|
|
def run_after_server_startup(self):
|
|
# Your function to run after the server starts
|
|
print("Development server is fully up and running! Run your function here.")
|
|
|
|
session = boto3.Session(
|
|
aws_access_key_id='AKIAQVLBBGCB45RMLKVW', # replace with your key
|
|
aws_secret_access_key='ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA', # replace with your key
|
|
)
|
|
sqs = session.resource('sqs', region_name='ap-south-1')
|
|
|
|
print("Started Executin this from conversion")
|
|
queue = sqs.get_queue_by_name(QueueName="mnfqueue")
|
|
# try:
|
|
while True:
|
|
|
|
messages = queue.receive_messages(
|
|
MessageAttributeNames=["All"],
|
|
MaxNumberOfMessages=1,
|
|
WaitTimeSeconds=5,
|
|
)
|
|
if len(messages) > 0:
|
|
|
|
for msg in messages:
|
|
try:
|
|
print("Received message: %s: %s", msg.message_id, msg.body)
|
|
print(type(msg.body))
|
|
body_dict = json.loads(msg.body)
|
|
if body_dict['service_type'] == "conversion":
|
|
print("Entering Conversion")
|
|
print(str(body_dict))
|
|
run_conversion(msg)
|
|
|
|
elif body_dict['service_type'] == "audit":
|
|
|
|
run_audit(msg)
|
|
|
|
elif body_dict['service_type'] == "lpp audit":
|
|
|
|
lpp_audit(msg)
|
|
|
|
except Exception as error:
|
|
print("error execution from queue: %s", error)
|
|
print(repr(error))
|
|
else:
|
|
break
|
|
|
|
print("Completed All Execution")
|