Conversion_Kitchen_Code/kitchen_counter/conversion/management/commands/custom_runserver.py

481 lines
21 KiB
Python
Raw Normal View History

2024-04-30 04:59:37 +00:00
import socket
import time
import boto3
import subprocess
import json
2024-05-11 10:04:00 +00:00
import os
import pandas as pd
2024-04-30 04:59:37 +00:00
import threading
from django.core.management.base import BaseCommand
from django.core.management import call_command
from mnfapp.models import ScriptTranslations
from MNF.settings import BasePath
from conversion.translation.external_conversion import Conversion
from scriptAudit.views import run_audit_in_counter
2024-04-30 07:33:51 +00:00
from django.core.files.base import ContentFile
from django.contrib.auth import get_user_model
2024-05-02 12:38:32 +00:00
from users.models import UserCredentialsForBlockchain
from django.core.files.base import File as DjangoFile
import tempfile
from MNF import settings
2024-05-02 12:38:32 +00:00
from io import BytesIO
2024-05-11 02:26:37 +00:00
from auto_email.views import sendmail
from lpp.certificate.createCertificate import certificateGenrate
2024-09-27 09:41:14 +00:00
from scriptAudit.utils import update_audit_status
2024-05-11 02:26:37 +00:00
2024-04-30 04:59:37 +00:00
basePath = BasePath()
2024-04-30 07:33:51 +00:00
User = get_user_model()
2024-04-30 04:59:37 +00:00
def background_execution(obj):
obj.convert()
2024-04-30 07:33:51 +00:00
def run_conversion(msg):
2024-11-12 10:22:32 +00:00
print("inside run_conversion")
2024-04-30 07:33:51 +00:00
body_dict = json.loads(msg.body)
2024-05-13 08:47:04 +00:00
msg.delete()
2024-04-30 07:33:51 +00:00
translated_script = ScriptTranslations.objects.get(translation_id=body_dict["translation_id"])
object_key = "INPUT/" + (translated_script.script_link_id.script.name.split("/"))[-1]
local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + translated_script.script_link_id.script.name
s3_client = boto3.client('s3',
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
aws_secret_access_key="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",
region_name="ap-south-1"
)
s3_client.download_file("conversion-kitchen", object_key, local_file_path)
2024-11-12 10:22:32 +00:00
print("i am here below s3_client")
audit_user = User.objects.get(id=1)
2024-04-30 07:33:51 +00:00
conversion_params = {
"audit_user": audit_user,
2024-04-30 07:33:51 +00:00
"user": translated_script.user_id,
"file_path": str(basePath) + "/media/" + translated_script.script_link_id.script.name,
"original_script_id": translated_script.script_link_id.script_id,
"translated_script_id": translated_script.translation_id,
"sample_id": body_dict.get('sample_id', None),
"existing_script": body_dict.get('existing_script', None),
2024-04-30 07:33:51 +00:00
"iteration_no": body_dict.get('iteration_no', None),
"juggernaut_pages_deduction": body_dict.get("juggernaut_pages_deduction", None),
"language_set": body_dict.get("language_set", None),
"amount_without_subscrption": body_dict.get("amount_without_subscrption", None)
}
print("reached here")
obj = Conversion(**conversion_params)
# obj.convert()
background_thread = threading.Thread(target=background_execution, args=(obj,))
background_thread.start()
background_thread.join()
# s3_client.delete_object(Bucket='conversion-kitchen', Key=object_key)
2024-05-02 07:48:38 +00:00
2024-04-30 07:33:51 +00:00
def run_audit(msg):
body_dict = json.loads(msg.body)
2024-05-13 08:47:04 +00:00
msg.delete()
2024-04-30 07:33:51 +00:00
user = body_dict.get("user")
s3_url = body_dict.get("s3-file-path")
screenplay_name = body_dict.get("screenplay_name")
author = body_dict.get("author")
language = body_dict.get("language")
script_ext = body_dict.get("script_ext")
script_file_name = body_dict.get("script_file_name")
2024-05-02 12:38:32 +00:00
language = "en"
print("112")
2024-04-30 07:33:51 +00:00
object_key = "INPUT/" + str(script_file_name)
2024-04-30 07:38:25 +00:00
local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/audit_counter_files/" + script_file_name
2024-04-30 07:33:51 +00:00
s3_client = boto3.client('s3',
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
aws_secret_access_key="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",
region_name="ap-south-1"
)
2024-05-02 12:38:32 +00:00
print(object_key, local_file_path)
2024-04-30 07:33:51 +00:00
s3_client.download_file("conversion-kitchen", object_key, local_file_path)
audit_parameters = {
"service_type" : "audit",
"user":user,
"s3-file-path": local_file_path,
"screenplay_name": screenplay_name,
"author": author,
"language": language,
"script_ext": script_ext,
"script_file_name": script_file_name
2024-05-02 12:38:32 +00:00
}
run_audit_in_counter(audit_parameters)
2024-04-30 07:33:51 +00:00
2024-09-27 09:41:14 +00:00
class Request:
def __init__(self, user):
self.user = user
def lpp_audit(msg):
import boto3
2024-11-12 10:22:32 +00:00
from lpp.models import LPPTASKDatabase,MNFLPPDDatabase
from decimal import Decimal
2024-09-27 09:41:14 +00:00
from utils import filesystem, utilities
from centralisedFileSystem.models import Script
from scriptAudit.models import ScriptAuditModel
from scriptAudit.models import States
from scriptAudit.mnf_script_audit import NeutralAudit
from django.core.files.base import ContentFile
from datetime import datetime
from Blockchain2.DataStorage import uploadDataToIPFSNode
from Blockchain2.decryption import decryptionOfPrivate, hash_decrypation
from Blockchain2.Conversion import UploadConversionData
from Blockchain2.blockchainsetting import OWNER_KEY
from Blockchain2.block_user_info import user_info
body_dict = json.loads(msg.body)
msg.delete()
2024-11-12 10:22:32 +00:00
with open(f"/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/MNF/json_keys/conversionRates.json") as c:
curr = json.load(c)
try:
usd_inr_rate = curr["rates"]["INR"]
# c = CurrencyRates()
# rate = c.get_rate("USD", str(currency.upper()))
except Exception as e:
print("checkout error", e)
usd_inr_rate = 80
2024-09-27 09:41:14 +00:00
print("ran till here 1")
task_id = body_dict.get("task_id", None)
outputfile = body_dict.get("lpp_file", None)
if outputfile is not None:
outputfile = "INPUT/" + str((outputfile.split("INPUT/"))[-1])
originalfile = body_dict.get("original_file", None)
if originalfile is not None:
originalfile = "INPUT/" + str((originalfile.split("INPUT/"))[-1])
userr = body_dict.get("userr")
print("ran till here 2")
if task_id is None:
pass
lpp_user = LPPTASKDatabase.objects.get(task_id=task_id)
2024-11-12 10:22:32 +00:00
# x = LPPTASKDatabase.objects.get(task_id=pk)
# lpp_amount = MNFLPPDDatabase.objects.get(lpp_id=lpp_user.assignedlpp_dialogue.lpp_id)
2024-09-27 09:41:14 +00:00
if outputfile is None:
pass
file_user = User.objects.get(id=userr)
print("ran till here 3")
# object_key = "INPUT/" + (translated_script.script_link_id.script.name.split("/"))[-1]
original_file = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + lpp_user.translated_script.script_link_id.script.name
local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + lpp_user.outputfile.name
s3_client = boto3.client('s3',
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
aws_secret_access_key="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",
region_name="ap-south-1")
print("outputs1", outputfile, local_file_path)
print("outputs2", originalfile, original_file)
s3_client.download_file("conversion-kitchen", outputfile, local_file_path)
s3_client.download_file("conversion-kitchen", originalfile, original_file)
print("ran till here 4")
try:
language_code = "en"
name_script = (str(local_file_path).split("."))[0] + "_vetted_a" + "." + (str(local_file_path).split("."))[1]
doc_file = f"{local_file_path}"
file1 = ContentFile(
open(doc_file, 'rb').read(),
(doc_file.split("/"))[-1],
)
result = filesystem.new_screenplay_without_audit_in_background(
file_user,
file_user.username,
str(name_script),
file1,
"script-original",
language_code,
)
print("already called", result)
audit_id = result.get("script", {}).get("id")
print("ran till here 10")
if audit_id is not None:
objectt = ScriptTranslations.objects.get(translation_id=lpp_user.translated_script.translation_id)
objectt.converted_audit_id = audit_id
objectt.save()
try:
update_audit_status(audit_id, States.STARTED)
except:
update_audit_status(audit_id, States.FAILURE)
# audit = NeutralAudit(audit_id)
# status = ScriptAuditModel.objects.get(
# script=Script.objects.get(
# id=audit_id
# )
# )
try:
audit = NeutralAudit(audit_id)
audit.audit_in_background()
ScriptAuditModel.objects.update_or_create(
script = Script.objects.get(
id = audit_id
),
defaults={"status" : "SUCCESS"}
)
2024-11-12 10:22:32 +00:00
lpp_user.status = "completed"
lpp_user.save()
2024-09-27 09:41:14 +00:00
except Exception as exp:
ScriptAuditModel.objects.update_or_create(script = Script.objects.get(id = audit_id),
defaults={"status" : "FAILURE",
"results" : exp})
print("audit will start")
2024-11-12 10:22:32 +00:00
2024-09-27 09:41:14 +00:00
except Exception as e:
print("Error in auditing vetted output is", e)
print("ran till here 11")
"""Blockchain Upload Starts here"""
# sys.stdout = original_stdout
status = ScriptAuditModel.objects.get(
script = Script.objects.get(
id = audit_id
))
scriptconversion = {}
2024-11-12 10:22:32 +00:00
print("ADDING LPP BALANCE HERE \n\n\n")
2024-09-27 09:41:14 +00:00
try:
if status.status == "SUCCESS":
2024-11-12 10:22:32 +00:00
print("#### STATUS iS SUCESSS @############\n\n")
# adding amount to the wallet here
lpp_amount = MNFLPPDDatabase.objects.get(lpp_id=lpp_user.assignedlpp_action.lpp_id)
print("##### try to add amount here#####\n\n\n")
try:
lpp_amount.lpp_task_balance = float(Decimal(
str(lpp_amount.lpp_task_balance))) + float(float(Decimal(str(lpp_user.amoutgiventolpp_dialogue))) * usd_inr_rate)
print("TRY BLOCK COMPLETE")
print(str(lpp_amount.lpp_task_balance))
except Exception as e:
print("\n\n#####LPP Balance Error", e)
lpp_amount.lpp_task_balance = 0.00 + \
float(
float(Decimal(lpp_user.amoutgiventolpp_dialogue)) * usd_inr_rate)
lpp_amount.save()
print("#######the AMOUNT iS ADDED ##########\n\n\n")
2024-09-27 09:41:14 +00:00
to = lpp_user.assignedlpp_action.lpp_email
key_value = {
"User": lpp_user.assignedlpp_action.user_id.username,
"Amount": lpp_user.amoutgiventolpp_action
}
sendmail(to_email=[to], email_code="PP14", key_value=key_value)
except Exception as exp:
print("error in mail sending PP14")
print(exp)
try:
print("trying blockchain 1")
current_datetime = datetime.now()
if UserCredentialsForBlockchain.objects.filter(user=lpp_user.user_id).exists():
obj = lpp_user.translated_script
obj2 = lpp_user.translated_script.script_link_id
# temp1 = str(obj2.script.name)
# temp2 = str(lpp_user.outputfile)
uploaded_script = f"{original_file}"
translated_scripttt = f"{local_file_path}"
print("blockchain script conversion 3", uploaded_script, translated_scripttt)
with open(uploaded_script, 'rb') as f:
hash = uploadDataToIPFSNode(f)
scriptconversion["original_scriptFile_hash"] = hash
scriptconversion["original_scriptFile_path"] = uploaded_script
scriptconversion["date_at"] = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
print("blockchain script conversion 4")
with open(translated_scripttt, 'rb') as f1:
hash = uploadDataToIPFSNode(f1)
scriptconversion["translated_scriptFile_hash"] = hash
scriptconversion["translated_scriptFile_path"] = translated_scripttt
print("blockchain script conversion 5")
from centralisedFileSystem.models import File
if obj.converted_audit_id:
print("audit blockchain")
print("blockchain script conversion 5.1")
# blockchain upload of csv-json data
adit_id = obj.converted_audit_id
2024-11-12 10:22:32 +00:00
print("######ADIT ID",adit_id)
print("###########@@@@@@@@@ AUDIT ID",audit_id)
2024-09-27 09:41:14 +00:00
file_to_audit = File.objects.get(
2024-11-12 10:22:32 +00:00
script=audit_id,
2024-09-27 09:41:14 +00:00
type="script-csv"
)
csv_script_path = file_to_audit.file.path
df = pd.read_csv(csv_script_path)
df = df.loc[:, ["content", "script_element"]]
script_json: dict = json.loads(utilities.csv_to_json(df))
with tempfile.TemporaryDirectory() as temp_dir:
print("Temporary directory created:", temp_dir)
temp_filename = os.path.join(temp_dir, 'script_json_file.json')
print(temp_filename)
with open(temp_filename, 'w') as json_file:
json.dump(script_json, json_file, indent=4)
script_json = {}
script_path1 = temp_filename
# script_size = file_to_audit_docx.file.size
with open(script_path1, 'rb') as _file:
hash2 = uploadDataToIPFSNode(_file)
script_json["script_file_path"] = script_path1
script_json["script_file"] = hash2
script_json["type"] = "script-json"
script_json["screenplay_name"] = str(name_script)
2024-09-27 09:41:14 +00:00
scriptconversion["script-json"] = script_json
print("blockchain script conversion 5.2")
blockchain_obj = UserCredentialsForBlockchain.objects.get(user=lpp_user.user_id)
UserId = blockchain_obj.user_id
Project = obj.translation_id
Data = str(scriptconversion)
userPrivateKey = blockchain_obj.privateKey
userkey = decryptionOfPrivate(userPrivateKey)
tx_id, tx_gas_fee = UploadConversionData(OWNER_KEY, blockchain_obj.publicKey, UserId,
str(Project),
Data)
# lpp_user.translated_script.blockchain_txhash = tx_id
print("Tx id -> ", tx_id, tx_gas_fee)
# user_infos = user_info(tx_hash=tx_id, service="Conversion", gas_fee=2345432345123456)
# addition_result = user_infos.update_info(request)
2024-11-12 10:22:32 +00:00
######## UPLOADING THE FILE FOR LPP USER (LLP SIDE VIEW BUTTON)
blockchain_obj_for_lpp = UserCredentialsForBlockchain.objects.get(user=lpp_user.assignedlpp_action.user_id)
UserId_for_lpp = blockchain_obj_for_lpp.user_id
Data_for_lpp = str(scriptconversion)
userPrivateKey_for_lpp = blockchain_obj_for_lpp.privateKey
userkey_for_lpp = decryptionOfPrivate(userPrivateKey_for_lpp)
tx_id_for_lpp, tx_gas_fee_for_lpp = UploadConversionData(OWNER_KEY, blockchain_obj_for_lpp.publicKey, UserId_for_lpp,
str(Project),
Data_for_lpp)
print("LPP USER BLOCKCHAIN and REAL USER BLOCKCHAIN UPLOADED")
print("Project-------->",Project)
2024-09-27 09:41:14 +00:00
name_script = (((uploaded_script.split("/"))[-1]).split("."))[0]
try:
user_infos = user_info(tx_hash=tx_id, service="Conversion", gas_fee=tx_gas_fee,
script_name=name_script)
request = Request(lpp_user.user_id)
addition_result = user_infos.update_info(request)
print("Blockchain Result -> ", addition_result)
except Exception as e:
print("Error in blockchain is:", e)
print("blockchain script conversion 6")
certificatepath = certificateGenrate(lpp_user.user_id.username, "script conversion", hash)
hash = hash_decrypation(hash)
to_email = [lpp_user.user_id.email]
email_code = 'BL1'
key_value = {
"service": "script conversion",
"hash": hash,
"public key": blockchain_obj.publicKey,
"private key": userkey.decode('utf-8'),
"Transaction Hash": tx_id
}
sendmail(to_email=to_email, email_code=email_code, key_value=key_value,
filePath=certificatepath)
import boto3
s3_client = boto3.client('s3',
aws_access_key_id="AKIAQVLBBGCB45RMLKVW",
aws_secret_access_key="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",
region_name="ap-south-1"
)
bucket = "conversion-kitchen"
# file_name = str(((lpp_user.translated_script.translated_script_pdf).split("OUTPUT/"))[1])
# object_key = "OUTPUT/" + file_name
s3_client.delete_object(Bucket=bucket, Key=outputfile)
s3_client.delete_object(Bucket=bucket, Key=originalfile)
to = lpp_user.translated_script.user_id.email
key_value = {
"User": lpp_user.translated_script.user_id.username,
"title": lpp_user.translated_script.script_link_id.script_title
}
sendmail(to_email=[to], email_code="PP18", key_value=key_value)
try:
objectt = ScriptTranslations.objects.get(translation_id=lpp_user.translated_script.translation_id)
objectt.blockchain_txhash = tx_id
objectt.save()
except Exception as e:
print("error while saving blockchain txhash is ->",str(e))
except Exception as e:
print("Error in blockchain is", e)
2024-04-30 07:33:51 +00:00
2024-04-30 04:59:37 +00:00
class Command(BaseCommand):
help = 'Custom Command to start django server and then start dequeuing from SQS'
def handle(self, *args, **options):
# Call the original runserver command"
# call_command('runserver', '0.0.0.0:4549')
command = 'python manage.py runserver 0.0.0.0:4549 > /home/ubuntu/Conversion_Kitchen_Code/logfile.log 2>&1'
# # Execute the command using subprocess.Popen
subprocess.Popen(command, shell=True)
# Wait for the server to start
while True:
try:
# Try to connect to the server
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect(('0.0.0.0', 4549))
break
except ConnectionRefusedError:
# If connection is refused, wait and try again
time.sleep(1)
# Once the server is up, run your function
self.run_after_server_startup()
def run_after_server_startup(self):
# Your function to run after the server starts
print("Development server is fully up and running! Run your function here.")
session = boto3.Session(
aws_access_key_id='AKIAQVLBBGCB45RMLKVW', # replace with your key
aws_secret_access_key='ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA', # replace with your key
)
sqs = session.resource('sqs', region_name='ap-south-1')
print("Started Executin this from conversion")
2024-11-12 12:36:00 +00:00
queue = sqs.get_queue_by_name(QueueName="mnfqueue")
2024-04-30 04:59:37 +00:00
# try:
2024-05-01 04:58:44 +00:00
while True:
2024-04-30 04:59:37 +00:00
2024-05-01 04:58:44 +00:00
messages = queue.receive_messages(
MessageAttributeNames=["All"],
2024-05-01 05:22:33 +00:00
MaxNumberOfMessages=1,
WaitTimeSeconds=5,
2024-05-01 04:58:44 +00:00
)
if len(messages) > 0:
for msg in messages:
try:
print("Received message: %s: %s", msg.message_id, msg.body)
print(type(msg.body))
body_dict = json.loads(msg.body)
if body_dict['service_type'] == "conversion":
2024-11-12 10:22:32 +00:00
print("Entering Conversion")
print(str(body_dict))
2024-05-01 04:58:44 +00:00
run_conversion(msg)
elif body_dict['service_type'] == "audit":
run_audit(msg)
2024-09-27 09:41:14 +00:00
elif body_dict['service_type'] == "lpp audit":
lpp_audit(msg)
2024-05-01 04:58:44 +00:00
except Exception as error:
print("error execution from queue: %s", error)
2024-11-12 10:22:32 +00:00
print(repr(error))
2024-05-01 04:58:44 +00:00
else:
break
2024-04-30 04:59:37 +00:00
print("Completed All Execution")