import socket import time import boto3 import subprocess import json import os import pandas as pd import threading from django.core.management.base import BaseCommand from django.core.management import call_command from mnfapp.models import ScriptTranslations from MNF.settings import BasePath from conversion.translation.external_conversion import Conversion from scriptAudit.views import run_audit_in_counter from django.core.files.base import ContentFile from django.contrib.auth import get_user_model from users.models import UserCredentialsForBlockchain from django.core.files.base import File as DjangoFile import tempfile from MNF import settings from io import BytesIO from auto_email.views import sendmail from lpp.certificate.createCertificate import certificateGenrate from scriptAudit.utils import update_audit_status basePath = BasePath() User = get_user_model() def background_execution(obj): obj.convert() def run_conversion(msg): print("inside run_conversion") body_dict = json.loads(msg.body) msg.delete() translated_script = ScriptTranslations.objects.get(translation_id=body_dict["translation_id"]) object_key = "INPUT/" + (translated_script.script_link_id.script.name.split("/"))[-1] local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + translated_script.script_link_id.script.name s3_client = boto3.client('s3', aws_access_key_id="AKIAQVLBBGCB45RMLKVW", aws_secret_access_key="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", region_name="ap-south-1" ) s3_client.download_file("conversion-kitchen", object_key, local_file_path) print("i am here below s3_client") audit_user = User.objects.get(id=1) conversion_params = { "audit_user": audit_user, "user": translated_script.user_id, "file_path": str(basePath) + "/media/" + translated_script.script_link_id.script.name, "original_script_id": translated_script.script_link_id.script_id, "translated_script_id": translated_script.translation_id, "sample_id": body_dict.get('sample_id', None), "existing_script": body_dict.get('existing_script', None), "iteration_no": body_dict.get('iteration_no', None), "juggernaut_pages_deduction": body_dict.get("juggernaut_pages_deduction", None), "language_set": body_dict.get("language_set", None), "amount_without_subscrption": body_dict.get("amount_without_subscrption", None) } print("reached here") obj = Conversion(**conversion_params) # obj.convert() background_thread = threading.Thread(target=background_execution, args=(obj,)) background_thread.start() background_thread.join() # s3_client.delete_object(Bucket='conversion-kitchen', Key=object_key) def run_audit(msg): body_dict = json.loads(msg.body) msg.delete() user = body_dict.get("user") s3_url = body_dict.get("s3-file-path") screenplay_name = body_dict.get("screenplay_name") author = body_dict.get("author") language = body_dict.get("language") script_ext = body_dict.get("script_ext") script_file_name = body_dict.get("script_file_name") language = "en" print("112") object_key = "INPUT/" + str(script_file_name) local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/audit_counter_files/" + script_file_name s3_client = boto3.client('s3', aws_access_key_id="AKIAQVLBBGCB45RMLKVW", aws_secret_access_key="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", region_name="ap-south-1" ) print(object_key, local_file_path) s3_client.download_file("conversion-kitchen", object_key, local_file_path) audit_parameters = { "service_type" : "audit", "user":user, "s3-file-path": local_file_path, "screenplay_name": screenplay_name, "author": author, "language": language, "script_ext": script_ext, "script_file_name": script_file_name } run_audit_in_counter(audit_parameters) class Request: def __init__(self, user): self.user = user def lpp_audit(msg): import boto3 from lpp.models import LPPTASKDatabase,MNFLPPDDatabase from decimal import Decimal from utils import filesystem, utilities from centralisedFileSystem.models import Script from scriptAudit.models import ScriptAuditModel from scriptAudit.models import States from scriptAudit.mnf_script_audit import NeutralAudit from django.core.files.base import ContentFile from datetime import datetime from Blockchain2.DataStorage import uploadDataToIPFSNode from Blockchain2.decryption import decryptionOfPrivate, hash_decrypation from Blockchain2.Conversion import UploadConversionData from Blockchain2.blockchainsetting import OWNER_KEY from Blockchain2.block_user_info import user_info body_dict = json.loads(msg.body) msg.delete() with open(f"/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/MNF/json_keys/conversionRates.json") as c: curr = json.load(c) try: usd_inr_rate = curr["rates"]["INR"] # c = CurrencyRates() # rate = c.get_rate("USD", str(currency.upper())) except Exception as e: print("checkout error", e) usd_inr_rate = 80 print("ran till here 1") task_id = body_dict.get("task_id", None) outputfile = body_dict.get("lpp_file", None) if outputfile is not None: outputfile = "INPUT/" + str((outputfile.split("INPUT/"))[-1]) originalfile = body_dict.get("original_file", None) if originalfile is not None: originalfile = "INPUT/" + str((originalfile.split("INPUT/"))[-1]) userr = body_dict.get("userr") print("ran till here 2") if task_id is None: pass lpp_user = LPPTASKDatabase.objects.get(task_id=task_id) # x = LPPTASKDatabase.objects.get(task_id=pk) # lpp_amount = MNFLPPDDatabase.objects.get(lpp_id=lpp_user.assignedlpp_dialogue.lpp_id) if outputfile is None: pass file_user = User.objects.get(id=userr) print("ran till here 3") # object_key = "INPUT/" + (translated_script.script_link_id.script.name.split("/"))[-1] original_file = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + lpp_user.translated_script.script_link_id.script.name local_file_path = "/home/ubuntu/Conversion_Kitchen_Code/kitchen_counter/media/" + lpp_user.outputfile.name s3_client = boto3.client('s3', aws_access_key_id="AKIAQVLBBGCB45RMLKVW", aws_secret_access_key="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", region_name="ap-south-1") print("outputs1", outputfile, local_file_path) print("outputs2", originalfile, original_file) s3_client.download_file("conversion-kitchen", outputfile, local_file_path) s3_client.download_file("conversion-kitchen", originalfile, original_file) print("ran till here 4") try: language_code = "en" name_script = (str(local_file_path).split("."))[0] + "_vetted_a" + "." + (str(local_file_path).split("."))[1] doc_file = f"{local_file_path}" file1 = ContentFile( open(doc_file, 'rb').read(), (doc_file.split("/"))[-1], ) result = filesystem.new_screenplay_without_audit_in_background( file_user, file_user.username, str(name_script), file1, "script-original", language_code, ) print("already called", result) audit_id = result.get("script", {}).get("id") print("ran till here 10") if audit_id is not None: objectt = ScriptTranslations.objects.get(translation_id=lpp_user.translated_script.translation_id) objectt.converted_audit_id = audit_id objectt.save() try: update_audit_status(audit_id, States.STARTED) except: update_audit_status(audit_id, States.FAILURE) # audit = NeutralAudit(audit_id) # status = ScriptAuditModel.objects.get( # script=Script.objects.get( # id=audit_id # ) # ) try: audit = NeutralAudit(audit_id) audit.audit_in_background() ScriptAuditModel.objects.update_or_create( script = Script.objects.get( id = audit_id ), defaults={"status" : "SUCCESS"} ) lpp_user.status = "completed" lpp_user.save() except Exception as exp: ScriptAuditModel.objects.update_or_create(script = Script.objects.get(id = audit_id), defaults={"status" : "FAILURE", "results" : exp}) print("audit will start") except Exception as e: print("Error in auditing vetted output is", e) print("ran till here 11") """Blockchain Upload Starts here""" # sys.stdout = original_stdout status = ScriptAuditModel.objects.get( script = Script.objects.get( id = audit_id )) scriptconversion = {} print("ADDING LPP BALANCE HERE \n\n\n") try: if status.status == "SUCCESS": print("#### STATUS iS SUCESSS @############\n\n") # adding amount to the wallet here lpp_amount = MNFLPPDDatabase.objects.get(lpp_id=lpp_user.assignedlpp_action.lpp_id) print("##### try to add amount here#####\n\n\n") try: lpp_amount.lpp_task_balance = float(Decimal( str(lpp_amount.lpp_task_balance))) + float(float(Decimal(str(lpp_user.amoutgiventolpp_dialogue))) * usd_inr_rate) print("TRY BLOCK COMPLETE") print(str(lpp_amount.lpp_task_balance)) except Exception as e: print("\n\n#####LPP Balance Error", e) lpp_amount.lpp_task_balance = 0.00 + \ float( float(Decimal(lpp_user.amoutgiventolpp_dialogue)) * usd_inr_rate) lpp_amount.save() print("#######the AMOUNT iS ADDED ##########\n\n\n") to = lpp_user.assignedlpp_action.lpp_email key_value = { "User": lpp_user.assignedlpp_action.user_id.username, "Amount": lpp_user.amoutgiventolpp_action } sendmail(to_email=[to], email_code="PP14", key_value=key_value) except Exception as exp: print("error in mail sending PP14") print(exp) try: print("trying blockchain 1") current_datetime = datetime.now() if UserCredentialsForBlockchain.objects.filter(user=lpp_user.user_id).exists(): obj = lpp_user.translated_script obj2 = lpp_user.translated_script.script_link_id # temp1 = str(obj2.script.name) # temp2 = str(lpp_user.outputfile) uploaded_script = f"{original_file}" translated_scripttt = f"{local_file_path}" print("blockchain script conversion 3", uploaded_script, translated_scripttt) with open(uploaded_script, 'rb') as f: hash = uploadDataToIPFSNode(f) scriptconversion["original_scriptFile_hash"] = hash scriptconversion["original_scriptFile_path"] = uploaded_script scriptconversion["date_at"] = current_datetime.strftime("%Y-%m-%d %H:%M:%S") print("blockchain script conversion 4") with open(translated_scripttt, 'rb') as f1: hash = uploadDataToIPFSNode(f1) scriptconversion["translated_scriptFile_hash"] = hash scriptconversion["translated_scriptFile_path"] = translated_scripttt print("blockchain script conversion 5") from centralisedFileSystem.models import File if obj.converted_audit_id: print("audit blockchain") print("blockchain script conversion 5.1") # blockchain upload of csv-json data adit_id = obj.converted_audit_id print("######ADIT ID",adit_id) print("###########@@@@@@@@@ AUDIT ID",audit_id) file_to_audit = File.objects.get( script=audit_id, type="script-csv" ) csv_script_path = file_to_audit.file.path df = pd.read_csv(csv_script_path) df = df.loc[:, ["content", "script_element"]] script_json: dict = json.loads(utilities.csv_to_json(df)) with tempfile.TemporaryDirectory() as temp_dir: print("Temporary directory created:", temp_dir) temp_filename = os.path.join(temp_dir, 'script_json_file.json') print(temp_filename) with open(temp_filename, 'w') as json_file: json.dump(script_json, json_file, indent=4) script_json = {} script_path1 = temp_filename # script_size = file_to_audit_docx.file.size with open(script_path1, 'rb') as _file: hash2 = uploadDataToIPFSNode(_file) script_json["script_file_path"] = script_path1 script_json["script_file"] = hash2 script_json["type"] = "script-json" script_json["screenplay_name"] = str(name_script) scriptconversion["script-json"] = script_json print("blockchain script conversion 5.2") blockchain_obj = UserCredentialsForBlockchain.objects.get(user=lpp_user.user_id) UserId = blockchain_obj.user_id Project = obj.translation_id Data = str(scriptconversion) userPrivateKey = blockchain_obj.privateKey userkey = decryptionOfPrivate(userPrivateKey) tx_id, tx_gas_fee = UploadConversionData(OWNER_KEY, blockchain_obj.publicKey, UserId, str(Project), Data) # lpp_user.translated_script.blockchain_txhash = tx_id print("Tx id -> ", tx_id, tx_gas_fee) # user_infos = user_info(tx_hash=tx_id, service="Conversion", gas_fee=2345432345123456) # addition_result = user_infos.update_info(request) ######## UPLOADING THE FILE FOR LPP USER (LLP SIDE VIEW BUTTON) blockchain_obj_for_lpp = UserCredentialsForBlockchain.objects.get(user=lpp_user.assignedlpp_action.user_id) UserId_for_lpp = blockchain_obj_for_lpp.user_id Data_for_lpp = str(scriptconversion) userPrivateKey_for_lpp = blockchain_obj_for_lpp.privateKey userkey_for_lpp = decryptionOfPrivate(userPrivateKey_for_lpp) tx_id_for_lpp, tx_gas_fee_for_lpp = UploadConversionData(OWNER_KEY, blockchain_obj_for_lpp.publicKey, UserId_for_lpp, str(Project), Data_for_lpp) print("LPP USER BLOCKCHAIN and REAL USER BLOCKCHAIN UPLOADED") print("Project-------->",Project) name_script = (((uploaded_script.split("/"))[-1]).split("."))[0] try: user_infos = user_info(tx_hash=tx_id, service="Conversion", gas_fee=tx_gas_fee, script_name=name_script) request = Request(lpp_user.user_id) addition_result = user_infos.update_info(request) print("Blockchain Result -> ", addition_result) except Exception as e: print("Error in blockchain is:", e) print("blockchain script conversion 6") certificatepath = certificateGenrate(lpp_user.user_id.username, "script conversion", hash) hash = hash_decrypation(hash) to_email = [lpp_user.user_id.email] email_code = 'BL1' key_value = { "service": "script conversion", "hash": hash, "public key": blockchain_obj.publicKey, "private key": userkey.decode('utf-8'), "Transaction Hash": tx_id } sendmail(to_email=to_email, email_code=email_code, key_value=key_value, filePath=certificatepath) import boto3 s3_client = boto3.client('s3', aws_access_key_id="AKIAQVLBBGCB45RMLKVW", aws_secret_access_key="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", region_name="ap-south-1" ) bucket = "conversion-kitchen" # file_name = str(((lpp_user.translated_script.translated_script_pdf).split("OUTPUT/"))[1]) # object_key = "OUTPUT/" + file_name s3_client.delete_object(Bucket=bucket, Key=outputfile) s3_client.delete_object(Bucket=bucket, Key=originalfile) to = lpp_user.translated_script.user_id.email key_value = { "User": lpp_user.translated_script.user_id.username, "title": lpp_user.translated_script.script_link_id.script_title } sendmail(to_email=[to], email_code="PP18", key_value=key_value) try: objectt = ScriptTranslations.objects.get(translation_id=lpp_user.translated_script.translation_id) objectt.blockchain_txhash = tx_id objectt.save() except Exception as e: print("error while saving blockchain txhash is ->",str(e)) except Exception as e: print("Error in blockchain is", e) class Command(BaseCommand): help = 'Custom Command to start django server and then start dequeuing from SQS' def handle(self, *args, **options): # Call the original runserver command" # call_command('runserver', '0.0.0.0:4549') command = 'python manage.py runserver 0.0.0.0:4549 > /home/ubuntu/Conversion_Kitchen_Code/logfile.log 2>&1' # # Execute the command using subprocess.Popen subprocess.Popen(command, shell=True) # Wait for the server to start while True: try: # Try to connect to the server with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect(('0.0.0.0', 4549)) break except ConnectionRefusedError: # If connection is refused, wait and try again time.sleep(1) # Once the server is up, run your function self.run_after_server_startup() def run_after_server_startup(self): # Your function to run after the server starts print("Development server is fully up and running! Run your function here.") session = boto3.Session( aws_access_key_id='AKIAQVLBBGCB45RMLKVW', # replace with your key aws_secret_access_key='ZWc6KOc5LuBLuCEBDDfQTor+Q7rp3fFH74gVt+AA', # replace with your key ) sqs = session.resource('sqs', region_name='ap-south-1') print("Started Executin this from conversion") queue = sqs.get_queue_by_name(QueueName="mnfqueue") # try: while True: messages = queue.receive_messages( MessageAttributeNames=["All"], MaxNumberOfMessages=1, WaitTimeSeconds=5, ) if len(messages) > 0: for msg in messages: try: print("Received message: %s: %s", msg.message_id, msg.body) print(type(msg.body)) body_dict = json.loads(msg.body) if body_dict['service_type'] == "conversion": print("Entering Conversion") print(str(body_dict)) run_conversion(msg) elif body_dict['service_type'] == "audit": run_audit(msg) elif body_dict['service_type'] == "lpp audit": lpp_audit(msg) except Exception as error: print("error execution from queue: %s", error) print(repr(error)) else: break print("Completed All Execution")