Nemcio/CyberPanel
1
0
Fork 0
mirror of https://github.com/usmannasir/cyberpanel.git synced 2025-11-13 08:46:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3uploaddone

Browse Source
This commit is contained in:
Hassan Hashmi
2022-06-29 17:52:35 +05:00
parent aa24855963
commit 8f4ebed722
4 changed files with 321 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

246
plogical/IncScheduler.py
View File

@@ -1,9 +1,11 @@
#!/usr/local/CyberCP/bin/python #!/usr/local/CyberCP/bin/python
import os.path import os.path
import sys import sys
sys.path.append('/usr/local/CyberCP') sys.path.append('/usr/local/CyberCP')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")
import django import django
django.setup() django.setup()
from IncBackups.IncBackupsControl import IncJobs from IncBackups.IncBackupsControl import IncJobs
from IncBackups.models import BackupJob from IncBackups.models import BackupJob
@@ -22,6 +24,7 @@ import requests
import socket import socket
from websiteFunctions.models import NormalBackupJobs, NormalBackupJobLogs from websiteFunctions.models import NormalBackupJobs, NormalBackupJobLogs
from boto3.s3.transfer import TransferConfig from boto3.s3.transfer import TransferConfig
try: try:
from s3Backups.models import BackupPlan, BackupLogs from s3Backups.models import BackupPlan, BackupLogs
import boto3 import boto3
@@ -378,13 +381,10 @@ class IncScheduler(multi.Thread):
GDriveJobLogs(owner=items, status=backupSchedule.INFO, GDriveJobLogs(owner=items, status=backupSchedule.INFO,
message='Job Completed').save() message='Job Completed').save()
print("job com[leted") print("job com[leted")
# logging.writeToFile('job completed') # logging.writeToFile('job completed')
url = "https://platform.cyberpersons.com/CyberpanelAdOns/Adonpermission" url = "https://platform.cyberpersons.com/CyberpanelAdOns/Adonpermission"
data = { data = {
"name": "backups-retention", "name": "backups-retention",
@@ -888,7 +888,6 @@ Automatic backup failed for %s on %s.
email.save() email.save()
print('Disk Usage of %s is %s' % (email.email, email.DiskUsage)) print('Disk Usage of %s is %s' % (email.email, email.DiskUsage))
config['DiskUsage'], config['DiskUsagePercentage'] = virtualHostUtilities.getDiskUsage( config['DiskUsage'], config['DiskUsagePercentage'] = virtualHostUtilities.getDiskUsage(
"/home/" + website.domain, website.package.diskSpace) "/home/" + website.domain, website.package.diskSpace)
@@ -987,108 +986,214 @@ Automatic backup failed for %s on %s.
wpsite = WPSites.objects.get(pk=backupsiteID) wpsite = WPSites.objects.get(pk=backupsiteID)
AdminID = wpsite.owner.admin_id AdminID = wpsite.owner.admin_id
Admin = Administrator.objects.get(pk=AdminID) Admin = Administrator.objects.get(pk=AdminID)
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = "SFTP"
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
Lastrun = config.lastrun Lastrun = config.lastrun
Currenttime = float(time.time()) Currenttime = float(time.time())
if config.timeintervel == "30 Minutes": if config.timeintervel == "30 Minutes":
al = float(Currenttime) - float(1800) al = float(Currenttime) - float(1800)
if float(al) >= float(Lastrun): # if float(al) >= float(Lastrun):
# if 1 == 1: if 1 == 1:
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = config.RemoteBackupConfig.configtype
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
elif config.timeintervel == "1 Hour": elif config.timeintervel == "1 Hour":
al = float(Currenttime) - float(3600) al = float(Currenttime) - float(3600)
if float(al) >= float(Lastrun): if float(al) >= float(Lastrun):
# if 1 == 1:
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = config.RemoteBackupConfig.configtype
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
elif config.timeintervel == "6 Hours": elif config.timeintervel == "6 Hours":
al = float(Currenttime) - float(21600) al = float(Currenttime) - float(21600)
if float(al) >= float(Lastrun): if float(al) >= float(Lastrun):
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = "SFTP"
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
elif config.timeintervel == "12 Hours": elif config.timeintervel == "12 Hours":
al = float(Currenttime) - float(43200) al = float(Currenttime) - float(43200)
if float(al) >= float(Lastrun): if float(al) >= float(Lastrun):
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = "SFTP"
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
elif config.timeintervel == "1 Day": elif config.timeintervel == "1 Day":
al = float(Currenttime) - float(86400) al = float(Currenttime) - float(86400)
if float(al) >= float(Lastrun): if float(al) >= float(Lastrun):
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = "SFTP"
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
elif config.timeintervel == "3 Days": elif config.timeintervel == "3 Days":
al = float(Currenttime) - float(259200) al = float(Currenttime) - float(259200)
if float(al) >= float(Lastrun): if float(al) >= float(Lastrun):
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = "SFTP"
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
elif config.timeintervel == "1 Week": elif config.timeintervel == "1 Week":
al = float(Currenttime) - float(604800) al = float(Currenttime) - float(604800)
if float(al) >= float(Lastrun): if float(al) >= float(Lastrun):
extraArgs = {}
extraArgs['adminID'] = Admin.pk
extraArgs['WPid'] = wpsite.pk
extraArgs['Backuptype'] = Backuptype
extraArgs['BackupDestination'] = "SFTP"
extraArgs['SFTPID'] = config.RemoteBackupConfig_id
extraArgs['tempStatusPath'] = "/home/cyberpanel/" + str(randint(1000, 9999))
background = ApplicationInstaller('WPCreateBackup', extraArgs) background = ApplicationInstaller('WPCreateBackup', extraArgs)
status, msg = background.WPCreateBackup() status, msg, backupID = background.WPCreateBackup()
if status == 1: if status == 1:
filename = msg filename = msg
if config.RemoteBackupConfig.configtype == "SFTP":
IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id) IncScheduler.SendTORemote(filename, config.RemoteBackupConfig_id)
command = f"rm -r {filename}" command = f"rm -r {filename}"
ProcessUtilities.executioner(command) ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id) obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time() obj.lastrun = time.time()
obj.save() obj.save()
elif config.RemoteBackupConfig.configtype == "S3":
IncScheduler.SendToS3Cloud(filename, config.RemoteBackupConfig_id, backupID)
command = f"rm -r {filename}"
ProcessUtilities.executioner(command)
obj = RemoteBackupSchedule.objects.get(pk=config.id)
obj.lastrun = time.time()
obj.save()
except BaseException as msg: except BaseException as msg:
print("Error in Sites:%s" % str(msg)) print("Error in Sites:%s" % str(msg))
continue continue
@@ -1096,7 +1201,6 @@ Automatic backup failed for %s on %s.
print("Error: [RemoteBackup]: %s" % str(msg)) print("Error: [RemoteBackup]: %s" % str(msg))
logging.writeToFile('%s. [RemoteBackup]' % (str(msg))) logging.writeToFile('%s. [RemoteBackup]' % (str(msg)))
@staticmethod @staticmethod
def SendTORemote(FileName, RemoteBackupID): def SendTORemote(FileName, RemoteBackupID):
import pysftp import pysftp
@@ -1118,7 +1222,6 @@ Automatic backup failed for %s on %s.
with pysftp.Connection(HostName, username=Username, password=Password, cnopts=cnopts) as sftp: with pysftp.Connection(HostName, username=Username, password=Password, cnopts=cnopts) as sftp:
print("Connection succesfully stablished ... ") print("Connection succesfully stablished ... ")
try: try:
with sftp.cd(Path): with sftp.cd(Path):
sftp.put(FileName) sftp.put(FileName)
@@ -1132,6 +1235,104 @@ Automatic backup failed for %s on %s.
except BaseException as msg: except BaseException as msg:
logging.writeToFile('%s. [SendTORemote]' % (str(msg))) logging.writeToFile('%s. [SendTORemote]' % (str(msg)))
@staticmethod
def SendToS3Cloud(FileName, RemoteBackupCofigID, backupID):
import boto3
import json
import time
from websiteFunctions.models import RemoteBackupConfig, WPSitesBackup
import plogical.randomPassword as randomPassword
try:
print("UPloading to S3")
Backupobj = WPSitesBackup.objects.get(pk=backupID)
backupConfig = json.loads(Backupobj.config)
websitedomain = backupConfig['WebDomain']
RemoteBackupOBJ = RemoteBackupConfig.objects.get(pk=RemoteBackupCofigID)
config = json.loads(RemoteBackupOBJ.config)
provider = config['Provider']
if provider == "Backblaze":
EndURl = config['EndUrl']
elif provider == "Amazon":
EndURl = "https://s3.us-east-1.amazonaws.com"
elif provider == "Wasabi":
EndURl = "https://s3.wasabisys.com"
AccessKey = config['AccessKey']
SecertKey = config['SecertKey']
session = boto3.session.Session()
client = session.client(
's3',
endpoint_url=EndURl,
aws_access_key_id=AccessKey,
aws_secret_access_key=SecertKey,
verify=False
)
############Creating Bucket
BucketName = randomPassword.generate_pass().lower()
print("BucketName...%s"%BucketName)
try:
client.create_bucket(Bucket=BucketName)
except BaseException as msg:
print("Error in Creating bucket...: %s" % str(msg))
logging.writeToFile("Create bucket error---%s:" % str(msg))
#####Uploading File
uploadfilename = 'backup-' + websitedomain + "-" + time.strftime("%m.%d.%Y_%H-%M-%S")
print("uploadfilename....%s"%uploadfilename)
try:
res = client.upload_file(Filename=FileName, Bucket=BucketName, Key=uploadfilename)
print("res of Uploading...: %s" % res)
except BaseException as msg:
print("Error in Uploading...: %s" % msg)
###################### version id, this only applied to blackbaze
try:
s3 = boto3.resource(
's3',
endpoint_url=EndURl,
aws_access_key_id=AccessKey,
aws_secret_access_key=SecertKey,
)
bucket = BucketName
key = uploadfilename
versions = s3.Bucket(bucket).object_versions.filter(Prefix=key)
data = {}
for version in versions:
obj = version.get()
print("VersionId---%s:" % obj.get('VersionId'))
data['backupVersionId'] = obj.get('VersionId')
ab = os.path.getsize(FileName)
filesize = float(ab) / 1024.0
backupConfig['uploadfilename'] = uploadfilename
backupConfig['backupVersionId'] = data['backupVersionId']
backupConfig['BucketName'] = BucketName
backupConfig['Uplaodingfilesize'] = filesize
Backupobj.config = json.dumps(backupConfig)
Backupobj.save()
except BaseException as msg:
print("Version ID Error: %s"%str(msg))
except BaseException as msg:
print('%s. [SendToS3Cloud]' % (str(msg)))
logging.writeToFile('%s. [SendToS3Cloud]' % (str(msg)))
def main(): def main():
@@ -1152,7 +1353,6 @@ def main():
ib = IncScheduler('startBackup', {'freq': args.function}) ib = IncScheduler('startBackup', {'freq': args.function})
ib.start() ib.start()
### ###
IncScheduler.startBackup(args.function) IncScheduler.startBackup(args.function)

8
plogical/applicationInstaller.py
View File

@@ -2444,7 +2444,7 @@ $parameters = array(
raise BaseException(stdout) raise BaseException(stdout)
logging.statusWriter(self.tempStatusPath, 'Completed.[200]') logging.statusWriter(self.tempStatusPath, 'Completed.[200]')
return 1, f"/home/backup/{config['name']}.tar.gz" return 1, f"/home/backup/{config['name']}.tar.gz", backupobj.id
#### Only Website Data === 2 #### Only Website Data === 2
elif Backuptype == "2": elif Backuptype == "2":
@@ -2576,7 +2576,7 @@ $parameters = array(
raise BaseException(stdout) raise BaseException(stdout)
logging.statusWriter(self.tempStatusPath, 'Completed.[200]') logging.statusWriter(self.tempStatusPath, 'Completed.[200]')
return 1, f"/home/backup/{config['name']}.tar.gz" return 1, f"/home/backup/{config['name']}.tar.gz", backupobj.id
#### Only Database === 3 #### Only Database === 3
else: else:
@@ -2705,7 +2705,7 @@ $parameters = array(
raise BaseException(stdout) raise BaseException(stdout)
logging.statusWriter(self.tempStatusPath, 'Completed.[200]') logging.statusWriter(self.tempStatusPath, 'Completed.[200]')
return 1, f"/home/backup/{config['name']}.tar.gz" return 1, f"/home/backup/{config['name']}.tar.gz", backupobj.id
except BaseException as msg: except BaseException as msg:
logging.writeToFile("Error WPCreateBackup ....... %s" % str(msg)) logging.writeToFile("Error WPCreateBackup ....... %s" % str(msg))
@@ -2715,7 +2715,7 @@ $parameters = array(
except: except:
pass pass
logging.statusWriter(self.tempStatusPath, f'{str(msg)}. [404]') logging.statusWriter(self.tempStatusPath, f'{str(msg)}. [404]')
return 0, str(msg) return 0, str(msg), None
def RestoreWPbackupNow(self): def RestoreWPbackupNow(self):
try: try:

9
plogical/test.py
View File

@@ -0,0 +1,9 @@
config= {
"name": "habi",
"dev": "python"
}
print(config)
config['klm'] ="tls"
print(config)

2
websiteFunctions/templates/websiteFunctions/RemoteBackupConfig.html
View File

@@ -105,7 +105,7 @@
<div ng-hide="SecretKeydiv" class="form-group"> <div ng-hide="SecretKeydiv" class="form-group">
<label class="col-sm-3 control-label">{% trans "Secret Key" %}</label> <label class="col-sm-3 control-label">{% trans "Secret Key" %}</label>
<div class="col-sm-6"> <div class="col-sm-6">
<input name="SecretKey" type="text" class="form-control" ng-model="SecretKey" <input name="SecretKey" type="password" class="form-control" ng-model="SecretKey"
required> required>
</div> </div>
</div> </div>