incremental backups: stage 5: generating backups

This commit is contained in:
Usman Nasir
2019-10-04 15:56:53 +05:00
parent a3ec139254
commit d81d00d8ca
9 changed files with 511 additions and 18 deletions

View File

@@ -60,7 +60,7 @@ class secMiddleware:
if request.build_absolute_uri().find('saveSpamAssassinConfigurations') > -1 or request.build_absolute_uri().find('docker') > -1 or request.build_absolute_uri().find('cloudAPI') > -1 or request.build_absolute_uri().find('filemanager') > -1 or request.build_absolute_uri().find('verifyLogin') > -1 or request.build_absolute_uri().find('submitUserCreation') > -1:
continue
if key == 'ports' or key == 'imageByPass' or key == 'passwordByPass' or key == 'cronCommand' or key == 'emailMessage' or key == 'configData' or key == 'rewriteRules' or key == 'modSecRules' or key == 'recordContentTXT' or key == 'SecAuditLogRelevantStatus' or key == 'fileContent':
if key == 'backupDestinations' or key == 'ports' or key == 'imageByPass' or key == 'passwordByPass' or key == 'cronCommand' or key == 'emailMessage' or key == 'configData' or key == 'rewriteRules' or key == 'modSecRules' or key == 'recordContentTXT' or key == 'SecAuditLogRelevantStatus' or key == 'fileContent':
continue
if value.find(';') > -1 or value.find('&&') > -1 or value.find('|') > -1 or value.find('...') > -1 \
or value.find("`") > -1 or value.find("$") > -1 or value.find("(") > -1 or value.find(")") > -1 \

View File

@@ -9,6 +9,18 @@ from plogical.processUtilities import ProcessUtilities
import time
from .models import IncJob, JobSnapshots
from websiteFunctions.models import Websites
import plogical.randomPassword as randomPassword
from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
from xml.etree.ElementTree import Element, SubElement
from xml.etree import ElementTree
from xml.dom import minidom
from backup.models import DBUsers
import plogical.mysqlUtilities as mysqlUtilities
from plogical.backupUtilities import backupUtilities
from plogical.dnsUtilities import DNS
from mailServer.models import Domains as eDomains
from random import randint
class IncJobs(multi.Thread):
@@ -16,27 +28,348 @@ class IncJobs(multi.Thread):
multi.Thread.__init__(self)
self.function = function
self.extraArgs = extraArgs
self.repoPath = ''
self.passwordFile = ''
self.statusPath = ''
self.website = ''
self.backupDestinations = ''
self.jobid = 0
def run(self):
if self.function == 'createBackup':
self.createBackup()
def prepareBackupMeta(self):
try:
######### Generating meta
## XML Generation
metaFileXML = Element('metaFile')
child = SubElement(metaFileXML, 'masterDomain')
child.text = self.website.domain
child = SubElement(metaFileXML, 'phpSelection')
child.text = self.website.phpSelection
child = SubElement(metaFileXML, 'externalApp')
child.text = self.website.externalApp
childDomains = self.website.childdomains_set.all()
databases = self.website.databases_set.all()
## Child domains XML
childDomainsXML = Element('ChildDomains')
for items in childDomains:
childDomainXML = Element('domain')
child = SubElement(childDomainXML, 'domain')
child.text = items.domain
child = SubElement(childDomainXML, 'phpSelection')
child.text = items.phpSelection
child = SubElement(childDomainXML, 'path')
child.text = items.path
childDomainsXML.append(childDomainXML)
metaFileXML.append(childDomainsXML)
## Databases XML
databasesXML = Element('Databases')
for items in databases:
try:
dbuser = DBUsers.objects.get(user=items.dbUser)
userToTry = items.dbUser
except:
dbusers = DBUsers.objects.all().filter(user=items.dbUser)
userToTry = items.dbUser
for it in dbusers:
dbuser = it
break
userToTry = mysqlUtilities.mysqlUtilities.fetchuser(items.dbUser)
try:
dbuser = DBUsers.objects.get(user=userToTry)
except:
dbusers = DBUsers.objects.all().filter(user=userToTry)
for it in dbusers:
dbuser = it
break
databaseXML = Element('database')
child = SubElement(databaseXML, 'dbName')
child.text = items.dbName
child = SubElement(databaseXML, 'dbUser')
child.text = userToTry
child = SubElement(databaseXML, 'password')
child.text = dbuser.password
databasesXML.append(databaseXML)
metaFileXML.append(databasesXML)
## Get Aliases
aliasesXML = Element('Aliases')
aliases = backupUtilities.getAliases(self.website.domain)
for items in aliases:
child = SubElement(aliasesXML, 'alias')
child.text = items
metaFileXML.append(aliasesXML)
## Finish Alias
## DNS Records XML
try:
dnsRecordsXML = Element("dnsrecords")
dnsRecords = DNS.getDNSRecords(self.website.domain)
for items in dnsRecords:
dnsRecordXML = Element('dnsrecord')
child = SubElement(dnsRecordXML, 'type')
child.text = items.type
child = SubElement(dnsRecordXML, 'name')
child.text = items.name
child = SubElement(dnsRecordXML, 'content')
child.text = items.content
child = SubElement(dnsRecordXML, 'priority')
child.text = str(items.prio)
dnsRecordsXML.append(dnsRecordXML)
metaFileXML.append(dnsRecordsXML)
except BaseException, msg:
logging.statusWriter(self.statusPath, '%s. [158:prepMeta]' % (str(msg)), 1)
## Email accounts XML
try:
emailRecordsXML = Element('emails')
eDomain = eDomains.objects.get(domain=self.website.domain)
emailAccounts = eDomain.eusers_set.all()
for items in emailAccounts:
emailRecordXML = Element('emailAccount')
child = SubElement(emailRecordXML, 'email')
child.text = items.email
child = SubElement(emailRecordXML, 'password')
child.text = items.password
emailRecordsXML.append(emailRecordXML)
metaFileXML.append(emailRecordsXML)
except BaseException, msg:
logging.writeToFile(self.statusPath, '%s. [warning:179:prepMeta]' % (str(msg)), 1)
## Email meta generated!
def prettify(elem):
"""Return a pretty-printed XML string for the Element.
"""
rough_string = ElementTree.tostring(elem, 'utf-8')
reparsed = minidom.parseString(rough_string)
return reparsed.toprettyxml(indent=" ")
## /home/example.com/backup/backup-example-06-50-03-Thu-Feb-2018/meta.xml -- metaPath
metaPath = '/home/cyberpanel/%s' % (str(randint(1000, 9999)))
xmlpretty = prettify(metaFileXML).encode('ascii', 'ignore')
metaFile = open(metaPath, 'w')
metaFile.write(xmlpretty)
metaFile.close()
os.chmod(metaPath, 0640)
## meta generated
logging.statusWriter(self.statusPath, 'Meta data is ready..', 1)
metaPathNew = '/home/%s/meta.xml' % (self.website.domain)
command = 'mv %s %s' % (metaPath, metaPathNew)
ProcessUtilities.executioner(command)
command = 'chown %s:%s %s' % (self.website.externalApp, self.website.externalApp, metaPathNew)
ProcessUtilities.executioner(command)
return 1
except BaseException, msg:
logging.statusWriter(self.statusPath, "%s [207][5009]" % (str(msg)), 1)
return 0
def backupData(self):
try:
logging.statusWriter(self.statusPath, 'Backing up data..', 1)
if self.backupDestinations == 'local':
backupPath = '/home/%s' % (self.website.domain)
command = 'restic -r %s backup %s --password-file %s --exclude %s' % (self.repoPath, backupPath, self.passwordFile, self.repoPath)
snapShotid = ProcessUtilities.outputExecutioner(command).split(' ')[-2]
newSnapshot = JobSnapshots(job=self.jobid, type='data:%s' % (backupPath), snapshotid=snapShotid, destination=self.backupDestinations)
newSnapshot.save()
elif self.backupDestinations[:4] == 'sftp':
remotePath = '/home/backup/%s' % (self.website.domain)
backupPath = '/home/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (self.backupDestinations, remotePath, backupPath, self.passwordFile, self.repoPath)
snapShotid = ProcessUtilities.outputExecutioner(command).split(' ')[-2]
newSnapshot = JobSnapshots(job=self.jobid, type='data:%s' % (remotePath), snapshotid=snapShotid,
destination=self.backupDestinations)
newSnapshot.save()
logging.statusWriter(self.statusPath, 'Data for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
return 1
except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.backupData.223][5009]' % str(msg), 1)
return 0
def backupDatabases(self):
try:
logging.statusWriter(self.statusPath, 'Backing up databases..', 1)
databases = self.website.databases_set.all()
for items in databases:
if mysqlUtilities.mysqlUtilities.createDatabaseBackup(items.dbName, '/home/cyberpanel') == 0:
return 0
dbPath = '/home/cyberpanel/%s.sql' % (items.dbName)
if self.backupDestinations == 'local':
command = 'restic -r %s backup %s --password-file %s' % (self.repoPath, dbPath, self.passwordFile)
snapShotid = ProcessUtilities.outputExecutioner(command).split(' ')[-2]
newSnapshot = JobSnapshots(job=self.jobid, type='database:%s' % (items.dbName), snapshotid=snapShotid, destination=self.backupDestinations)
newSnapshot.save()
elif self.backupDestinations[:4] == 'sftp':
remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (
self.backupDestinations, remotePath, dbPath, self.passwordFile, self.repoPath)
snapShotid = ProcessUtilities.outputExecutioner(command).split(' ')[-2]
newSnapshot = JobSnapshots(job=self.jobid, type='database:%s' % (items.dbName), snapshotid=snapShotid,
destination=self.backupDestinations)
newSnapshot.save()
return 1
except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.backupDatabases.269][5009]' % str(msg), 1)
return 0
def emailBackup(self):
try:
logging.statusWriter(self.statusPath, 'Backing up emails..', 1)
backupPath = '/home/vmail/%s' % (self.website.domain)
if os.path.exists(backupPath):
if self.backupDestinations == 'local':
logging.statusWriter(self.statusPath, 'hello world', 1)
command = 'restic -r %s backup %s --password-file %s' % (
self.repoPath, backupPath, self.passwordFile)
snapShotid = ProcessUtilities.outputExecutioner(command).split(' ')[-2]
newSnapshot = JobSnapshots(job=self.jobid, type='email:%s' % (backupPath), snapshotid=snapShotid,
destination=self.backupDestinations)
newSnapshot.save()
logging.statusWriter(self.statusPath, 'hello world 2', 1)
elif self.backupDestinations[:4] == 'sftp':
remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (
self.backupDestinations, remotePath, backupPath, self.passwordFile, self.repoPath)
snapShotid = ProcessUtilities.outputExecutioner(command).split(' ')[-2]
newSnapshot = JobSnapshots(job=self.jobid, type='email:%s' % (backupPath), snapshotid=snapShotid,
destination=self.backupDestinations)
newSnapshot.save()
logging.statusWriter(self.statusPath, 'Emails for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
return 1
except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.backupDatabases.269][5009]' % str(msg), 1)
return 0
def initiateRepo(self):
try:
logging.statusWriter(self.statusPath, 'Will first initiate backup repo..', 1)
if self.backupDestinations == 'local':
command = 'restic init --repo %s --password-file %s' % (self.repoPath, self.passwordFile)
ProcessUtilities.executioner(command, self.website.externalApp)
elif self.backupDestinations[:4] == 'sftp':
remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic init --repo %s:%s --password-file %s' % (self.backupDestinations, remotePath, self.passwordFile)
ProcessUtilities.executioner(command)
logging.statusWriter(self.statusPath, 'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1)
return 1
except BaseException, msg:
logging.statusWriter(self.statusPath,'%s. [IncJobs.initiateRepo.47][5009]' % str(msg), 1)
return 0
def createBackup(self):
tempPath = self.extraArgs['tempPath']
self.statusPath = self.extraArgs['tempPath']
website = self.extraArgs['website']
backupDestinations = self.extraArgs['backupDestinations']
self.backupDestinations = self.extraArgs['backupDestinations']
websiteData = self.extraArgs['websiteData']
websiteEmails = self.extraArgs['websiteEmails']
websiteSSLs = self.extraArgs['websiteSSLs']
websiteDatabases = self.extraArgs['websiteDatabases']
website = Websites.objects.get(domain=website)
self.website = Websites.objects.get(domain=website)
newJob = IncJob(website=website)
newJob = IncJob(website=self.website)
newJob.save()
writeToFile = open(tempPath, 'w')
writeToFile.write('Completed')
writeToFile.close()
self.jobid = newJob
self.passwordFile = '/home/%s/%s' % (self.website.domain, self.website.domain)
password = randomPassword.generate_pass()
self.repoPath = '/home/%s/incbackup' % (self.website.domain)
if not os.path.exists(self.passwordFile):
command = 'echo "%s" > %s' % (password, self.passwordFile)
ProcessUtilities.executioner(command, self.website.externalApp)
if self.initiateRepo() == 0:
return
if self.prepareBackupMeta() == 0:
return
if websiteData:
if self.backupData() == 0:
return
if websiteDatabases:
if self.backupDatabases() == 0:
return
if websiteEmails:
if self.emailBackup() == 0:
return
logging.statusWriter(self.statusPath, 'Completed', 1)

View File

@@ -10,5 +10,6 @@ class IncJob(models.Model):
class JobSnapshots(models.Model):
job = models.ForeignKey(IncJob)
type = models.CharField(max_length=50)
type = models.CharField(max_length=300)
snapshotid = models.CharField(max_length=50)
destination = models.CharField(max_length=200, default='')

View File

@@ -133,7 +133,8 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
backupDestinations: $scope.backupDestinations,
websiteData: $scope.websiteData,
websiteEmails: $scope.websiteEmails,
websiteSSLs: $scope.websiteSSLs
websiteSSLs: $scope.websiteSSLs,
websiteDatabases: $scope.websiteDatabases
};
@@ -198,6 +199,52 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
};
$scope.restore = function (id) {
$scope.cyberpanelLoading = false;
url = "/IncrementalBackups/fetchRestorePoints";
var data = {
id: id
};
var config = {
headers: {
'X-CSRFToken': getCookie('csrftoken')
}
};
$http.post(url, data, config).then(ListInitialDatas, cantLoadInitialDatas);
function ListInitialDatas(response) {
$scope.cyberpanelLoading = true;
if (response.data.status === 1) {
$scope.jobs = JSON.parse(response.data.data);
} else {
new PNotify({
title: 'Operation Failed!',
text: response.data.error_message,
type: 'error'
});
}
}
function cantLoadInitialDatas(response) {
$scope.cyberpanelLoading = true;
new PNotify({
title: 'Operation Failed!',
text: 'Could not connect to server, please refresh this page',
type: 'error'
});
}
};
});

View File

@@ -63,6 +63,15 @@
</div>
</div>
<label class="col-sm-3 control-label"></label>
<div class="col-sm-9">
<div class="checkbox">
<label>
<input ng-model="websiteDatabases" type="checkbox" value="">
Databases
</label>
</div>
</div>
<label class="col-sm-3 control-label"></label>
<div class="col-sm-9">
<div class="checkbox">
<label>
@@ -71,6 +80,7 @@
</label>
</div>
</div>
<!---
<label class="col-sm-3 control-label"></label>
<div class="col-sm-9">
<div class="checkbox">
@@ -79,7 +89,7 @@
SSL Certificates
</label>
</div>
</div>
</div> -->
</div>
@@ -116,7 +126,7 @@
<tr>
<th>{% trans "ID" %}</th>
<th>{% trans "Date" %}</th>
<th>{% trans "Includes" %}</th>
<th>{% trans "Restore" %}</th>
<th>{% trans "Delete" %}</th>
</tr>
</thead>
@@ -124,7 +134,63 @@
<tr ng-repeat="record in records track by $index">
<td ng-bind="record.id"></td>
<td ng-bind="record.date"></td>
<td ng-bind="record.includes"></td>
<td>
<a ng-click="restore(record.id)" data-toggle="modal" data-target="#settings"
ng-click='deleteCLPackage()'
class="btn btn-border btn-alt border-green btn-link font-green"
title=""><span>Restore Points</span></a>
<div id="settings" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal">
&times;
</button>
<h4 class="modal-title">Restore Points
<img id="cyberpanelLoading"
src="/static/images/loading.gif"
style="display: none;">
</h4>
</div>
<div class="modal-body">
<table class="table">
<thead>
<tr>
<th>{% trans "Job ID" %}</th>
<th>{% trans "Snapshot ID" %}</th>
<th>{% trans "Type" %}</th>
<th>{% trans "Destination" %}</th>
<th>{% trans "Action" %}</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="job in jobs track by $index">
<td ng-bind="job.id"></td>
<td ng-bind="job.snapshotid"></td>
<td ng-bind="job.type"></td>
<td ng-bind="job.destination"></td>
<td>
<a class="btn btn-border btn-alt border-green btn-link font-green"
title=""><span>Restore</span></a>
</td>
</tr>
</tbody>
</table>
</div>
<div class="modal-footer">
<button type="button" ng-disabled="savingSettings"
class="btn btn-default" data-dismiss="modal">
Close
</button>
</div>
</div>
</div>
</div>
</td>
<a href="">
<td ng-click="deleteBackup(record.id)"><img
src="{% static 'images/delete.png' %}"></td>

View File

@@ -11,4 +11,5 @@ urlpatterns = [
url(r'^submitBackupCreation$', views.submitBackupCreation, name='submitBackupCreationInc'),
url(r'^getBackupStatus$', views.getBackupStatus, name='getBackupStatusInc'),
url(r'^deleteBackup$', views.deleteBackup, name='deleteBackupInc'),
url(r'^fetchRestorePoints$', views.fetchRestorePoints, name='fetchRestorePointsInc'),
]

View File

@@ -284,7 +284,7 @@ def fetchCurrentBackups(request):
website = Websites.objects.get(domain=backupDomain)
backups = website.incjob_set.all()
backups = website.incjob_set.all().reverse()
json_data = "["
checker = 0
@@ -349,6 +349,12 @@ def submitBackupCreation(request):
except:
websiteSSLs = False
try:
websiteDatabases = data['websiteDatabases']
except:
websiteDatabases = False
extraArgs = {}
extraArgs['website'] = backupDomain
extraArgs['tempPath'] = tempPath
@@ -356,6 +362,7 @@ def submitBackupCreation(request):
extraArgs['websiteData'] = websiteData
extraArgs['websiteEmails'] = websiteEmails
extraArgs['websiteSSLs'] = websiteSSLs
extraArgs['websiteDatabases'] = websiteDatabases
startJob = IncJobs('createBackup', extraArgs)
startJob.start()
@@ -427,7 +434,6 @@ def getBackupStatus(request):
logging.writeToFile(str(msg) + " [backupStatus]")
return HttpResponse(final_json)
def deleteBackup(request):
try:
userID = request.session['userID']
@@ -450,3 +456,41 @@ def deleteBackup(request):
final_dic = {'destStatus': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
def fetchRestorePoints(request):
try:
userID = request.session['userID']
currentACL = ACLManager.loadedACL(userID)
admin = Administrator.objects.get(pk=userID)
data = json.loads(request.body)
id = data['id']
incJob = IncJob.objects.get(id=id)
backups = incJob.jobsnapshots_set.all()
json_data = "["
checker = 0
for items in backups:
dic = {'id': items.id,
'snapshotid': items.snapshotid,
'type': items.type,
'destination': items.destination,
}
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
json_data = json_data + ']'
final_json = json.dumps({'status': 1, 'error_message': "None", "data": json_data})
return HttpResponse(final_json)
except BaseException, msg:
final_dic = {'status': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)

View File

@@ -7,7 +7,6 @@ django.setup()
import threading as multi
from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
import subprocess
import shlex
from vhost import vhost
from websiteFunctions.models import ChildDomains, Websites
import randomPassword

View File

@@ -189,11 +189,13 @@ class ProcessUtilities(multi.Thread):
# for items in CommandArgs:
# finalCommand = '%s %s' % (finalCommand, items)
#logging.writeToFile(command)
if user == None:
logging.writeToFile(ProcessUtilities.token + command)
sock.sendall(ProcessUtilities.token + command)
else:
command = '%s-u %s %s' % (ProcessUtilities.token, user, command)
logging.writeToFile(command)
command = command.replace('sudo', '')
sock.sendall(command)