incremental backups: stage 4: job structure

This commit is contained in:
Usman Nasir
2019-10-02 16:08:31 +05:00
parent 398d74ea3b
commit a3ec139254
10 changed files with 400 additions and 201 deletions

42
IncBackups/IncBackups.py Normal file
View File

@@ -0,0 +1,42 @@
#!/usr/local/CyberCP/bin/python2
import os,sys
sys.path.append('/usr/local/CyberCP')
import django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")
django.setup()
import threading as multi
from plogical.processUtilities import ProcessUtilities
import time
from .models import IncJob, JobSnapshots
from websiteFunctions.models import Websites
class IncJobs(multi.Thread):
def __init__(self, function, extraArgs):
multi.Thread.__init__(self)
self.function = function
self.extraArgs = extraArgs
def run(self):
if self.function == 'createBackup':
self.createBackup()
def createBackup(self):
tempPath = self.extraArgs['tempPath']
website = self.extraArgs['website']
backupDestinations = self.extraArgs['backupDestinations']
websiteData = self.extraArgs['websiteData']
websiteEmails = self.extraArgs['websiteEmails']
websiteSSLs = self.extraArgs['websiteSSLs']
website = Websites.objects.get(domain=website)
newJob = IncJob(website=website)
newJob.save()
writeToFile = open(tempPath, 'w')
writeToFile.write('Completed')
writeToFile.close()

View File

@@ -1,6 +1,14 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
from websiteFunctions.models import Websites
from datetime import datetime
# Create your models here.
class IncJob(models.Model):
website = models.ForeignKey(Websites)
date = models.DateTimeField(default=datetime.now, blank=True)
class JobSnapshots(models.Model):
job = models.ForeignKey(IncJob)
type = models.CharField(max_length=50)
snapshotid = models.CharField(max_length=50)

View File

@@ -6,50 +6,24 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
$scope.backupButton = true;
$scope.cyberpanelLoading = true;
$scope.runningBackup = true;
$scope.cancelButton = true;
populateCurrentRecords();
$scope.cancelBackup = function () {
var backupCancellationDomain = $scope.websiteToBeBacked;
url = "/backup/cancelBackupCreation";
var data = {
backupCancellationDomain: backupCancellationDomain,
fileName: $scope.fileName,
};
var config = {
headers: {
'X-CSRFToken': getCookie('csrftoken')
}
};
$http.post(url, data, config).then(ListInitialDatas, cantLoadInitialDatas);
};
$scope.fetchDetails = function () {
getBackupStatus();
populateCurrentRecords();
$scope.populateCurrentRecords();
$scope.destination = false;
$scope.runningBackup = true;
};
function getBackupStatus() {
$scope.cyberpanelLoadingBottom = false;
var websiteToBeBacked = $scope.websiteToBeBacked;
url = "/backup/backupStatus";
url = "/IncrementalBackups/getBackupStatus";
var data = {
websiteToBeBacked: websiteToBeBacked,
websiteToBeBacked: $scope.websiteToBeBacked,
tempPath: $scope.tempPath
};
var config = {
@@ -72,18 +46,16 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
$scope.cyberpanelLoadingBottom = true;
$scope.destination = false;
$scope.runningBackup = false;
$scope.cancelButton = true;
$scope.backupButton = false;
$scope.cyberpanelLoading = true;
$scope.fileName = response.data.fileName;
$scope.status = response.data.status;
populateCurrentRecords();
$scope.populateCurrentRecords();
return;
} else {
$scope.destination = true;
$scope.backupButton = true;
$scope.runningBackup = false;
$scope.cancelButton = false;
$scope.fileName = response.data.fileName;
$scope.status = response.data.status;
@@ -94,7 +66,6 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
$timeout.cancel();
$scope.cyberpanelLoadingBottom = true;
$scope.cyberpanelLoading = true;
$scope.cancelButton = true;
$scope.backupButton = false;
}
@@ -103,22 +74,18 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
function cantLoadInitialDatas(response) {
}
};
}
$scope.destinationSelection = function () {
$scope.backupButton = false;
};
$scope.populateCurrentRecords = function () {
function populateCurrentRecords() {
var websiteToBeBacked = $scope.websiteToBeBacked;
url = "/backup/getCurrentBackups";
url = "/IncrementalBackups/fetchCurrentBackups";
var data = {
websiteToBeBacked: websiteToBeBacked,
websiteToBeBacked: $scope.websiteToBeBacked,
};
var config = {
@@ -132,31 +99,42 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
function ListInitialDatas(response) {
if (response.data.fetchStatus == 1) {
if (response.data.status === 1) {
$scope.records = JSON.parse(response.data.data);
} else {
new PNotify({
title: 'Error!',
text: response.data.error_message,
type: 'error'
});
}
}
function cantLoadInitialDatas(response) {
new PNotify({
title: 'Operation Failed!',
text: 'Could not connect to server, please refresh this page',
type: 'error'
});
}
};
$scope.createBackup = function () {
var websiteToBeBacked = $scope.websiteToBeBacked;
$scope.cyberpanelLoading = false;
url = "/backup/submitBackupCreation";
url = "/IncrementalBackups/submitBackupCreation";
var data = {
websiteToBeBacked: websiteToBeBacked,
websiteToBeBacked: $scope.websiteToBeBacked,
backupDestinations: $scope.backupDestinations,
websiteData: $scope.websiteData,
websiteEmails: $scope.websiteEmails,
websiteSSLs: $scope.websiteSSLs
};
var config = {
@@ -171,8 +149,8 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
function ListInitialDatas(response) {
if (response.data.metaStatus === 1) {
if (response.data.status === 1) {
$scope.tempPath = response.data.tempPath;
getBackupStatus();
}
@@ -183,11 +161,10 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
};
$scope.deleteBackup = function (id) {
url = "/backup/deleteBackup";
url = "/IncrementalBackups/deleteBackup";
var data = {
backupID: id,
@@ -206,20 +183,16 @@ app.controller('createIncrementalBackups', function ($scope, $http, $timeout) {
function ListInitialDatas(response) {
if (response.data.deleteStatus == 1) {
if (response.data.status === 1) {
populateCurrentRecords();
$scope.populateCurrentRecords();
} else {
}
}
function cantLoadInitialDatas(response) {
}
@@ -256,9 +229,9 @@ app.controller('incrementalDestinations', function ($scope, $http) {
url = "/IncrementalBackups/populateCurrentRecords";
var type = 'SFTP';
if ($scope.destinationType === 'SFTP'){
if ($scope.destinationType === 'SFTP') {
type = 'SFTP';
}else{
} else {
type = 'AWS';
}
@@ -307,14 +280,14 @@ app.controller('incrementalDestinations', function ($scope, $http) {
url = "/IncrementalBackups/addDestination";
if(type === 'SFTP'){
if (type === 'SFTP') {
var data = {
type: type,
IPAddress: $scope.IPAddress,
password: $scope.password,
backupSSHPort: $scope.backupSSHPort
};
}else {
} else {
var data = {
type: type,
AWS_ACCESS_KEY_ID: $scope.AWS_ACCESS_KEY_ID,

View File

@@ -1,52 +1,58 @@
{% extends "baseTemplate/index.html" %}
{% load i18n %}
{% block title %}{% trans "Create Incremental Backup" %}{% endblock %}
{% block content %}
{% load static %}
{% load static %}
{% get_current_language as LANGUAGE_CODE %}
<!-- Current language: {{ LANGUAGE_CODE }} -->
{% get_current_language as LANGUAGE_CODE %}
<!-- Current language: {{ LANGUAGE_CODE }} -->
<div class="container">
<div id="page-title">
<h2>{% trans "Back up Website" %} - <a target="_blank" href="http://go.cyberpanel.net/backup" style="height: 23px;line-height: 21px;" class="btn btn-border btn-alt border-red btn-link font-red" title=""><span>{% trans "Backup Docs" %}</span></a></h2>
<p>{% trans "This page can be used to create incremental backups for your websites." %}</p>
</div>
<div class="container">
<div id="page-title">
<h2>{% trans "Back up Website" %} - <a target="_blank" href="http://go.cyberpanel.net/backup"
style="height: 23px;line-height: 21px;"
class="btn btn-border btn-alt border-red btn-link font-red" title=""><span>{% trans "Backup Docs" %}</span></a>
</h2>
<p>{% trans "This page can be used to create incremental backups for your websites." %}</p>
</div>
<div ng-controller="createIncrementalBackups" class="panel">
<div class="panel-body">
<h3 class="title-hero">
{% trans "Back up Website" %} <img ng-hide="backupLoading" src="{% static 'images/loading.gif' %}">
</h3>
<div class="example-box-wrapper">
<div ng-controller="createIncrementalBackups" class="panel">
<div class="panel-body">
<h3 class="title-hero">
{% trans "Back up Website" %} <img ng-hide="cyberpanelLoading"
src="{% static 'images/loading.gif' %}">
</h3>
<div class="example-box-wrapper">
<form action="/" class="form-horizontal bordered-row">
<form action="/" class="form-horizontal bordered-row">
<div class="form-group">
<label class="col-sm-3 control-label">{% trans "Select Website" %} </label>
<div class="col-sm-6">
<select ng-change="fetchDetails()" ng-model="websiteToBeBacked" class="form-control">
{% for items in websiteList %}
<option>{{ items }}</option>
{% endfor %}
</select>
</div>
</div>
<div class="form-group">
<label class="col-sm-3 control-label">{% trans "Select Website" %} </label>
<div class="col-sm-6">
<select ng-change="fetchDetails()" ng-model="websiteToBeBacked" class="form-control">
{% for items in websiteList %}
<option>{{ items }}</option>
{% endfor %}
</select>
</div>
</div>
<div ng-hide="destination" class="form-group">
<label class="col-sm-3 control-label">{% trans "Destination" %}</label>
<div class="col-sm-6">
<select ng-change="destinationSelection()" ng-model="backupDestinations" class="form-control">
<option>{% trans "Local" %}</option>
</select>
</div>
</div>
<div ng-hide="destination" class="form-group">
<label class="col-sm-3 control-label">{% trans "Destination" %}</label>
<div class="col-sm-6">
<select ng-change="destinationSelection()" ng-model="backupDestinations"
class="form-control">
{% for items in destinations %}
<option>{{ items }}</option>
{% endfor %}
</select>
</div>
</div>
<div ng-hide="destination" class="form-group">
<div ng-hide="destination" class="form-group">
<label class="col-sm-3 control-label">{% trans "Backup Content" %}</label>
<div class="col-sm-9">
<div class="checkbox">
@@ -77,110 +83,70 @@
</div>
<!---- if Back up is running ----->
<!---- if Back up is running ----->
<div ng-hide="runningBackup" class="form-group">
<div class="col-sm-12">
<table class="table">
<thead>
<tr>
<th></th>
<th>{% trans "File Name" %}</th>
<th>{% trans "Status" %} <img ng-hide="backupLoadingBottom" src="{% static 'images/loading.gif' %}"></th>
</tr>
</thead>
<tbody>
<tr>
<td>{% trans "Running" %}</td>
<td>{$ fileName $}</td>
<td style="color: red"><strong>{$ status $}</strong></td>
</tr>
</tbody>
</table>
</div>
</div>
<!---- if Back up is running------>
<div ng-hide="backupButton" class="form-group">
<label class="col-sm-3 control-label"></label>
<div class="col-sm-4">
<button type="button" ng-click="createBackup()" class="btn btn-primary btn-lg btn-block">{% trans "Create Back up" %}</button>
</div>
</div>
<div ng-hide="cancelButton" class="form-group">
<label class="col-sm-3 control-label"></label>
<div class="col-sm-4">
<button type="button" ng-click="cancelBackup()" class="btn btn-primary btn-lg btn-block">{% trans "Cancel Backup" %}</button>
</div>
</div>
<!------ List of records --------------->
<div class="form-group">
<div class="col-sm-12">
<table class="table">
<thead>
<tr>
<th>{% trans "ID" %}</th>
<th>{% trans "File" %}</th>
<th>{% trans "Date" %}</th>
<th>{% trans "Size" %}</th>
<th>{% trans "Status" %}</th>
<th>{% trans "Delete" %}</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="record in records track by $index">
<td ng-bind="record.id"></td>
<td ng-bind="record.file"></td>
<td ng-bind="record.date"></td>
<td ng-bind="record.size"></td>
<td ng-bind="record.status"></td>
<a href=""><td ng-click="deleteBackup(record.id)"><img src="{% static 'images/delete.png' %}"></td></a>
</tr>
</tbody>
</table>
</div>
</div>
<!------ List of records --------------->
<div class="form-group">
<label class="col-sm-3 control-label"></label>
<div class="col-sm-4">
<div id="websiteDeleteFailure" class="alert alert-danger">
<p>{% trans "Cannot delete website, Error message: " %}{$ errorMessage $}</p>
<div ng-hide="runningBackup" class="form-group">
<div class="col-sm-12">
<div class="col-sm-12">
<textarea ng-model="status" class="form-control" rows="10"></textarea>
</div>
</div>
</div>
<div id="websiteDeleteSuccess" class="alert alert-success">
<p>Website <strong>{$ deletedWebsite $}</strong> {% trans "Successfully Deleted" %}</p>
<!---- if Back up is running------>
<div ng-hide="backupButton" class="form-group">
<label class="col-sm-3 control-label"></label>
<div class="col-sm-4">
<button type="button" ng-click="createBackup()"
class="btn btn-primary btn-lg btn-block">{% trans "Create Back up" %}</button>
</div>
</div>
</div>
<!------ List of records --------------->
<div class="form-group">
<div class="col-sm-12">
<table class="table">
<thead>
<tr>
<th>{% trans "ID" %}</th>
<th>{% trans "Date" %}</th>
<th>{% trans "Includes" %}</th>
<th>{% trans "Delete" %}</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="record in records track by $index">
<td ng-bind="record.id"></td>
<td ng-bind="record.date"></td>
<td ng-bind="record.includes"></td>
<a href="">
<td ng-click="deleteBackup(record.id)"><img
src="{% static 'images/delete.png' %}"></td>
</a>
</tr>
</tbody>
</table>
</div>
</div>
<!------ List of records --------------->
</form>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
{% endblock %}

View File

@@ -7,5 +7,8 @@ urlpatterns = [
url(r'^addDestination$', views.addDestination, name='addDestinationInc'),
url(r'^populateCurrentRecords$', views.populateCurrentRecords, name='populateCurrentRecordsInc'),
url(r'^removeDestination$', views.removeDestination, name='removeDestinationInc'),
url(r'^fetchCurrentBackups$', views.fetchCurrentBackups, name='fetchCurrentBackupsInc'),
url(r'^submitBackupCreation$', views.submitBackupCreation, name='submitBackupCreationInc'),
url(r'^getBackupStatus$', views.getBackupStatus, name='getBackupStatusInc'),
url(r'^deleteBackup$', views.deleteBackup, name='deleteBackupInc'),
]

View File

@@ -8,6 +8,13 @@ from plogical.processUtilities import ProcessUtilities
from plogical.virtualHostUtilities import virtualHostUtilities
import json
import os
from loginSystem.models import Administrator
from websiteFunctions.models import Websites
from .models import IncJob, JobSnapshots
from .IncBackups import IncJobs
from random import randint
import time
from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
# Create your views here.
@@ -23,7 +30,19 @@ def createBackup(request):
return ACLManager.loadError()
websitesName = ACLManager.findAllSites(currentACL, userID)
return defRenderer(request, 'IncBackups/createBackup.html', {'websiteList': websitesName})
destinations = []
destinations.append('local')
path = '/home/cyberpanel/sftp'
for items in os.listdir(path):
destinations.append('sftp:%s' % (items))
for items in os.listdir(path):
destinations.append('s3:s3.amazonaws.com/%s' % (items))
return defRenderer(request, 'IncBackups/createBackup.html', {'websiteList': websitesName, 'destinations': destinations})
except BaseException, msg:
return HttpResponse(str(msg))
@@ -239,6 +258,194 @@ def removeDestination(request):
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
except BaseException, msg:
final_dic = {'destStatus': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
def fetchCurrentBackups(request):
try:
userID = request.session['userID']
currentACL = ACLManager.loadedACL(userID)
admin = Administrator.objects.get(pk=userID)
data = json.loads(request.body)
backupDomain = data['websiteToBeBacked']
if ACLManager.checkOwnership(backupDomain, admin, currentACL) == 1:
pass
else:
return ACLManager.loadErrorJson('fetchStatus', 0)
if ACLManager.checkOwnership(backupDomain, admin, currentACL) == 1:
pass
else:
return ACLManager.loadErrorJson()
website = Websites.objects.get(domain=backupDomain)
backups = website.incjob_set.all()
json_data = "["
checker = 0
for items in backups:
includes = ""
jobs = items.jobsnapshots_set.all()
for job in jobs:
includes = '%s,%s:%s' % (includes, job.type, job.snapshotid)
dic = {'id': items.id,
'date': str(items.date),
'includes': includes
}
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
json_data = json_data + ']'
final_json = json.dumps({'status': 1, 'error_message': "None", "data": json_data})
return HttpResponse(final_json)
except BaseException, msg:
final_dic = {'status': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
def submitBackupCreation(request):
try:
userID = request.session['userID']
currentACL = ACLManager.loadedACL(userID)
admin = Administrator.objects.get(pk=userID)
data = json.loads(request.body)
backupDomain = data['websiteToBeBacked']
backupDestinations = data['backupDestinations']
if ACLManager.checkOwnership(backupDomain, admin, currentACL) == 1:
pass
else:
return ACLManager.loadErrorJson('metaStatus', 0)
tempPath = "/home/cyberpanel/" + str(randint(1000, 9999))
try:
websiteData = data['websiteData']
except:
websiteData = False
try:
websiteEmails = data['websiteEmails']
except:
websiteEmails = False
try:
websiteSSLs = data['websiteSSLs']
except:
websiteSSLs = False
extraArgs = {}
extraArgs['website'] = backupDomain
extraArgs['tempPath'] = tempPath
extraArgs['backupDestinations'] = backupDestinations
extraArgs['websiteData'] = websiteData
extraArgs['websiteEmails'] = websiteEmails
extraArgs['websiteSSLs'] = websiteSSLs
startJob = IncJobs('createBackup', extraArgs)
startJob.start()
time.sleep(2)
final_json = json.dumps({'status': 1, 'error_message': "None", 'tempPath': tempPath})
return HttpResponse(final_json)
except BaseException, msg:
logging.writeToFile(str(msg))
final_dic = {'status': 0, 'metaStatus': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
def getBackupStatus(request):
try:
userID = request.session['userID']
data = json.loads(request.body)
status = data['tempPath']
backupDomain = data['websiteToBeBacked']
domain = Websites.objects.get(domain=backupDomain)
## file name read ends
if os.path.exists(status):
command = "sudo cat " + status
status = ProcessUtilities.outputExecutioner(command, 'cyberpanel')
if status.find("Completed") > -1:
### Removing Files
command = 'sudo rm -f ' + status
ProcessUtilities.executioner(command, 'cyberpanel')
final_json = json.dumps(
{'backupStatus': 1, 'error_message': "None", "status": status, "abort": 1})
return HttpResponse(final_json)
elif status.find("[5009]") > -1:
## removing status file, so that backup can re-run
try:
command = 'sudo rm -f ' + status
ProcessUtilities.executioner(command, 'cyberpanel')
except:
pass
final_json = json.dumps(
{'backupStatus': 1, 'error_message': "None", "status": status,
"abort": 1})
return HttpResponse(final_json)
else:
final_json = json.dumps(
{'backupStatus': 1, 'error_message': "None", "status": status,
"abort": 0})
return HttpResponse(final_json)
else:
final_json = json.dumps({'backupStatus': 1, 'error_message': "None", "status": 1, "abort": 0})
return HttpResponse(final_json)
except BaseException, msg:
final_dic = {'backupStatus': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)
logging.writeToFile(str(msg) + " [backupStatus]")
return HttpResponse(final_json)
def deleteBackup(request):
try:
userID = request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if ACLManager.currentContextPermission(currentACL, 'addDeleteDestinations') == 0:
return ACLManager.loadErrorJson('destStatus', 0)
data = json.loads(request.body)
id = data['backupID']
IncJob.objects.get(id=id).delete()
final_dic = {'status': 1, 'error_message': 'None'}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
except BaseException, msg:
final_dic = {'destStatus': 0, 'error_message': str(msg)}
final_json = json.dumps(final_dic)

View File

@@ -117,7 +117,7 @@
</extension>
<extension>
<extensionName>lsphp70-pecl-imagick</extensionName>
<extensionName>lsphp70-imagick</extensionName>
<extensionDescription>Extension to create and modify images using ImageMagick</extensionDescription>
<status>0</status>
</extension>

View File

@@ -117,7 +117,7 @@
</extension>
<extension>
<extensionName>lsphp71-pecl-imagick</extensionName>
<extensionName>lsphp71-imagick</extensionName>
<extensionDescription>Extension to create and modify images using ImageMagick</extensionDescription>
<status>0</status>
</extension>

View File

@@ -117,7 +117,7 @@
</extension>
<extension>
<extensionName>lsphp72-pecl-imagick</extensionName>
<extensionName>lsphp72-imagick</extensionName>
<extensionDescription>Extension to create and modify images using ImageMagick</extensionDescription>
<status>0</status>
</extension>

View File

@@ -117,7 +117,7 @@
</extension>
<extension>
<extensionName>lsphp73-pecl-imagick</extensionName>
<extensionName>lsphp73-imagick</extensionName>
<extensionDescription>Extension to create and modify images using ImageMagick</extensionDescription>
<status>0</status>
</extension>