backups to aws s3

This commit is contained in:
usmannasir
2018-12-13 04:23:08 +05:00
parent 7a42f9444a
commit 1094b7d686
30 changed files with 1316 additions and 109 deletions

View File

@@ -63,7 +63,8 @@ INSTALLED_APPS = [
'emailPremium',
'emailMarketing',
'cloudAPI',
'highAvailability'
'highAvailability',
's3Backups'
]
MIDDLEWARE = [

View File

@@ -999,7 +999,7 @@ app.controller('remoteBackupControl', function($scope, $http, $timeout) {
$scope.addRemoveWebsite = function (website,websiteStatus) {
if(websiteStatus==true)
if(websiteStatus === true)
{
var check = 1;
for(var j = 0; j < websitesToBeBacked.length; j++){
@@ -1028,9 +1028,7 @@ app.controller('remoteBackupControl', function($scope, $http, $timeout) {
$scope.allChecked = function (webSiteStatus) {
if(webSiteStatus==true) {
if(webSiteStatus === true) {
websitesToBeBacked = websitesToBeBackedTemp;
$scope.webSiteStatus = true;
@@ -1142,7 +1140,7 @@ app.controller('remoteBackupControl', function($scope, $http, $timeout) {
if(websitesToBeBacked.length === 0){
alert("No websites selected for transfer.")
alert("No websites selected for transfer.");
return;
}

View File

@@ -289,6 +289,10 @@
<i class="glyph-icon tooltip-button icon-cloud-upload" title="{% trans 'Version Management' %}" data-original-title=".icon-cloud-upload" aria-describedby="tooltip896208"></i>
<span>{% trans "Version Management" %}</span>
</a>
<a class="versionManagement" href="https://platform.cyberpanel.net" title="{% trans 'Connect' %}">
<i class="glyph-icon tooltip-button icon-cloud" title="{% trans 'Connect' %}" data-original-title=".icon-cloud-upload" aria-describedby="tooltip896208"></i>
<span>{% trans "Connect" %}</span>
</a>
</li>
<li class="divider"></li>
<li class="header"><span>{% trans "Main" %}</span></li>
@@ -479,12 +483,10 @@
<div class="sidebar-submenu">
<ul>
<li><a href="{% url 'topProcesses' %}" title="{% trans 'Top Processes' %}"><span>{% trans "Top Processes" %}</span></a></li>
<li><a href="{% url 'litespeedStatus' %}" title="{% trans 'LiteSpeed Status' %}"><span>{% trans "LiteSpeed Status" %}</span></a></li>
<li><a href="{% url 'cyberCPMainLogFile' %}" title="{% trans 'CyberPanel Main Log File' %}"><span>{% trans "CyberPanel Main Log File" %}</span></a></li>
<li><a href="{% url 'services' %}" title="{% trans 'Services Status' %}"><span>{% trans "Services Status" %}</span></a></li>
</ul>
</div><!-- .sidebar-submenu -->

View File

@@ -20,7 +20,7 @@ from serverLogs.views import getLogsFromFile
from random import randint
from highAvailability.haManager import HAManager
from plogical.httpProc import httpProc
from api.views import fetchSSHkey
from s3Backups.s3Backups import S3Backups
import os
class CloudManager:
@@ -830,3 +830,84 @@ class CloudManager:
return ham.submitEditCluster()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def connectAccount(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'connectAccount')
return s3.connectAccount()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def fetchBuckets(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'fetchBuckets')
return s3.fetchBuckets()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def createPlan(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'createPlan')
return s3.createPlan()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def fetchBackupPlans(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'fetchBackupPlans')
return s3.fetchBackupPlans()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def deletePlan(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'deletePlan')
return s3.deletePlan()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def fetchWebsitesInPlan(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'fetchWebsitesInPlan')
return s3.fetchWebsitesInPlan()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def deleteDomainFromPlan(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'deleteDomainFromPlan')
return s3.deleteDomainFromPlan()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def savePlanChanges(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'savePlanChanges')
return s3.savePlanChanges()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def fetchBackupLogs(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'fetchBackupLogs')
return s3.fetchBackupLogs()
except BaseException, msg:
return self.ajaxPre(0, str(msg))
def forceRunAWSBackup(self, request):
try:
request.session['userID'] = self.admin.pk
s3 = S3Backups(request, self.data, 'forceRunAWSBackup')
s3.start()
return self.ajaxPre(1, None)
except BaseException, msg:
return self.ajaxPre(0, str(msg))

View File

@@ -177,6 +177,26 @@ def router(request):
return cm.setUpDataNode(request)
elif controller == 'submitEditCluster':
return cm.submitEditCluster(request)
elif controller == 'connectAccount':
return cm.connectAccount(request)
elif controller == 'fetchBuckets':
return cm.fetchBuckets(request)
elif controller == 'createPlan':
return cm.createPlan(request)
elif controller == 'fetchBackupPlans':
return cm.fetchBackupPlans(request)
elif controller == 'deletePlan':
return cm.deletePlan(request)
elif controller == 'fetchWebsitesInPlan':
return cm.fetchWebsitesInPlan(request)
elif controller == 'deleteDomainFromPlan':
return cm.deleteDomainFromPlan(request)
elif controller == 'savePlanChanges':
return cm.savePlanChanges(request)
elif controller == 'fetchBackupLogs':
return cm.fetchBackupLogs(request)
elif controller == 'forceRunAWSBackup':
return cm.forceRunAWSBackup(request)
else:
return cm.ajaxPre(0, 'This function is not available in your version of CyberPanel.')

View File

@@ -116,24 +116,10 @@ class preFlightsChecks:
try:
if self.distro == centos:
count = 0
while (1):
command = "yum install sudo -y"
cmd = shlex.split(command)
res = subprocess.call(cmd)
if preFlightsChecks.resFailed(self.distro, res):
count = count + 1
preFlightsChecks.stdOut("SUDO install failed, trying again, try number: " + str(count))
if count == 3:
logging.InstallLog.writeToFile(
"We are not able to install SUDO, exiting the installer. [setup_account_cyberpanel]")
preFlightsChecks.stdOut("Installation failed, consult: /var/log/installLogs.txt")
os._exit(0)
else:
logging.InstallLog.writeToFile("SUDO successfully installed!")
preFlightsChecks.stdOut("SUDO successfully installed!")
break
preFlightsChecks.call(command, self.distro, '[setup_account_cyberpanel]',
'Install sudo.',
1, 0, os.EX_OSERR)
##
@@ -168,49 +154,19 @@ class preFlightsChecks:
preFlightsChecks.stdOut("CyberPanel user added")
else:
while (1):
command = "adduser cyberpanel"
cmd = shlex.split(command)
res = subprocess.call(cmd)
if preFlightsChecks.resFailed(self.distro, res):
count = count + 1
preFlightsChecks.stdOut(
"Not able to add user cyberpanel to system, trying again, try number: " + str(count) + "\n")
if count == 3:
logging.InstallLog.writeToFile(
"We are not able add user cyberpanel to system, exiting the installer. [setup_account_cyberpanel]")
preFlightsChecks.stdOut("Installation failed, consult: /var/log/installLogs.txt")
os._exit(0)
else:
logging.InstallLog.writeToFile("CyberPanel user added!")
preFlightsChecks.stdOut("CyberPanel user added!")
break
preFlightsChecks.call(command, self.distro, '[setup_account_cyberpanel]',
'add user cyberpanel',
1, 0, os.EX_OSERR)
##
count = 0
while (1):
command = "usermod -aG wheel cyberpanel"
cmd = shlex.split(command)
res = subprocess.call(cmd)
preFlightsChecks.call(command, self.distro, '[setup_account_cyberpanel]',
'add user cyberpanel',
1, 0, os.EX_OSERR)
if preFlightsChecks.resFailed(self.distro, res):
count = count + 1
preFlightsChecks.stdOut(
"We are trying to add CyberPanel user to SUDO group, trying again, try number: " + str(
count) + "\n")
if count == 3:
logging.InstallLog.writeToFile(
"Not able to add user CyberPanel to SUDO group, exiting the installer. [setup_account_cyberpanel]")
preFlightsChecks.stdOut("Installation failed, consult: /var/log/installLogs.txt")
os._exit(0)
else:
logging.InstallLog.writeToFile("CyberPanel user was successfully added to SUDO group!")
preFlightsChecks.stdOut("CyberPanel user was successfully added to SUDO group!")
break
###############################
@@ -230,35 +186,13 @@ class preFlightsChecks:
###############################
count = 0
while (1):
command = "mkdir -p /etc/letsencrypt/live/"
preFlightsChecks.call(command, self.distro, '[setup_account_cyberpanel]',
'add user cyberpanel',
1, 0, os.EX_OSERR)
cmd = shlex.split(command)
res = subprocess.call(cmd)
if preFlightsChecks.resFailed(self.distro, res):
count = count + 1
preFlightsChecks.stdOut(
"We are trying to create Let's Encrypt directory to store SSLs, trying again, try number: " + str(
count))
if count == 3:
logging.InstallLog.writeToFile(
"Failed to create Let's Encrypt directory to store SSLs. Installer can continue without this.. [setup_account_cyberpanel]")
else:
logging.InstallLog.writeToFile("Successfully created Let's Encrypt directory!")
preFlightsChecks.stdOut("Successfully created Let's Encrypt directory!")
break
##
except:
logging.InstallLog.writeToFile("[116] setup_account_cyberpanel")
preFlightsChecks.stdOut("Installation failed, consult: /var/log/installLogs.txt")
os._exit(0)
except BaseException, msg:
logging.InstallLog.writeToFile("[ERROR] setup_account_cyberpanel. " + str(msg))
def yum_update(self):
try:

View File

@@ -213,6 +213,7 @@ class InstallCyberPanel:
logging.InstallLog.writeToFile(str(msg) + " [setupFileManager]")
def installAllPHPVersions(self):
if self.distro == ubuntu:
command = 'DEBIAN_FRONTEND=noninteractive apt-get -y install ' \
'lsphp7? lsphp7?-common lsphp7?-curl lsphp7?-dev lsphp7?-imap lsphp7?-intl lsphp7?-json ' \
@@ -364,7 +365,11 @@ class InstallCyberPanel:
self.startMariaDB()
def changeMYSQLRootPassword(self):
if self.distro == ubuntu:
passwordCMD = "use mysql;update user set password=PASSWORD('" + InstallCyberPanel.mysql_Root_password + "') where User='root';UPDATE user SET plugin='' WHERE User='root';flush privileges;"
else:
passwordCMD = "use mysql;update user set password=PASSWORD('" + InstallCyberPanel.mysql_Root_password + "') where User='root';flush privileges;"
command = 'mysql -u root -e "' + passwordCMD + '"'
install.preFlightsChecks.call(command, self.distro, '[changeMYSQLRootPassword]', 'MYSQL Root Password change.',
1, 1, os.EX_OSERR)

View File

@@ -8,7 +8,7 @@ import CyberCPLogFileWriter as logging
import subprocess
import shlex
from dns.models import Domains,Records
from processUtilities import ProcessUtilities
class DNS:
@@ -153,7 +153,6 @@ class DNS:
disabled=0,
auth=1)
record.save()
else:
if Domains.objects.filter(name=topLevelDomain).count() == 0:
zone = Domains(admin=admin, name=topLevelDomain, type="NATIVE")
@@ -290,6 +289,10 @@ class DNS:
DNS.createDNSRecord(zone, cNameValue, "CNAME", actualSubDomain, 0, 3600)
if ProcessUtilities.decideDistro() == ProcessUtilities.ubuntu:
command = 'sudo systemctl restart pdns'
ProcessUtilities.executioner(command)
except BaseException, msg:
logging.CyberCPLogFileWriter.writeToFile(
"We had errors while creating DNS records for: " + domain + ". Error message: " + str(msg))
@@ -322,6 +325,10 @@ class DNS:
auth=1)
record.save()
if ProcessUtilities.decideDistro() == ProcessUtilities.ubuntu:
command = 'sudo systemctl restart pdns'
ProcessUtilities.executioner(command)
except BaseException, msg:
logging.CyberCPLogFileWriter.writeToFile(
"We had errors while creating DKIM record for: " + domain + ". Error message: " + str(msg))
@@ -348,6 +355,11 @@ class DNS:
disabled=0,
auth=1)
record.save()
if ProcessUtilities.decideDistro() == ProcessUtilities.ubuntu:
command = 'sudo systemctl restart pdns'
ProcessUtilities.executioner(command)
return
if type == 'TXT':
@@ -362,6 +374,10 @@ class DNS:
disabled=0,
auth=1)
record.save()
if ProcessUtilities.decideDistro() == ProcessUtilities.ubuntu:
command = 'sudo systemctl restart pdns'
ProcessUtilities.executioner(command)
return
if type == 'MX':
@@ -375,6 +391,10 @@ class DNS:
disabled=0,
auth=1)
record.save()
if ProcessUtilities.decideDistro() == ProcessUtilities.ubuntu:
command = 'sudo systemctl restart pdns'
ProcessUtilities.executioner(command)
return
@@ -389,6 +409,9 @@ class DNS:
disabled=0,
auth=1)
record.save()
if ProcessUtilities.decideDistro() == ProcessUtilities.ubuntu:
command = 'sudo systemctl restart pdns'
ProcessUtilities.executioner(command)
except BaseException, msg:
logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [createDNSRecord]")

View File

@@ -18,7 +18,6 @@ class httpProc:
final_dic = {'status': status, 'error_message': errorMessage, 'success': success}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
def ajax(self, status, errorMessage, data = None):
if data == None:
finalDic = {'status': status, 'error_message': errorMessage}

View File

@@ -22,15 +22,19 @@ class mysqlUtilities:
data = f.read()
password = data.split('\n', 1)[0]
logging.CyberCPLogFileWriter.writeToFile(password)
conn = mysql.connect(user='root', passwd=password)
cursor = conn.cursor()
logging.CyberCPLogFileWriter.writeToFile('hekkk')
return conn, cursor
except BaseException, msg:
logging.CyberCPLogFileWriter.writeToFile(str(msg))
return 0, 0
@staticmethod
def createDatabase(dbname,dbuser,dbpassword):
try:

View File

@@ -0,0 +1,8 @@
import boto3
# Let's use Amazon S3
s3 = boto3.resource('s3')
# Print out bucket names
for bucket in s3.buckets.all():
print(bucket.name)

View File

@@ -338,7 +338,7 @@ class vhost:
accessControlEnds = " }\n"
rewriteInherit = """ rewrite {
inherit 1
enable 0
}
"""

0
s3Backups/__init__.py Normal file
View File

6
s3Backups/admin.py Normal file
View File

@@ -0,0 +1,6 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.contrib import admin
# Register your models here.

8
s3Backups/apps.py Normal file
View File

@@ -0,0 +1,8 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.apps import AppConfig
class S3BackupsConfig(AppConfig):
name = 's3Backups'

View File

24
s3Backups/models.py Normal file
View File

@@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
from loginSystem.models import Administrator
# Create your models here.
class BackupPlan(models.Model):
owner = models.ForeignKey(Administrator, on_delete=models.CASCADE)
name = models.CharField(max_length=50, unique=True)
bucket = models.CharField(max_length=50, default='NONE')
freq = models.CharField(max_length=50)
retention = models.IntegerField()
lastRun = models.CharField(max_length=50, default='NEVER')
class WebsitesInPlan(models.Model):
owner = models.ForeignKey(BackupPlan,on_delete=models.CASCADE)
domain = models.CharField(max_length=100)
class BackupLogs(models.Model):
owner = models.ForeignKey(BackupPlan,on_delete=models.CASCADE)
level = models.CharField(max_length=5)
msg = models.CharField(max_length=500)

407
s3Backups/s3Backups.py Normal file
View File

@@ -0,0 +1,407 @@
#!/usr/local/CyberCP/bin/python2
import os.path
import sys
import django
sys.path.append('/usr/local/CyberCP')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")
django.setup()
from django.shortcuts import HttpResponse
from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
from plogical.httpProc import httpProc
from plogical.acl import ACLManager
import threading as multi
import argparse
from plogical.mailUtilities import mailUtilities
import boto3
import json
from .models import *
from math import ceil
import requests
import time
class S3Backups(multi.Thread):
def __init__(self, request = None, data = None, function = None):
multi.Thread.__init__(self)
self.request = request
self.data = data
self.function = function
def run(self):
try:
if self.function == 'connectAccount':
self.connectAccount()
elif self.function == 'forceRunAWSBackup':
self.forceRunAWSBackup()
except BaseException, msg:
logging.writeToFile( str(msg) + ' [S3Backups.run]')
@staticmethod
def getPagination(records, toShow):
pages = float(records) / float(toShow)
pagination = []
counter = 1
if pages <= 1.0:
pages = 1
pagination.append(counter)
else:
pages = ceil(pages)
finalPages = int(pages) + 1
for i in range(1, finalPages):
pagination.append(counter)
counter = counter + 1
return pagination
@staticmethod
def recordsPointer(page, toShow):
finalPageNumber = ((page * toShow)) - toShow
endPageNumber = finalPageNumber + toShow
return endPageNumber, finalPageNumber
@staticmethod
def getLogsInJson(logs):
json_data = "["
checker = 0
counter = 1
for items in reversed(logs):
dic = { 'id': items.id, 'level': items.level, 'mesg': items.msg }
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
counter = counter + 1
json_data = json_data + ']'
return json_data
def connectAccount(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
mailUtilities.checkHome()
path = '/home/cyberpanel/.aws'
if not os.path.exists(path):
os.mkdir(path)
credentials = path + '/credentials'
credFile = open(credentials, 'w')
credFile.write(self.data['credData'])
credFile.close()
return proc.ajax(1, None)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def fetchBuckets(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
s3 = boto3.resource('s3')
json_data = "["
checker = 0
for bucket in s3.buckets.all():
dic = {'name': bucket.name}
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
json_data = json_data + ']'
final_json = json.dumps({'status': 1, 'error_message': "None", "data": json_data})
return HttpResponse(final_json)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def createPlan(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
admin = Administrator.objects.get(pk=userID)
newPlan = BackupPlan(owner=admin, name=self.data['planName'], freq = self.data['frequency'],
retention= self.data['retenion'], bucket= self.data['bucketName'])
newPlan.save()
for items in self.data['websitesInPlan']:
wp = WebsitesInPlan(owner=newPlan, domain=items)
wp.save()
return proc.ajax(1, None)
except BaseException, msg:
logging.writeToFile(str(msg) + ' [createPlan]')
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def fetchBackupPlans(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
admin = Administrator.objects.get(pk=userID)
json_data = "["
checker = 0
for plan in admin.backupplan_set.all():
dic = {
'name': plan.name,
'bucket': plan.bucket,
'freq': plan.freq,
'retention': plan.retention,
'lastRun': plan.lastRun,
}
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
json_data = json_data + ']'
final_json = json.dumps({'status': 1, 'error_message': "None", "data": json_data})
return HttpResponse(final_json)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def deletePlan(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
delPlan = BackupPlan.objects.get(name=self.data['planName'])
delPlan.delete()
return proc.ajax(1, None)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def fetchWebsitesInPlan(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
plan = BackupPlan.objects.get(name=self.data['planName'])
json_data = "["
checker = 0
for website in plan.websitesinplan_set.all():
dic = {
'id': website.id,
'domain': website.domain,
}
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
json_data = json_data + ']'
final_json = json.dumps({'status': 1, 'error_message': "None", "data": json_data})
return HttpResponse(final_json)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def deleteDomainFromPlan(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
plan = BackupPlan.objects.get(name=self.data['planName'])
web = WebsitesInPlan.objects.get(owner=plan, domain=self.data['domainName'])
web.delete()
return proc.ajax(1, None)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def savePlanChanges(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
logging.writeToFile('hello world')
changePlan = BackupPlan.objects.get(name=self.data['planName'])
changePlan.bucket = self.data['bucketName']
changePlan.freq = self.data['frequency']
changePlan.retention = self.data['retention']
changePlan.save()
return proc.ajax(1, None)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajax(0, str(msg))
def fetchBackupLogs(self):
try:
proc = httpProc(self.request, None, None)
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
return proc.ajax(0, 'Only administrators can use AWS S3 Backups.')
recordsToShow = int(self.data['recordsToShow'])
page = int(self.data['page'])
backupPlan = BackupPlan.objects.get(name=self.data['planName'])
logs = backupPlan.backuplogs_set.all()
pagination = S3Backups.getPagination(len(logs), recordsToShow)
endPageNumber, finalPageNumber = S3Backups.recordsPointer(page, recordsToShow)
jsonData = S3Backups.getLogsInJson(logs[finalPageNumber:endPageNumber])
data = {}
data['data'] = jsonData
data['pagination'] = pagination
return proc.ajax(1, None, data)
except BaseException, msg:
proc = httpProc(self.request, None, None)
return proc.ajaxPre(0, str(msg))
def createBackup(self, virtualHost):
finalData = json.dumps({'websiteToBeBacked': virtualHost})
r = requests.post("http://localhost:5003/backup/submitBackupCreation", data=finalData)
data = json.loads(r.text)
backupPath = data['tempStorage']
while (1):
r = requests.post("http://localhost:5003/backup/backupStatus", data=finalData)
time.sleep(2)
data = json.loads(r.text)
if data['backupStatus'] == 0:
return 0, data['error_message']
elif data['abort'] == 1:
return 1, backupPath
def forceRunAWSBackup(self):
try:
s3 = boto3.resource('s3')
plan = BackupPlan.objects.get(name=self.data['planName'])
bucketName = plan.bucket.strip('\n').strip(' ')
userID = self.request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 0:
BackupLogs(owner=plan, level='INFO', msg='Unauthorised user tried to run AWS Backups.').save()
BackupLogs(owner=plan,level='INFO', msg='Starting backup process..').save()
for items in plan.websitesinplan_set.all():
result = self.createBackup(items.domain)
if result[0]:
data = open(result[1] + ".tar.gz", 'rb')
s3.Bucket(bucketName).put_object(Key=result[1].split('/')[-1] + ".tar.gz", Body=data)
BackupLogs(owner=plan, level='INFO', msg='Backup successful for ' + items.domain + '.').save()
else:
BackupLogs(owner=plan, level='ERROR', msg='Backup failed for ' + items.domain + '. Error: ' + result[1]).save()
BackupLogs(owner=plan, level='INFO', msg='Backup Process Finished.').save()
except BaseException, msg:
logging.writeToFile(str(msg) + ' [S3Backups.runBackupPlan]')
plan = BackupPlan.objects.get(name=self.data['planName'])
BackupLogs(owner=plan, level='ERROR', msg=str(msg)).save()
def main():
parser = argparse.ArgumentParser(description='CyberPanel S3 Backups')
parser.add_argument('function', help='Specify a function to call!')
args = parser.parse_args()
if __name__ == "__main__":
main()

6
s3Backups/tests.py Normal file
View File

@@ -0,0 +1,6 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.test import TestCase
# Create your tests here.

6
s3Backups/views.py Normal file
View File

@@ -0,0 +1,6 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.shortcuts import render
# Create your views here.

View File

@@ -637,3 +637,142 @@ app.controller('lswsSwitch', function ($scope, $http, $timeout, $window) {
}
});
app.controller('topProcesses', function ($scope, $http, $timeout) {
$scope.cyberPanelLoading = true;
$scope.topProcessesStatus = function () {
$scope.cyberPanelLoading = false;
url = "/serverstatus/topProcessesStatus";
var data = {};
var config = {
headers: {
'X-CSRFToken': getCookie('csrftoken')
}
};
$http.post(url, data, config).then(ListInitialDatas, cantLoadInitialDatas);
function ListInitialDatas(response) {
$scope.cyberPanelLoading = true;
if (response.data.status === 1) {
$scope.processes = JSON.parse(response.data.data);
//CPU Details
$scope.cores = response.data.cores;
$scope.modelName = response.data.modelName;
$scope.cpuMHZ = response.data.cpuMHZ;
$scope.cacheSize = response.data.cacheSize;
//CPU Load
$scope.cpuNow = response.data.cpuNow;
$scope.cpuOne = response.data.cpuOne;
$scope.cpuFive = response.data.cpuFive;
$scope.cpuFifteen = response.data.cpuFifteen;
//CPU Time spent
$scope.ioWait = response.data.ioWait;
$scope.idleTime = response.data.idleTime;
$scope.hwInterrupts = response.data.hwInterrupts;
$scope.Softirqs = response.data.Softirqs;
//Memory
$scope.totalMemory = response.data.totalMemory;
$scope.freeMemory = response.data.freeMemory;
$scope.usedMemory = response.data.usedMemory;
$scope.buffCache = response.data.buffCache;
//Swap
$scope.swapTotalMemory = response.data.swapTotalMemory;
$scope.swapFreeMemory = response.data.swapFreeMemory;
$scope.swapUsedMemory = response.data.swapUsedMemory;
$scope.swapBuffCache = response.data.swapBuffCache;
//Processes
$scope.totalProcesses = response.data.totalProcesses;
$scope.runningProcesses = response.data.runningProcesses;
$scope.sleepingProcesses = response.data.sleepingProcesses;
$scope.stoppedProcesses = response.data.stoppedProcesses;
$scope.zombieProcesses = response.data.zombieProcesses;
$timeout($scope.topProcessesStatus, 3000);
}
else {
new PNotify({
title: 'Operation Failed!',
text: response.data.error_message,
type: 'error'
});
}
}
function cantLoadInitialDatas(response) {
$scope.cyberPanelLoading = true;
new PNotify({
title: 'Operation Failed!',
text: 'Could not connect to server, please refresh this page',
type: 'error'
});
}
};
$scope.topProcessesStatus();
$scope.killProcess = function (pid) {
$scope.cyberPanelLoading = false;
url = "/serverstatus/killProcess";
var data = {
pid: pid
};
var config = {
headers: {
'X-CSRFToken': getCookie('csrftoken')
}
};
$http.post(url, data, config).then(ListInitialDatas, cantLoadInitialDatas);
function ListInitialDatas(response) {
$scope.cyberPanelLoading = true;
if (response.data.status === 1) {
new PNotify({
title: 'Success',
text: 'Process successfully killed.',
type: 'success'
});
}
else {
new PNotify({
title: 'Operation Failed!',
text: response.data.error_message,
type: 'error'
});
}
}
function cantLoadInitialDatas(response) {
$scope.cyberPanelLoading = true;
new PNotify({
title: 'Operation Failed!',
text: 'Could not connect to server, please refresh this page',
type: 'error'
});
}
};
});

View File

@@ -0,0 +1,229 @@
{% extends "baseTemplate/index.html" %}
{% load i18n %}
{% block title %}{% trans "Top Processes - CyberPanel" %}{% endblock %}
{% block content %}
{% load static %}
{% get_current_language as LANGUAGE_CODE %}
<!-- Current language: {{ LANGUAGE_CODE }} -->
<div ng-controller="topProcesses" class="container">
<div id="page-title">
<h2>{% trans "Top Processes" %} <img height="20px" ng-hide="cyberPanelLoading"
src="/static/images/loading.gif"></h2>
<p>{% trans "List of top processes on your server. (Refresh every 3 seconds)" %}</p>
</div>
<div class="row">
<div class="col-md-6">
<div class="example-box-wrapper">
<div class="content-box-wrapper">
<table class="table">
<thead>
<tr >
<th style="background: #3498db;color:white">{% trans 'Cores' %}</th>
<th style="background: #3498db;color:white">{% trans 'Model Name' %}</th>
<th style="background: #3498db;color:white">{% trans 'CPU Mhz' %}</th>
<th style="background: #3498db;color:white">{% trans 'Cache Size' %}</th>
</tr>
</thead>
<tbody>
<tr>
<td>{$ cores $}</td>
<td>{$ modelName $}</td>
<td>{$ cpuMHZ $}</td>
<td>{$ cacheSize $}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="col-md-6">
<div class="example-box-wrapper">
<div class="content-box-wrapper">
<table class="table">
<thead>
<tr>
<th style="background: #3498db;color:white">{% trans 'Processes' %}</th>
<th style="background: #3498db;color:white">{% trans 'Running' %}</th>
<th style="background: #3498db;color:white">{% trans 'Sleeping' %}</th>
<th style="background: #3498db;color:white">{% trans 'Stopped' %}</th>
<th style="background: #3498db;color:white">{% trans 'Zombie' %}</th>
</tr>
</thead>
<tbody>
<tr>
<td>{$ totalProcesses $}</td>
<td>{$ runningProcesses $}</td>
<td>{$ sleepingProcesses $}</td>
<td>{$ stoppedProcesses $}</td>
<td>{$ zombieProcesses $}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6">
<div class="example-box-wrapper">
<div class="content-box-wrapper">
<table class="table">
<thead>
<tr>
<th style="background: #3498db;color:white">{% trans 'CPU Load' %}</th>
<th style="background: #3498db;color:white">{% trans '1 Min' %}</th>
<th style="background: #3498db;color:white">{% trans '5 Min' %}</th>
<th style="background: #3498db;color:white">{% trans '15 Min' %}</th>
</tr>
</thead>
<tbody>
<tr>
<td>{$ cpuNow $}</td>
<td>{$ cpuOne $}</td>
<td>{$ cpuFive $}</td>
<td>{$ cpuFifteen $}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="col-md-6">
<div class="example-box-wrapper">
<div class="content-box-wrapper">
<table class="table">
<thead>
<tr>
<th style="background: #3498db;color:white">{% trans 'I/O Wait' %}</th>
<th style="background: #3498db;color:white">{% trans 'Idle Time' %}</th>
<th style="background: #3498db;color:white">{% trans 'HW Interrupts' %}</th>
<th style="background: #3498db;color:white">{% trans 'Softirqs' %}</th>
</tr>
</thead>
<tbody>
<tr>
<td>{$ ioWait $}</td>
<td>{$ idleTime $}</td>
<td>{$ hwInterrupts $}</td>
<td>{$ Softirqs $}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-md-6">
<div class="example-box-wrapper">
<div class="content-box-wrapper">
<table class="table">
<thead>
<tr>
<th style="background: #3498db;color:white">{% trans 'Memory' %}</th>
<th style="background: #3498db;color:white">{% trans 'Free' %}</th>
<th style="background: #3498db;color:white">{% trans 'Used' %}</th>
<th style="background: #3498db;color:white">{% trans 'buff/cache' %}</th>
</tr>
</thead>
<tbody>
<tr>
<td>{$ totalMemory $}</td>
<td>{$ freeMemory $}</td>
<td>{$ usedMemory $}</td>
<td>{$ buffCache $}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="col-md-6">
<div class="example-box-wrapper">
<div class="content-box-wrapper">
<table class="table">
<thead>
<tr>
<th style="background: #3498db;color:white">{% trans 'SWAP' %}</th>
<th style="background: #3498db;color:white">{% trans 'Free' %}</th>
<th style="background: #3498db;color:white">{% trans 'Used' %}</th>
<th style="background: #3498db;color:white">{% trans 'buff/cache' %}</th>
</tr>
</thead>
<tbody>
<tr>
<td>{$ swapTotalMemory $}</td>
<td>{$ swapFreeMemory $}</td>
<td>{$ swapUsedMemory $}</td>
<td>{$ swapBuffCache $}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="example-box-wrapper">
<h3 style="background:#3498db; color: white" class="content-box-header bg-black">
{% trans 'Top Processes' %}
</h3>
<div class="content-box-wrapper">
<div style="margin-top: 2%; margin-bottom: 2%" class="row">
<div class="col-sm-12">
<input placeholder="Search..." ng-model="search" name="dom" type="text"
class="form-control" ng-model="domainNameCreate" required>
</div>
</div>
<table class="table">
<thead>
<tr>
<th>{% trans 'PID' %}</th>
<th>{% trans 'User' %}</th>
<th>{% trans 'VIRT' %}</th>
<th>{% trans 'RES' %}</th>
<th>{% trans 'State' %}</th>
<th>{% trans '%CPU' %}</th>
<th>{% trans '%MEM' %}</th>
<th>{% trans 'Time' %}</th>
<th>{% trans 'Command' %}</th>
<th>{% trans 'Actions' %}</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="process in processes | filter:search">
<td ng-bind="process.PID"></td>
<td ng-bind="process.User"></td>
<td ng-bind="process.VIRT"></td>
<td ng-bind="process.RES"></td>
<td ng-bind="process.S"></td>
<td ng-bind="process.CPU"></td>
<td ng-bind="process.MEM"></td>
<td ng-bind="process.Time"></td>
<td ng-bind="process.Command"></td>
<td>
<button ng-click="killProcess(process.PID)" class="btn btn-danger">KILL</button>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
{% endblock %}

View File

@@ -15,5 +15,8 @@ urlpatterns = [
url(r'^switchTOLSWSStatus$', views.switchTOLSWSStatus, name='switchTOLSWSStatus'),
url(r'^licenseStatus$', views.licenseStatus, name='licenseStatus'),
url(r'^changeLicense$', views.changeLicense, name='changeLicense'),
url(r'^topProcesses$', views.topProcesses, name='topProcesses'),
url(r'^topProcessesStatus$', views.topProcessesStatus, name='topProcessesStatus'),
url(r'^killProcess$', views.killProcess, name='killProcess'),
]

View File

@@ -16,6 +16,7 @@ from plogical.virtualHostUtilities import virtualHostUtilities
import time
import serverStatusUtil
from plogical.processUtilities import ProcessUtilities
from plogical.httpProc import httpProc
# Create your views here.
def serverStatusHome(request):
@@ -444,3 +445,169 @@ def changeLicense(request):
final_dic = {'status': 0, 'erroMessage': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
def topProcesses(request):
try:
userID = request.session['userID']
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 1:
pass
else:
return ACLManager.loadError()
templateName = "serverStatus/topProcesses.html"
proc = httpProc(request, templateName)
return proc.renderPre()
except KeyError,msg:
logging.CyberCPLogFileWriter.writeToFile(str(msg) + "[litespeedStatus]")
return redirect(loadLoginPage)
def topProcessesStatus(request):
try:
with open("/home/cyberpanel/top", "w") as outfile:
subprocess.call("sudo top -n1 -b", shell=True, stdout=outfile)
data = open('/home/cyberpanel/top', 'r').readlines()
json_data = "["
checker = 0
counter = 0
loadAVG = data[0].split(' ')
loadAVG = filter(lambda a: a != '', loadAVG)
logging.CyberCPLogFileWriter.writeToFile(str(loadAVG))
loadNow = data[2].split(' ')
loadNow = filter(lambda a: a != '', loadNow)
memory = data[3].split(' ')
memory = filter(lambda a: a != '', memory)
swap = data[4].split(' ')
swap = filter(lambda a: a != '', swap)
processes = data[1].split(' ')
processes = filter(lambda a: a != '', processes)
for items in data:
counter = counter + 1
if counter <= 7:
continue
points = items.split(' ')
points = filter(lambda a: a != '', points)
dic = {'PID': points[0], 'User': points[1], 'VIRT': points[4],
'RES': points[5], 'S': points[7], 'CPU': points[8], 'MEM': points[9],
'Time': points[10], 'Command': points[11]
}
if checker == 0:
json_data = json_data + json.dumps(dic)
checker = 1
else:
json_data = json_data + ',' + json.dumps(dic)
json_data = json_data + ']'
data = {}
data['status'] = 1
data['error_message'] = 'None'
data['data'] = json_data
## CPU
data['cpuNow'] = loadNow[1]
data['cpuOne'] = loadAVG[-3].rstrip(',')
data['cpuFive'] = loadAVG[-2].rstrip(',')
data['cpuFifteen'] = loadAVG[-1]
## CPU Time spent
data['ioWait'] = loadNow[9] + '%'
data['idleTime'] = loadNow[7] + '%'
data['hwInterrupts'] = loadNow[11] + '%'
data['Softirqs'] = loadNow[13] + '%'
## Memory
data['totalMemory'] = str(int(float(memory[3])/1024)) + 'MB'
data['freeMemory'] = str(int(float(memory[5])/1024)) + 'MB'
data['usedMemory'] = str(int(float(memory[7])/1024)) + 'MB'
data['buffCache'] = str(int(float(memory[9])/1024)) + 'MB'
## Swap
data['swapTotalMemory'] = str(int(float(swap[2]) / 1024)) + 'MB'
data['swapFreeMemory'] = str(int(float(swap[4]) / 1024)) + 'MB'
data['swapUsedMemory'] = str(int(float(swap[6]) / 1024)) + 'MB'
data['swapBuffCache'] = str(int(float(swap[8]) / 1024)) + 'MB'
## Processes
data['totalProcesses'] = processes[1]
data['runningProcesses'] = processes[3]
data['sleepingProcesses'] = processes[5]
data['stoppedProcesses'] = processes[7]
data['zombieProcesses'] = processes[9]
## CPU Details
command = 'sudo cat /proc/cpuinfo'
output = subprocess.check_output(shlex.split(command)).splitlines()
import psutil
data['cores'] = psutil.cpu_count()
for items in output:
if items.find('model name') > -1:
modelName = items.split(':')[1].strip(' ')
index = modelName.find('CPU')
data['modelName'] = modelName[0:index]
elif items.find('cpu MHz') > -1:
data['cpuMHZ'] = items.split(':')[1].strip(' ')
elif items.find('cache size') > -1:
data['cacheSize'] = items.split(':')[1].strip(' ')
break
final_json = json.dumps(data)
return HttpResponse(final_json)
except BaseException, msg:
data_ret = {'status': 0, 'error_message': str(msg)}
json_data = json.dumps(data_ret)
return HttpResponse(json_data)
def killProcess(request):
try:
userID = request.session['userID']
try:
currentACL = ACLManager.loadedACL(userID)
if currentACL['admin'] == 1:
pass
else:
return ACLManager.loadErrorJson('status', 0)
data = json.loads(request.body)
pid = data['pid']
ProcessUtilities.executioner('sudo kill ' + pid)
proc = httpProc(request, None)
return proc.ajax(1, None)
except BaseException, msg:
final_dic = {'status': 0, 'erroMessage': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)
except KeyError, msg:
final_dic = {'status': 0, 'erroMessage': str(msg)}
final_json = json.dumps(final_dic)
return HttpResponse(final_json)

View File

@@ -637,3 +637,142 @@ app.controller('lswsSwitch', function ($scope, $http, $timeout, $window) {
}
});
app.controller('topProcesses', function ($scope, $http, $timeout) {
$scope.cyberPanelLoading = true;
$scope.topProcessesStatus = function () {
$scope.cyberPanelLoading = false;
url = "/serverstatus/topProcessesStatus";
var data = {};
var config = {
headers: {
'X-CSRFToken': getCookie('csrftoken')
}
};
$http.post(url, data, config).then(ListInitialDatas, cantLoadInitialDatas);
function ListInitialDatas(response) {
$scope.cyberPanelLoading = true;
if (response.data.status === 1) {
$scope.processes = JSON.parse(response.data.data);
//CPU Details
$scope.cores = response.data.cores;
$scope.modelName = response.data.modelName;
$scope.cpuMHZ = response.data.cpuMHZ;
$scope.cacheSize = response.data.cacheSize;
//CPU Load
$scope.cpuNow = response.data.cpuNow;
$scope.cpuOne = response.data.cpuOne;
$scope.cpuFive = response.data.cpuFive;
$scope.cpuFifteen = response.data.cpuFifteen;
//CPU Time spent
$scope.ioWait = response.data.ioWait;
$scope.idleTime = response.data.idleTime;
$scope.hwInterrupts = response.data.hwInterrupts;
$scope.Softirqs = response.data.Softirqs;
//Memory
$scope.totalMemory = response.data.totalMemory;
$scope.freeMemory = response.data.freeMemory;
$scope.usedMemory = response.data.usedMemory;
$scope.buffCache = response.data.buffCache;
//Swap
$scope.swapTotalMemory = response.data.swapTotalMemory;
$scope.swapFreeMemory = response.data.swapFreeMemory;
$scope.swapUsedMemory = response.data.swapUsedMemory;
$scope.swapBuffCache = response.data.swapBuffCache;
//Processes
$scope.totalProcesses = response.data.totalProcesses;
$scope.runningProcesses = response.data.runningProcesses;
$scope.sleepingProcesses = response.data.sleepingProcesses;
$scope.stoppedProcesses = response.data.stoppedProcesses;
$scope.zombieProcesses = response.data.zombieProcesses;
$timeout($scope.topProcessesStatus, 3000);
}
else {
new PNotify({
title: 'Operation Failed!',
text: response.data.error_message,
type: 'error'
});
}
}
function cantLoadInitialDatas(response) {
$scope.cyberPanelLoading = true;
new PNotify({
title: 'Operation Failed!',
text: 'Could not connect to server, please refresh this page',
type: 'error'
});
}
};
$scope.topProcessesStatus();
$scope.killProcess = function (pid) {
$scope.cyberPanelLoading = false;
url = "/serverstatus/killProcess";
var data = {
pid: pid
};
var config = {
headers: {
'X-CSRFToken': getCookie('csrftoken')
}
};
$http.post(url, data, config).then(ListInitialDatas, cantLoadInitialDatas);
function ListInitialDatas(response) {
$scope.cyberPanelLoading = true;
if (response.data.status === 1) {
new PNotify({
title: 'Success',
text: 'Process successfully killed.',
type: 'success'
});
}
else {
new PNotify({
title: 'Operation Failed!',
text: response.data.error_message,
type: 'error'
});
}
}
function cantLoadInitialDatas(response) {
$scope.cyberPanelLoading = true;
new PNotify({
title: 'Operation Failed!',
text: 'Could not connect to server, please refresh this page',
type: 'error'
});
}
};
});

View File

@@ -10,5 +10,3 @@
border-radius: 25px;
border-color:#3498db
}

View File

@@ -10,5 +10,3 @@
border-radius: 25px;
border-color:#3498db
}

View File

@@ -38,7 +38,9 @@
<tr ng-repeat="web in WebSitesList track by $index">
<td ng-bind="web.domain"></td>
<td><a href="{$ web.domain $}"><img width="30px" height="30" class="center-block" src="{% static 'baseTemplate/assets/image-resources/webPanel.png' %}"></a></td>
<td><a href="{$ web.domain $}"><img width="30px" height="30" class="center-block"
src="{% static 'baseTemplate/assets/image-resources/webPanel.png' %}"></a>
</td>
<td ng-bind="web.ipAddress"></td>
<td ng-bind="web.package"></td>
<td ng-bind="web.admin"></td>