s3 completion

api/urls.py

@@ -25,7 +25,7 @@ urlpatterns = [
     url(r'^cyberPanelVersion', views.cyberPanelVersion, name='cyberPanelVersion'),
     url(r'^putSSHkey', views.putSSHkey, name='putSSHkey'),
 
-
     url(r'^changeAdminPassword', views.changeAdminPassword, name='changeAdminPassword'),
+    url(r'^runAWSBackups$', views.runAWSBackups, name='runAWSBackups'),
 
 ]

api/views.py

@@ -19,10 +19,11 @@ from plogical.website import WebsiteManager
 from loginSystem.models import ACL
 from plogical.acl import ACLManager
 from firewall.models import FirewallRules
+from s3Backups.s3Backups import S3Backups
+from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
 # Create your views here.
 
 
-
 def verifyConn(request):
     try:
         if request.method == 'POST':
@@ -641,3 +642,15 @@ def changeAdminPassword(request):
 
         json_data = json.dumps(data_ret)
         return HttpResponse(json_data)
+
+def runAWSBackups(request):
+    try:
+
+        data = json.loads(request.body)
+        randomFile = data['randomFile']
+
+        if os.path.exists(randomFile):
+            s3 = S3Backups(request, None, 'runAWSBackups')
+            s3.start()
+    except BaseException, msg:
+        logging.writeToFile(str(msg) + ' [API.runAWSBackups]')

install/install.py

@@ -797,8 +797,8 @@ class preFlightsChecks:
 
         count = 0
         while (1):
-            command = "wget http://cyberpanel.net/CyberPanel.1.7.4.tar.gz"
+            #command = "wget http://cyberpanel.net/CyberPanel.1.7.4.tar.gz"
-            # command = "wget http://cyberpanel.net/CyberPanelTemp.tar.gz"
+            command = "wget http://cyberpanel.sh/CyberPanelTemp.tar.gz"
             res = subprocess.call(shlex.split(command))
 
             if preFlightsChecks.resFailed(self.distro, res):
@@ -818,8 +818,8 @@ class preFlightsChecks:
 
         count = 0
         while (1):
-            command = "tar zxf CyberPanel.1.7.4.tar.gz"
+            #command = "tar zxf CyberPanel.1.7.4.tar.gz"
-            # command = "tar zxf CyberPanelTemp.tar.gz"
+            command = "tar zxf CyberPanelTemp.tar.gz"
 
             res = subprocess.call(shlex.split(command))
 

install/installCyberPanel.py

@@ -219,12 +219,17 @@ class InstallCyberPanel:
                       'lsphp7? lsphp7?-common lsphp7?-curl lsphp7?-dev lsphp7?-imap lsphp7?-intl lsphp7?-json ' \
                       'lsphp7?-ldap lsphp7?-mysql lsphp7?-opcache lsphp7?-pspell lsphp7?-recode ' \
                       'lsphp7?-sqlite3 lsphp7?-tidy'
+
+            res = os.system(command)
+            if res != 0:
+                InstallCyberPanel.stdOut("Failed to install PHP on Ubuntu.", 1, 1)
+
         else:
             command = 'yum -y groupinstall lsphp-all'
 
-        install.preFlightsChecks.call(command, self.distro, '[installAllPHPVersions]',
+            install.preFlightsChecks.call(command, self.distro, '[installAllPHPVersions]',
-                                      'Install PHP',
+                                          'Install PHP',
-                                      1, 1, os.EX_OSERR)
+                                          1, 1, os.EX_OSERR)
 
         InstallCyberPanel.stdOut("LiteSpeed PHPs successfully installed!", 1)
 
@@ -391,7 +396,7 @@ class InstallCyberPanel:
             command = "systemctl enable mysql"
 
         install.preFlightsChecks.call(command, self.distro, '[installMySQL]',
-                                      'Install MySQL',
+                                      'Enable MySQL',
                                       1, 1, os.EX_OSERR)
 
     def fixMariaDB(self):

requirments.txt

@@ -2,6 +2,8 @@ acme==0.21.1
 asn1crypto==0.24.0
 Babel==0.9.6
 backports.ssl-match-hostname==3.5.0.1
+boto3==1.9.64
+botocore==1.12.64
 certbot==0.21.1
 certifi==2018.4.16
 cffi==1.11.5
@@ -11,21 +13,25 @@ configobj==4.7.2
 cryptography==2.2.2
 decorator==3.4.0
 Django==1.11
+docutils==0.14
 enum34==1.1.6
 funcsigs==1.0.2
 future==0.16.0
+futures==3.2.0
 gunicorn==19.8.1
 idna==2.6
 iniparse==0.4
 ipaddress==1.0.16
 IPy==0.75
 Jinja2==2.7.2
+jmespath==0.9.3
 josepy==1.1.0
 jsonpatch==1.2
 jsonpointer==1.9
 kitchen==1.1.1
 MarkupSafe==0.11
 mock==2.0.0
+MySQL-python==1.2.5
 parsedatetime==2.4
 pbr==4.0.4
 perf==0.1
@@ -36,20 +42,34 @@ ptyprocess==0.6.0
 pycparser==2.18
 pycurl==7.19.0
 pydns==2.3.6
+pygobject==3.22.0
 pygpgme==0.3
 pyliblzma==0.5.3
 pyOpenSSL==17.5.0
 pyRFC3339==1.1
 pyserial==2.6
+python-dateutil==2.7.5
+python-linux-procfs==0.4.9
 pytz==2018.4
 pyudev==0.15
 pyxattr==0.5.1
 PyYAML==3.10
 requests==2.18.4
 requests-file==1.4.3
+s3transfer==0.1.13
+schedutils==0.4
 six==1.9.0
+slip==0.4.0
+slip.dbus==0.4.0
 tldextract==2.2.0
+urlgrabber==3.10
 urllib3==1.22
+virtualenv==16.0.0
+yum-metadata-parser==1.1.4
 zope.component==4.4.1
+zope.deferredimport==4.3
+zope.deprecation==4.3.0
 zope.event==4.3.0
+zope.hookable==4.2.0
 zope.interface==4.5.0
+zope.proxy==4.3.1

s3Backups/models.py

@@ -11,7 +11,8 @@ class BackupPlan(models.Model):
     bucket = models.CharField(max_length=50, default='NONE')
     freq = models.CharField(max_length=50)
     retention = models.IntegerField()
-    lastRun = models.CharField(max_length=50, default='NEVER')
+    type = models.CharField(max_length=5, default='AWS')
+    lastRun = models.CharField(max_length=50, default='0:0:0')
 
 class WebsitesInPlan(models.Model):
     owner = models.ForeignKey(BackupPlan,on_delete=models.CASCADE)
@@ -20,5 +21,6 @@ class WebsitesInPlan(models.Model):
 
 class BackupLogs(models.Model):
     owner = models.ForeignKey(BackupPlan,on_delete=models.CASCADE)
+    timeStamp = models.CharField(max_length=200)
     level = models.CharField(max_length=5)
     msg = models.CharField(max_length=500)

s3Backups/s3Backups.py

@@ -1,23 +1,29 @@
 #!/usr/local/CyberCP/bin/python2
-import os.path
+try:
-import sys
+    import os
-import django
+    import os.path
-sys.path.append('/usr/local/CyberCP')
+    from django.shortcuts import HttpResponse
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")
+    from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
-django.setup()
+    from plogical.httpProc import httpProc
-from django.shortcuts import HttpResponse
+    from plogical.acl import ACLManager
-from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
+    import threading as multi
-from plogical.httpProc import httpProc
+    from plogical.mailUtilities import mailUtilities
-from plogical.acl import ACLManager
+    import boto3
-import threading as multi
+    import json
-import argparse
+    from .models import *
-from plogical.mailUtilities import mailUtilities
+    from math import ceil
-import boto3
+    import requests
-import json
+    import time
-from .models import *
+    from random import randint
-from math import ceil
+    import subprocess, shlex
-import requests
+except BaseException, msg:
-import time
+    from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
+    logging.writeToFile(str(msg))
+    import threading as multi
+    from random import randint
+    import json
+    import requests
+    import subprocess, shlex
 
 class S3Backups(multi.Thread):
 
@@ -33,6 +39,8 @@ class S3Backups(multi.Thread):
                 self.connectAccount()
             elif self.function == 'forceRunAWSBackup':
                 self.forceRunAWSBackup()
+            elif self.function == 'runAWSBackups':
+                self.runAWSBackups()
         except BaseException, msg:
             logging.writeToFile( str(msg) + ' [S3Backups.run]')
 
@@ -69,7 +77,7 @@ class S3Backups(multi.Thread):
         counter = 1
 
         for items in reversed(logs):
-            dic = { 'id': items.id, 'level': items.level, 'mesg': items.msg }
+            dic = { 'id': items.id, 'timeStamp': items.timeStamp, 'level': items.level, 'mesg': items.msg }
             if checker == 0:
                 json_data = json_data + json.dumps(dic)
                 checker = 1
@@ -104,6 +112,24 @@ class S3Backups(multi.Thread):
             credFile.write(self.data['credData'])
             credFile.close()
 
+            ##
+
+            cronPath = '/etc/crontab'
+
+            command = 'sudo cat ' + cronPath
+            output = subprocess.check_output(shlex.split(command)).split('\n')
+
+            insertCron = 1
+
+            for items in output:
+                if items.find('s3backups.py') > -1:
+                    insertCron = 0
+                    break
+
+            if insertCron:
+                command = 'echo "0 24 * * * root /usr/local/CyberCP/bin/python2 /usr/local/CyberCP/s3Backups/s3Backups.py" >> ' + cronPath
+                subprocess.call(command, shell=True)
+
             return proc.ajax(1, None)
 
         except BaseException, msg:
@@ -350,7 +376,10 @@ class S3Backups(multi.Thread):
         r = requests.post("http://localhost:5003/backup/submitBackupCreation", data=finalData)
 
         data = json.loads(r.text)
-        backupPath = data['tempStorage']
+        try:
+            backupPath = data['tempStorage']
+        except:
+            pass
 
         while (1):
             r = requests.post("http://localhost:5003/backup/backupStatus", data=finalData)
@@ -368,39 +397,105 @@ class S3Backups(multi.Thread):
             s3 = boto3.resource('s3')
             plan = BackupPlan.objects.get(name=self.data['planName'])
             bucketName = plan.bucket.strip('\n').strip(' ')
+            runTime = time.strftime("%d:%m:%Y")
+
+            ## Set Expiration for objects
+            try:
+                client = boto3.client('s3')
+                client.put_bucket_lifecycle_configuration(
+                    Bucket='string',
+                    LifecycleConfiguration={
+                        'Rules': [
+                            {
+                                'Expiration': {
+                                    'Days': plan.retention,
+                                    'ExpiredObjectDeleteMarker': True
+                                },
+                                'ID': plan.name,
+                                'Prefix': '',
+                                'Filter': {
+                                    'Prefix': plan.name + '/',
+                                },
+                                'Status': 'Enabled',
+
+                            },
+                        ]
+                    }
+                )
+            except BaseException, msg:
+                BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', msg=str(msg)).save()
+
+            ##
 
             userID = self.request.session['userID']
             currentACL = ACLManager.loadedACL(userID)
 
             if currentACL['admin'] == 0:
-                BackupLogs(owner=plan, level='INFO', msg='Unauthorised user tried to run AWS Backups.').save()
+                BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='INFO', msg='Unauthorised user tried to run AWS Backups.').save()
+                return 0
 
-            BackupLogs(owner=plan,level='INFO',  msg='Starting backup process..').save()
+            BackupLogs(owner=plan,level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"),  msg='Starting backup process..').save()
 
             for items in plan.websitesinplan_set.all():
                 result = self.createBackup(items.domain)
                 if result[0]:
                     data = open(result[1] + ".tar.gz", 'rb')
-                    s3.Bucket(bucketName).put_object(Key=result[1].split('/')[-1] + ".tar.gz", Body=data)
+                    s3.Bucket(bucketName).put_object(Key=plan.name + '/' + runTime + '/' + result[1].split('/')[-1] + ".tar.gz", Body=data)
-                    BackupLogs(owner=plan, level='INFO', msg='Backup successful for ' + items.domain + '.').save()
+                    BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), msg='Backup successful for ' + items.domain + '.').save()
                 else:
-                    BackupLogs(owner=plan, level='ERROR', msg='Backup failed for ' + items.domain + '. Error: ' + result[1]).save()
+                    BackupLogs(owner=plan, level='ERROR', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), msg='Backup failed for ' + items.domain + '. Error: ' + result[1]).save()
 
-            BackupLogs(owner=plan, level='INFO', msg='Backup Process Finished.').save()
+
+            plan.lastRun = runTime
+            plan.save()
+
+            BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), msg='Backup Process Finished.').save()
 
 
         except BaseException, msg:
             logging.writeToFile(str(msg) + ' [S3Backups.runBackupPlan]')
             plan = BackupPlan.objects.get(name=self.data['planName'])
-            BackupLogs(owner=plan, level='ERROR', msg=str(msg)).save()
+            BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', msg=str(msg)).save()
 
+    def runAWSBackups(self):
+        try:
+            admin = Administrator.objects.get(pk=1)
+            self.request.session['userID'] = admin.pk
+
+            for plan in BackupPlan.objects.all():
+                lastRunDay = plan.lastRun.split(':')[0]
+                lastRunMonth = plan.lastRun.split(':')[1]
+
+                if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"):
+                    self.data = {}
+                    self.data['planName'] = plan.name
+                    self.forceRunAWSBackup()
+                else:
+                    if lastRunMonth == time.strftime("%m"):
+                        days = int(time.strftime("%d")) - int(lastRunDay)
+                        if days >=6:
+                            self.data = {}
+                            self.data['planName'] = plan.name
+                            self.forceRunAWSBackup()
+                    else:
+                        days = 30 - int(lastRunDay)
+                        days = days + int(time.strftime("%d"))
+                        if days >=6:
+                            self.data = {}
+                            self.data['planName'] = plan.name
+                            self.forceRunAWSBackup()
+
+        except BaseException, msg:
+            logging.writeToFile(str(msg) + ' [S3Backups.runAWSBackups]')
 
 def main():
 
-    parser = argparse.ArgumentParser(description='CyberPanel S3 Backups')
+    pathToFile = "/home/cyberpanel/" + str(randint(1000, 9999))
-    parser.add_argument('function', help='Specify a function to call!')
+    file = open(pathToFile, "w")
+    file.close()
 
-    args = parser.parse_args()
+    finalData = json.dumps({'randomFile': pathToFile})
+    requests.post("http://localhost:5003/api/runAWSBackups", data=finalData,verify=False)
 
 if __name__ == "__main__":
     main()

serverStatus/views.py

@@ -478,7 +478,6 @@ def topProcessesStatus(request):
 
         loadAVG = data[0].split(' ')
         loadAVG = filter(lambda a: a != '', loadAVG)
-        logging.CyberCPLogFileWriter.writeToFile(str(loadAVG))
 
         loadNow = data[2].split(' ')
         loadNow = filter(lambda a: a != '', loadNow)
@@ -493,7 +492,6 @@ def topProcessesStatus(request):
         processes = data[1].split(' ')
         processes = filter(lambda a: a != '', processes)
 
-
         for items in data:
             counter = counter + 1
             if counter <= 7:
@@ -513,7 +511,6 @@ def topProcessesStatus(request):
             else:
                 json_data = json_data + ',' + json.dumps(dic)
 
-
         json_data = json_data + ']'
 
         data = {}

static/backup/backup.js

@@ -999,7 +999,7 @@ app.controller('remoteBackupControl', function($scope, $http, $timeout) {
 
     $scope.addRemoveWebsite = function (website,websiteStatus) {
 
-        if(websiteStatus==true)
+        if(websiteStatus === true)
         {
             var check = 1;
             for(var j = 0; j < websitesToBeBacked.length; j++){
@@ -1028,9 +1028,7 @@ app.controller('remoteBackupControl', function($scope, $http, $timeout) {
 
     $scope.allChecked = function (webSiteStatus) {
 
-
+        if(webSiteStatus === true) {
-
-        if(webSiteStatus==true) {
 
             websitesToBeBacked = websitesToBeBackedTemp;
             $scope.webSiteStatus = true;
@@ -1142,7 +1140,7 @@ app.controller('remoteBackupControl', function($scope, $http, $timeout) {
 
 
         if(websitesToBeBacked.length === 0){
-            alert("No websites selected for transfer.")
+            alert("No websites selected for transfer.");
             return;
         }