add detailed schedule backup logging

backup/backupManager.py

@@ -9,7 +9,7 @@ django.setup()
 import json
 from plogical.acl import ACLManager
 import plogical.CyberCPLogFileWriter as logging
-from websiteFunctions.models import Websites, Backups, dest, backupSchedules
+from websiteFunctions.models import Websites, Backups, dest, backupSchedules, BackupJob, BackupJobLogs
 from plogical.virtualHostUtilities import virtualHostUtilities
 import subprocess
 import shlex
@@ -1186,3 +1186,96 @@ class BackupManager:
             data = {'cancelStatus': 0, 'error_message': str(msg)}
             json_data = json.dumps(data)
             return HttpResponse(json_data)
+
+    def backupLogs(self, request = None, userID = None, data = None):
+        try:
+            currentACL = ACLManager.loadedACL(userID)
+
+            if currentACL['admin'] == 1:
+                pass
+            else:
+                return ACLManager.loadError()
+
+            all_files = []
+
+            logFiles = BackupJob.objects.all().order_by('-id')
+
+            for logFile in logFiles:
+                    all_files.append(logFile.logFile)
+
+            return render(request, 'backup/backupLogs.html', {'backups': all_files})
+
+        except BaseException as msg:
+            return HttpResponse(str(msg))
+
+    def fetchLogs(self, userID = None, data = None):
+        try:
+            currentACL = ACLManager.loadedACL(userID)
+
+            if currentACL['admin'] == 1:
+                pass
+            else:
+                return ACLManager.loadError()
+
+            page = int(str(data['page']).rstrip('\n'))
+            recordsToShow = int(data['recordsToShow'])
+            logFile = data['logFile']
+
+            logJob = BackupJob.objects.get(logFile=logFile)
+
+            logs = logJob.backupjoblogs_set.all()
+
+            from s3Backups.s3Backups import S3Backups
+            from plogical.backupSchedule import backupSchedule
+
+            pagination = S3Backups.getPagination(len(logs), recordsToShow)
+            endPageNumber, finalPageNumber = S3Backups.recordsPointer(page, recordsToShow)
+            finalLogs = logs[finalPageNumber:endPageNumber]
+
+            json_data = "["
+            checker = 0
+            counter = 0
+
+            for log in finalLogs:
+
+                if log.status == backupSchedule.INFO:
+                    status = 'INFO'
+                else:
+                    status = 'ERROR'
+
+                dic = {
+                    'LEVEL': status, "Message": log.message
+                }
+                if checker == 0:
+                    json_data = json_data + json.dumps(dic)
+                    checker = 1
+                else:
+                    json_data = json_data + ',' + json.dumps(dic)
+                counter = counter + 1
+
+            json_data = json_data + ']'
+
+            if logJob.location == backupSchedule.LOCAL:
+                location = 'local'
+            else:
+                location = 'remote'
+
+
+            data = {
+                    'status': 1,
+                    'error_message': 'None',
+                    'logs': json_data,
+                    'pagination': pagination,
+                    'jobSuccessSites': logJob.jobSuccessSites,
+                    'jobFailedSites': logJob.jobFailedSites,
+                    'location': location
+                    }
+
+
+            json_data = json.dumps(data)
+            return HttpResponse(json_data)
+
+        except BaseException as msg:
+            data = {'remoteRestoreStatus': 0, 'error_message': str(msg)}
+            json_data = json.dumps(data)
+            return HttpResponse(json_data)

backup/static/backup/backup.js

@@ -1542,3 +1542,67 @@ app.controller('remoteBackupControl', function ($scope, $http, $timeout) {
 });
 
 ///** Backup site ends **///
+
+
+//*** Remote Backup site ****//
+app.controller('backupLogsScheduled', function ($scope, $http, $timeout) {
+
+    $scope.cyberpanelLoading = true;
+    $scope.logDetails = true;
+
+    $scope.currentPage = 1;
+    $scope.recordsToShow = 10;
+
+    $scope.fetchLogs = function () {
+
+        $scope.cyberpanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+
+        var data = {
+            logFile: $scope.logFile,
+            recordsToShow: $scope.recordsToShow,
+            page: $scope.currentPage
+        };
+
+        dataurl = "/backup/fetchLogs";
+
+        $http.post(dataurl, data, config).then(ListInitialData, cantLoadInitialData);
+
+        function ListInitialData(response) {
+            $scope.cyberpanelLoading = true;
+            if (response.data.status === 1) {
+                $scope.logDetails = false;
+                $scope.logs = JSON.parse(response.data.logs);
+                $scope.pagination = response.data.pagination;
+                $scope.jobSuccessSites = response.data.jobSuccessSites;
+                $scope.jobFailedSites = response.data.jobFailedSites;
+                $scope.location = response.data.location;
+            } else {
+                new PNotify({
+                    title: 'Error!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+        function cantLoadInitialData(response) {
+            $scope.cyberpanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page',
+                type: 'error'
+            });
+        }
+
+
+    };
+
+
+});
+
+///** Backup site ends **///

backup/templates/backup/backupLogs.html

@@ -0,0 +1,125 @@
+{% extends "baseTemplate/index.html" %}
+{% load i18n %}
+{% block title %}{% trans "Backup Logs - CyberPanel" %}{% endblock %}
+{% block content %}
+
+    {% load static %}
+
+    {% get_current_language as LANGUAGE_CODE %}
+    <!-- Current language: {{ LANGUAGE_CODE }} -->
+
+
+    <div class="container">
+        <div id="page-title">
+            <h2>{% trans "Backup Logs" %}</h2>
+            <p>{% trans "On this page you can view detailed logs of your local and remote scheduled backups." %}</p>
+        </div>
+
+        <div ng-controller="backupLogsScheduled" class="panel">
+            <div class="panel-body">
+                <h3 class="title-hero">
+                    {% trans "Restore Website" %} <img ng-hide="cyberpanelLoading" src="{% static 'images/loading.gif' %}">
+                </h3>
+                <div class="example-box-wrapper">
+
+                    <form action="/" class="form-horizontal bordered-row">
+
+
+                        <div class="form-group">
+                            <label class="col-sm-3 control-label">{% trans "Select Log File" %}</label>
+                            <div class="col-sm-6">
+                                <select ng-change="fetchLogs()" ng-model="logFile" class="form-control">
+                                    {% for items in backups %}
+                                        <option>{{ items }}</option>
+                                    {% endfor %}
+                                </select>
+                            </div>
+                        </div>
+
+                    </form>
+
+                    <div ng-hide="logDetails" class="form-group">
+                        <div style="margin-bottom: 2%" class="col-sm-12">
+
+                            <table class="table" style="margin: 0px; padding: 0px">
+                                <thead>
+                                <tr>
+                                    <th>{% trans "Successful Sites" %}</th>
+                                    <th>{% trans "Failed Sites" %}</th>
+                                    <th>{% trans "Location" %}</th>
+                                </tr>
+                                </thead>
+                                <tbody>
+                                <tr>
+                                    <td>{$ jobSuccessSites $}</td>
+                                    <td>{$ jobFailedSites $}</td>
+                                    <td>{$ location $}</td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </div>
+                    </div>
+
+                    <div ng-hide="logDetails" class="col-sm-10"
+                         style="padding: 0px; box-shadow: 0px 0px 1px 0px #888888; margin-bottom: 2%">
+                        <input placeholder="Search..."
+                               ng-model="packSearch" name="packSearch" type="text"
+                               class="form-control" required>
+                    </div>
+                    <div ng-hide="logDetails" class="col-sm-2">
+                        <div class="form-group">
+                            <select ng-model="recordsToShow" ng-change="fetchLogs()"
+                                    class="form-control" id="example-select">
+                                <option>10</option>
+                                <option>50</option>
+                                <option>100</option>
+                                <option>500</option>
+                            </select>
+                        </div>
+                    </div>
+                    <div ng-hide="logDetails" class="form-group">
+                        <div class="col-sm-12">
+
+                            <table class="table" style="margin: 0px; padding: 0px">
+                                <thead>
+                                <tr>
+                                    <th>{% trans "LEVEL" %}</th>
+                                    <th>{% trans "Message" %}</th>
+                                </tr>
+                                </thead>
+                                <tbody>
+                                <tr ng-repeat="log in logs | filter:packSearch">
+                                    <td ng-bind="log.LEVEL"></td>
+                                    <td ng-bind="log.Message"></td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </div>
+                    </div>
+                    <div ng-hide="logDetails" style="margin-top: 2%" class="row">
+                        <div style="margin-top: 2%" class="col-md-12">
+                            <div class="row">
+                                <div class="col-md-9">
+                                </div>
+                                <div class="col-md-3">
+                                    <div class="form-group">
+                                        <select ng-model="currentPage" class="form-control"
+                                                ng-change="fetchLogs()">
+                                            <option ng-repeat="page in pagination">{$ $index + 1 $}
+                                            </option>
+                                        </select>
+                                    </div>
+                                </div>
+                            </div> <!-- end row -->
+                        </div>
+                    </div>
+
+                </div>
+            </div>
+        </div>
+
+
+    </div>
+
+
+{% endblock %}

backup/templates/backup/backupSchedule.html

@@ -11,7 +11,7 @@
     <div class="container">
         <div id="page-title">
             <h2>{% trans "Schedule Back up" %} - <a target="_blank" href="http://go.cyberpanel.net/remote-backup"
-                                                    style="height: 23px;line-height: 21px;"
+                                                    style="height: 23px;line-height: 21px; text-decoration: underline"
                                                     class="btn btn-border btn-alt border-red btn-link font-red"
                                                     title=""><span>{% trans "Remote Backups" %}</span></a></h2>
             <p>{% trans "On this page you can schedule Back ups to localhost or remote server (If you have added one)." %}</p>
@@ -20,8 +20,11 @@
         <div ng-controller="scheduleBackup" class="panel">
             <div class="panel-body">
                 <h3 class="title-hero">
-                    {% trans "Schedule Back up" %} <img ng-hide="scheduleBackupLoading"
+                    {% trans "Schedule Back up" %} - <img ng-hide="scheduleBackupLoading"
-                                                        src="{% static 'images/loading.gif' %}">
+                                                          src="{% static 'images/loading.gif' %}"> <a target="_blank"
+                                                                                                      href="{% url 'backupLogs' %}"
+                                                                                                      style="height: 23px;line-height: 21px;"
+                                                                                                      title="">{% trans "Backup Logs" %}</a>
                 </h3>
                 <div class="example-box-wrapper">
 

backup/urls.py

@@ -48,7 +48,7 @@ urlpatterns = [
 
     url(r'^localInitiate$', views.localInitiate, name='localInitiate'),
 
-
+    url(r'^backupLogs$', views.backupLogs, name='backupLogs'),
-
+    url(r'^fetchLogs$', views.fetchLogs, name='fetchLogs'),
 
 ]

backup/views.py

@@ -327,6 +327,23 @@ def cancelRemoteBackup(request):
     except KeyError:
         return redirect(loadLoginPage)
 
+def backupLogs(request):
+    try:
+        userID = request.session['userID']
+        bm = BackupManager()
+        return bm.backupLogs(request, userID)
+    except KeyError:
+        return redirect(loadLoginPage)
+
+def fetchLogs(request):
+    try:
+        userID = request.session['userID']
+
+        wm = BackupManager()
+        return wm.fetchLogs(userID, json.loads(request.body))
+
+    except KeyError:
+        return redirect(loadLoginPage)
 
 @csrf_exempt
 def localInitiate(request):

plogical/backupSchedule.py

@@ -12,7 +12,7 @@ import os
 import time
 from plogical.backupUtilities import backupUtilities
 from re import match,I,M
-from websiteFunctions.models import Backups
+from websiteFunctions.models import Backups, BackupJob, BackupJobLogs
 from plogical.processUtilities import ProcessUtilities
 from random import randint
 import json, requests
@@ -22,14 +22,22 @@ import signal
 
 class backupSchedule:
     now = datetime.now()
+    LOCAL = 0
+    REMOTE = 1
+    INFO = 0
+    ERROR = 1
+    backupLog = ''
 
     @staticmethod
-    def remoteBackupLogging(fileName, message):
+    def remoteBackupLogging(fileName, message, status = 0):
         try:
             file = open(fileName,'a')
             file.writelines("[" + time.strftime("%m.%d.%Y_%H-%M-%S") + "] "+ message + "\n")
             print(("[" + time.strftime("%m.%d.%Y_%H-%M-%S") + "] "+ message + "\n"))
             file.close()
+
+            BackupJobLogs(owner=backupSchedule.backupLog, status=status, message="[" + time.strftime("%m.%d.%Y_%H-%M-%S") + "] "+ message).save()
+
         except IOError as msg:
             return "Can not write to error file."
 
@@ -120,18 +128,25 @@ class backupSchedule:
                         except:
                             pass
 
-                        backupSchedule.remoteBackupLogging(backupLogPath, "An error occurred, Error message: " + status)
+                        backupSchedule.remoteBackupLogging(backupLogPath, "Local backup creating failed for %s, Error message: %s" % (virtualHost, status), backupSchedule.ERROR)
+
                         try:
                             os.remove(pathToFile)
                         except:
                             pass
                         return 0, tempStoragePath
+
                     elif os.path.exists(schedulerPath):
+                        backupSchedule.remoteBackupLogging(backupLogPath, 'Backup process killed without reporting any error.',
+                                                           backupSchedule.ERROR)
                         os.remove(schedulerPath)
                         return 0, 'Backup process killed without reporting any error.'
 
         except BaseException as msg:
             logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [119:startBackup]")
+            backupSchedule.remoteBackupLogging(backupLogPath,
+                                               "Local backup creating failed for %s, Error message: %s" % (
+                                               virtualHost, str(msg)), backupSchedule.ERROR)
             return 0, str(msg)
 
     @staticmethod

plogical/backupScheduleLocal.py

@@ -29,6 +29,12 @@ class backupScheduleLocal:
             backupRunTime = time.strftime("%m.%d.%Y_%H-%M-%S")
             backupLogPath = "/usr/local/lscp/logs/local_backup_log." + backupRunTime
 
+            jobSuccessSites = 0
+            jobFailedSites = 0
+
+            backupSchedule.backupLog = BackupJob(logFile=backupLogPath, location=backupSchedule.LOCAL, jobSuccessSites=jobSuccessSites, jobFailedSites=jobFailedSites)
+            backupSchedule.backupLog.save()
+
             writeToFile = open(backupLogPath, "a")
 
             backupSchedule.remoteBackupLogging(backupLogPath, "#################################################")
@@ -43,6 +49,9 @@ class backupScheduleLocal:
                     try:
                         retValues = backupSchedule.createLocalBackup(virtualHost, backupLogPath)
 
+                        if retValues[0] == 0:
+                            continue
+
                         if os.path.exists(backupScheduleLocal.localBackupPath):
                             backupPath = retValues[1] + ".tar.gz"
                             localBackupPath = '%s/%s' % (open(backupScheduleLocal.localBackupPath, 'r').read().rstrip('/'), backupRunTime)
@@ -52,10 +61,14 @@ class backupScheduleLocal:
 
                             command = 'mv %s %s' % (backupPath, localBackupPath)
                             ProcessUtilities.normalExecutioner(command)
-                    except BaseException as msg:
-                        backupSchedule.remoteBackupLogging(backupLogPath,
-                                                           '[ERROR] Backup failed for %s, error: %s moving on..' % (virtualHost, str(msg)))
 
+                        jobSuccessSites = jobSuccessSites + 1
+                    except BaseException as msg:
+
+                        jobFailedSites = jobFailedSites + 1
+
+                        backupSchedule.remoteBackupLogging(backupLogPath,
+                                                           '[ERROR] Backup failed for %s, error: %s moving on..' % (virtualHost, str(msg)), backupSchedule.ERROR)
 
 
 
@@ -71,6 +84,11 @@ class backupScheduleLocal:
 
             writeToFile.close()
 
+            job = BackupJob.objects.get(logFile=backupLogPath)
+            job.jobFailedSites = jobFailedSites
+            job.jobSuccessSites = jobSuccessSites
+            job.save()
+
         except BaseException as msg:
             logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [214:startBackup]")
 

serverStatus/views.py

@@ -93,7 +93,6 @@ def litespeedStatus(request):
         logging.CyberCPLogFileWriter.writeToFile(str(msg) + "[litespeedStatus]")
         return redirect(loadLoginPage)
 
-
 def stopOrRestartLitespeed(request):
     try:
         userID = request.session['userID']
@@ -127,7 +126,6 @@ def stopOrRestartLitespeed(request):
         logging.CyberCPLogFileWriter.writeToFile(str(msg) + "[stopOrRestartLitespeed]")
         return HttpResponse("Not Logged in as admin")
 
-
 def cyberCPMainLogFile(request):
     try:
         userID = request.session['userID']
@@ -145,7 +143,6 @@ def cyberCPMainLogFile(request):
         logging.CyberCPLogFileWriter.writeToFile(str(msg) + "[cyberCPMainLogFile]")
         return redirect(loadLoginPage)
 
-
 def getFurtherDataFromLogFile(request):
     try:
         userID = request.session['userID']
@@ -196,7 +193,6 @@ def services(request):
     except KeyError:
         return redirect(loadLoginPage)
 
-
 def servicesStatus(request):
     try:
         userID = request.session['userID']
@@ -291,7 +287,6 @@ def servicesStatus(request):
     except KeyError:
         return redirect(loadLoginPage)
 
-
 def servicesAction(request):
     try:
         userID = request.session['userID']
@@ -344,7 +339,6 @@ def servicesAction(request):
         final_json = json.dumps(final_dic)
         return HttpResponse(final_json)
 
-
 def switchTOLSWS(request):
     try:
         userID = request.session['userID']
@@ -378,7 +372,6 @@ def switchTOLSWS(request):
         json_data = json.dumps(data_ret)
         return HttpResponse(json_data)
 
-
 def switchTOLSWSStatus(request):
     try:
 

static/backup/backup.js

@@ -1542,3 +1542,67 @@ app.controller('remoteBackupControl', function ($scope, $http, $timeout) {
 });
 
 ///** Backup site ends **///
+
+
+//*** Remote Backup site ****//
+app.controller('backupLogsScheduled', function ($scope, $http, $timeout) {
+
+    $scope.cyberpanelLoading = true;
+    $scope.logDetails = true;
+
+    $scope.currentPage = 1;
+    $scope.recordsToShow = 10;
+
+    $scope.fetchLogs = function () {
+
+        $scope.cyberpanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+
+        var data = {
+            logFile: $scope.logFile,
+            recordsToShow: $scope.recordsToShow,
+            page: $scope.currentPage
+        };
+
+        dataurl = "/backup/fetchLogs";
+
+        $http.post(dataurl, data, config).then(ListInitialData, cantLoadInitialData);
+
+        function ListInitialData(response) {
+            $scope.cyberpanelLoading = true;
+            if (response.data.status === 1) {
+                $scope.logDetails = false;
+                $scope.logs = JSON.parse(response.data.logs);
+                $scope.pagination = response.data.pagination;
+                $scope.jobSuccessSites = response.data.jobSuccessSites;
+                $scope.jobFailedSites = response.data.jobFailedSites;
+                $scope.location = response.data.location;
+            } else {
+                new PNotify({
+                    title: 'Error!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+        function cantLoadInitialData(response) {
+            $scope.cyberpanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page',
+                type: 'error'
+            });
+        }
+
+
+    };
+
+
+});
+
+///** Backup site ends **///