complete google drive backups;

2025-11-15 09:46:11 +01:00 · 2020-06-13 21:56:37 +05:00
parent 50871b2144
commit fb12c2679e
7 changed files with 396 additions and 31 deletions
--- a/backup/backupManager.py
+++ b/backup/backupManager.py
@@ -10,8 +10,7 @@ django.setup()
 import json
 from plogical.acl import ACLManager
 import plogical.CyberCPLogFileWriter as logging
-from websiteFunctions.models import Websites, Backups, dest, backupSchedules, BackupJob, BackupJobLogs, GDrive, \
-    GDriveSites, GDriveJobLogs
+from websiteFunctions.models import Websites, Backups, dest, backupSchedules, BackupJob, BackupJobLogs, GDrive, GDriveSites, GDriveJobLogs
 from plogical.virtualHostUtilities import virtualHostUtilities
 import subprocess
 import shlex
@@ -93,18 +92,10 @@ class BackupManager:
            gDriveData['client_secret'] = request.GET.get('cl')
            gDriveData['scopes'] = request.GET.get('s')

-            credentials = google.oauth2.credentials.Credentials(gDriveData['token'], gDriveData['refresh_token'],
-                                                                gDriveData['token_uri'], gDriveData['client_id'],
-                                                                gDriveData['client_secret'], gDriveData['scopes'])
-
-            drive = build('drive', 'v3', credentials=credentials)
-
            gD = GDrive(owner=admin, name=request.GET.get('n'), auth=json.dumps(gDriveData))
            gD.save()

-            final_json = json.dumps({'status': 1, 'message': 'Successfully saved.',
-                                     'data': str(drive.files().list(pageSize=10, fields="files(id, name)").execute())})
-            return HttpResponse(final_json)
+            return self.gDrive(request, userID)
        except BaseException as msg:
            final_dic = {'status': 0, 'fetchStatus': 0, 'error_message': str(msg)}
            final_json = json.dumps(final_dic)
@@ -318,8 +309,10 @@ class BackupManager:
            website = data['website']

            gD = GDrive.objects.get(name=selectedAccount)
-            gDSite = GDriveSites.objects.get(owner=gD, domain=website)
-            gDSite.delete()
+            sites = GDriveSites.objects.filter(owner=gD, domain=website)
+
+            for items in sites:
+                items.delete()

            data_ret = {'status': 1}
            json_data = json.dumps(data_ret)
--- a/backup/static/backup/backup.js
+++ b/backup/static/backup/backup.js
@@ -1611,7 +1611,7 @@ app.controller('googleDrive', function ($scope, $http) {

    $scope.fetchWebsites = function () {

-        $scope.cyberpanelLoading = false;
+        $scope.cyberPanelLoading = false;

        var config = {
            headers: {
@@ -1631,7 +1631,7 @@ app.controller('googleDrive', function ($scope, $http) {
        $http.post(dataurl, data, config).then(ListInitialDatas, cantLoadInitialDatas);

        function ListInitialDatas(response) {
-            $scope.cyberpanelLoading = true;
+            $scope.cyberPanelLoading = true;
            if (response.data.status === 1) {
                $scope.driveHidden = false;
                new PNotify({
@@ -1652,7 +1652,7 @@ app.controller('googleDrive', function ($scope, $http) {
        }

        function cantLoadInitialDatas(response) {
-            $scope.cyberpanelLoading = true;
+            $scope.cyberPanelLoading = true;
            new PNotify({
                title: 'Operation Failed!',
                text: 'Could not connect to server, please refresh this page.',
@@ -1859,7 +1859,7 @@ app.controller('googleDrive', function ($scope, $http) {

    $scope.fetchLogs = function () {

-        $scope.cyberpanelLoading = false;
+        $scope.cyberPanelLoading = false;

        var config = {
            headers: {
@@ -1879,7 +1879,7 @@ app.controller('googleDrive', function ($scope, $http) {
        $http.post(dataurl, data, config).then(ListInitialDatas, cantLoadInitialDatas);

        function ListInitialDatas(response) {
-            $scope.cyberpanelLoading = true;
+            $scope.cyberPanelLoading = true;
            if (response.data.status === 1) {
                $scope.driveHidden = false;
                new PNotify({
@@ -1899,14 +1899,13 @@ app.controller('googleDrive', function ($scope, $http) {
        }

        function cantLoadInitialDatas(response) {
-            $scope.cyberpanelLoading = true;
+            $scope.cyberPanelLoading = true;
            new PNotify({
                title: 'Operation Failed!',
                text: 'Could not connect to server, please refresh this page.',
                type: 'error'
            });

-
        }

    };
--- a/backup/templates/backup/googleDrive.html
+++ b/backup/templates/backup/googleDrive.html
@@ -114,10 +114,10 @@

                        <div ng-hide="driveHidden" class="form-group">
                            <div class="row">
-                                <div class="col-sm-3">
+                                <div style="margin-left: 2%" class="col-sm-3">
                                    <button data-toggle="modal" data-target="#backupLogs" ng-hide="driveHidden"
                                            type="button" ng-click="fetchLogs()"
-                                            class="btn btn-danger">{% trans "View Logs" %}</button>
+                                            class="btn btn-gray">{% trans "View Logs" %}</button>
                                    <div id="backupLogs" class="modal fade" role="dialog">
                                        <div class="modal-dialog modal-lg">
                                            <!-- Modal content-->
@@ -126,14 +126,14 @@
                                                    <button type="button" class="close" data-dismiss="modal">&times;
                                                    </button>
                                                    <h4 class="modal-title">{% trans "Git Logs" %} <img
-                                                            ng-hide="cyberpanelLoading"
+                                                            ng-hide="cyberPanelLoading"
                                                            src="{% static 'images/loading.gif' %}">
                                                    </h4>
                                                </div>
                                                <div class="modal-body">

                                                    <div class="row">
-                                                        <div class="col-sm-10">
+                                                        <div class="col-sm-9">
                                                        </div>
                                                        <div class="col-sm-2">
                                                            <div class="form-group">
@@ -148,7 +148,7 @@
                                                        </div>
                                                    </div>

-                                                    <table style="margin-top: 2%" class="table">
+                                                    <table style="margin: 0px; padding-bottom: 2%" class="table">
                                                        <thead>
                                                        <tr>
                                                            <th>Type</th>
@@ -166,7 +166,7 @@
                                                    <div style="margin-top: 2%" class="row">
                                                        <div class="col-md-12">
                                                            <div class="row">
-                                                                <div class="col-md-9">
+                                                                <div class="col-md-8">
                                                                </div>
                                                                <div class="col-md-3">
                                                                    <div class="form-group">
@@ -189,7 +189,7 @@
                                        </div>
                                    </div>
                                </div>
-                                <div class="col-sm-7">
+                                <div class="col-sm-6">
                                </div>
                                <div class="col-sm-2">
                                    <div class="form-group">
--- a/plogical/IncScheduler.py
+++ b/plogical/IncScheduler.py
@@ -10,16 +10,20 @@ from IncBackups.models import BackupJob
 from random import randint
 import argparse
 import json
-from websiteFunctions.models import GitLogs, Websites
+from websiteFunctions.models import GitLogs, Websites, GDrive, GDriveJobLogs
 from websiteFunctions.website import WebsiteManager
 import time
-
+import google.oauth2.credentials
+import googleapiclient.discovery
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
 try:
    from plogical.virtualHostUtilities import virtualHostUtilities
    from plogical.mailUtilities import mailUtilities
    from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
 except:
    pass
+from plogical.backupSchedule import backupSchedule

 class IncScheduler():
    logPath = '/home/cyberpanel/incbackuplogs'
@@ -192,6 +196,59 @@ class IncScheduler():
        except BaseException as msg:
            logging.writeToFile('[IncScheduler:193:checkDiskUsage] %s.' % str(msg))

+    @staticmethod
+    def runGoogleDriveBackups(type):
+
+        backupRunTime = time.strftime("%m.%d.%Y_%H-%M-%S")
+        backupLogPath = "/usr/local/lscp/logs/local_backup_log." + backupRunTime
+
+        for items in GDrive.objects.all():
+            try:
+                if items.runTime == type:
+                    gDriveData = json.loads(items.auth)
+                    try:
+                        credentials = google.oauth2.credentials.Credentials(gDriveData['token'], gDriveData['refresh_token'],
+                                                                    gDriveData['token_uri'], gDriveData['client_id'],
+                                                                    gDriveData['client_secret'], gDriveData['scopes'])
+                        drive = build('drive', 'v3', credentials=credentials)
+                    except BaseException as msg:
+                        GDriveJobLogs(owner=items, status=backupSchedule.ERROR, message='Connection to this account failed. Delete and re-setup this account. Error: %s' % (str(msg))).save()
+                        continue
+
+                    GDriveJobLogs(owner=items, status=backupSchedule.INFO, message='Starting backup job..').save()
+
+                    for website in items.gdrivesites_set.all():
+                        try:
+                            GDriveJobLogs(owner=items, status=backupSchedule.INFO, message='Local backup creation started for %s..' % (website.domain)).save()
+
+                            retValues = backupSchedule.createLocalBackup(website.domain, backupLogPath)
+
+                            if retValues[0] == 0:
+                                GDriveJobLogs(owner=items, status=backupSchedule.ERROR,
+                                              message='[ERROR] Backup failed for %s, error: %s moving on..' % (website.domain, retValues[1])).save()
+                                continue
+
+                            completeFileToSend = retValues[1] + ".tar.gz"
+                            fileName = completeFileToSend.split('/')[-1]
+
+                            file_metadata = {'name': '%s' % (fileName)}
+                            media = MediaFileUpload(completeFileToSend, mimetype='application/gzip', resumable=True)
+                            drive.files().create(body=file_metadata, media_body=media, fields='id').execute()
+
+                            GDriveJobLogs(owner=items, status=backupSchedule.INFO,
+                                          message='Backup for %s successfully sent to Googe Drive.' % (website.domain)).save()
+
+                        except BaseException as msg:
+                            GDriveJobLogs(owner=items, status=backupSchedule.ERROR,
+                                          message='[Site] Site backup failed, Error message: %s.' % (str(msg))).save()
+
+
+                    GDriveJobLogs(owner=items, status=backupSchedule.INFO,
+                                  message='Job Completed').save()
+            except BaseException as msg:
+                GDriveJobLogs(owner=items, status=backupSchedule.ERROR,
+                              message='[Completely] Job failed, Error message: %s.' % (str(msg))).save()
+

 def main():

@@ -200,6 +257,7 @@ def main():
    args = parser.parse_args()

    IncScheduler.startBackup(args.function)
+    IncScheduler.runGoogleDriveBackups(args.function)
    IncScheduler.git(args.function)
    IncScheduler.checkDiskUsage()

--- a/static/CLManager/CLManager.js
+++ b/static/CLManager/CLManager.js
@@ -489,7 +489,6 @@ app.controller('listCloudLinuxPackages', function ($scope, $http) {

 });

-
 app.controller('websiteContainerLimitCL', function ($scope, $http, $timeout, $window) {


--- a/static/backup/backup.js
+++ b/static/backup/backup.js
@@ -1534,6 +1534,7 @@ app.controller('remoteBackupControl', function ($scope, $http, $timeout) {
 ///** Backup site ends **///

 //*** Remote Backup site ****//
+
 app.controller('backupLogsScheduled', function ($scope, $http, $timeout) {

    $scope.cyberpanelLoading = true;
@@ -1595,3 +1596,318 @@ app.controller('backupLogsScheduled', function ($scope, $http, $timeout) {
 });

 ///** Backup site ends **///
+
+app.controller('googleDrive', function ($scope, $http) {
+
+    $scope.cyberPanelLoading = true;
+    $scope.driveHidden = true;
+
+    $scope.setupAccount = function(){
+        window.open("https://platform.cyberpanel.net/gDrive?name=" + $scope.accountName + '&server=' + window.location.href + 'Setup');
+    };
+
+    $scope.currentPage = 1;
+    $scope.recordsToShow = 10;
+
+    $scope.fetchWebsites = function () {
+
+        $scope.cyberPanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+
+        var data = {
+            selectedAccount: $scope.selectedAccount,
+            page: $scope.currentPage,
+            recordsToShow: $scope.recordsToShow
+        };
+
+
+        dataurl = "/backup/fetchgDriveSites";
+
+        $http.post(dataurl, data, config).then(ListInitialDatas, cantLoadInitialDatas);
+
+        function ListInitialDatas(response) {
+            $scope.cyberPanelLoading = true;
+            if (response.data.status === 1) {
+                $scope.driveHidden = false;
+                new PNotify({
+                    title: 'Success',
+                    text: 'Successfully fetched.',
+                    type: 'success'
+                });
+                $scope.websites = JSON.parse(response.data.websites);
+                $scope.pagination = response.data.pagination;
+                $scope.currently = response.data.currently;
+            } else {
+                new PNotify({
+                    title: 'Operation Failed!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+
+        function cantLoadInitialDatas(response) {
+            $scope.cyberPanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page.',
+                type: 'error'
+            });
+
+
+        }
+
+    };
+
+    $scope.addSite = function () {
+        $scope.cyberPanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+        var data = {
+            selectedWebsite: $scope.selectedWebsite,
+            selectedAccount: $scope.selectedAccount
+        };
+
+        dataurl = "/backup/addSitegDrive";
+
+        $http.post(dataurl, data, config).then(ListInitialData, cantLoadInitialData);
+
+
+        function ListInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            if (response.data.status === 1) {
+                new PNotify({
+                    title: 'Success',
+                    text: 'Site successfully added.',
+                    type: 'success'
+                });
+                $scope.fetchWebsites();
+            } else {
+                new PNotify({
+                    title: 'Operation Failed!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+
+        function cantLoadInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page',
+                type: 'error'
+            });
+        }
+
+
+    };
+
+    $scope.deleteAccount = function () {
+        $scope.cyberPanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+        var data = {
+            selectedAccount: $scope.selectedAccount
+        };
+
+        dataurl = "/backup/deleteAccountgDrive";
+
+        $http.post(dataurl, data, config).then(ListInitialData, cantLoadInitialData);
+
+
+        function ListInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            if (response.data.status === 1) {
+                new PNotify({
+                    title: 'Success',
+                    text: 'Account successfully deleted.',
+                    type: 'success'
+                });
+            } else {
+                new PNotify({
+                    title: 'Operation Failed!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+
+        function cantLoadInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page',
+                type: 'error'
+            });
+        }
+
+
+    };
+
+    $scope.changeFrequency = function () {
+        $scope.cyberPanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+        var data = {
+            selectedAccount: $scope.selectedAccount,
+            backupFrequency: $scope.backupFrequency
+        };
+
+        dataurl = "/backup/changeAccountFrequencygDrive";
+
+        $http.post(dataurl, data, config).then(ListInitialData, cantLoadInitialData);
+
+
+        function ListInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            if (response.data.status === 1) {
+                new PNotify({
+                    title: 'Success',
+                    text: 'Changes successfully applied',
+                    type: 'success'
+                });
+                $scope.fetchWebsites();
+            } else {
+                new PNotify({
+                    title: 'Operation Failed!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+
+        function cantLoadInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page',
+                type: 'error'
+            });
+        }
+
+
+    };
+
+    $scope.deleteSite = function (website) {
+        $scope.cyberPanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+        var data = {
+            selectedAccount: $scope.selectedAccount,
+            website: website
+        };
+
+        dataurl = "/backup/deleteSitegDrive";
+
+        $http.post(dataurl, data, config).then(ListInitialData, cantLoadInitialData);
+
+
+        function ListInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            if (response.data.status === 1) {
+                new PNotify({
+                    title: 'Success',
+                    text: 'Website Deleted.',
+                    type: 'success'
+                });
+                $scope.fetchWebsites();
+            } else {
+                new PNotify({
+                    title: 'Operation Failed!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+
+        function cantLoadInitialData(response) {
+            $scope.cyberPanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page',
+                type: 'error'
+            });
+        }
+
+
+    };
+
+    $scope.currentPageLogs = 1;
+    $scope.recordsToShowLogs = 10;
+
+    $scope.fetchLogs = function () {
+
+        $scope.cyberPanelLoading = false;
+
+        var config = {
+            headers: {
+                'X-CSRFToken': getCookie('csrftoken')
+            }
+        };
+
+        var data = {
+            selectedAccount: $scope.selectedAccount,
+            page: $scope.currentPageLogs,
+            recordsToShow: $scope.recordsToShowLogs
+        };
+
+
+        dataurl = "/backup/fetchDriveLogs";
+
+        $http.post(dataurl, data, config).then(ListInitialDatas, cantLoadInitialDatas);
+
+        function ListInitialDatas(response) {
+            $scope.cyberPanelLoading = true;
+            if (response.data.status === 1) {
+                $scope.driveHidden = false;
+                new PNotify({
+                    title: 'Success',
+                    text: 'Successfully fetched.',
+                    type: 'success'
+                });
+                $scope.logs = JSON.parse(response.data.logs);
+                $scope.paginationLogs = response.data.pagination;
+            } else {
+                new PNotify({
+                    title: 'Operation Failed!',
+                    text: response.data.error_message,
+                    type: 'error'
+                });
+            }
+        }
+
+        function cantLoadInitialDatas(response) {
+            $scope.cyberPanelLoading = true;
+            new PNotify({
+                title: 'Operation Failed!',
+                text: 'Could not connect to server, please refresh this page.',
+                type: 'error'
+            });
+
+        }
+
+    };
+
+});
--- a/websiteFunctions/models.py
+++ b/websiteFunctions/models.py
@@ -70,7 +70,7 @@ class GDrive(models.Model):

 class GDriveSites(models.Model):
    owner = models.ForeignKey(GDrive, on_delete=models.CASCADE)
-    domain = models.CharField(max_length=200, unique=True)
+    domain = models.CharField(max_length=200)

 class GDriveJobLogs(models.Model):
    owner = models.ForeignKey(GDrive, on_delete=models.CASCADE)