Merge branch 'stable' into p3 to pass on bug fix for backups

install/install.py

@@ -1772,7 +1772,7 @@ imap_folder_list_limit = 0
             preFlightsChecks.call(command, self.distro, command, command, 1, 0, os.EX_OSERR)
 
             if self.distro == ubuntu:
-                command = 'apt install opendkim-tools'
+                command = 'apt install opendkim-tools -y'
                 preFlightsChecks.call(command, self.distro, command, command, 1, 0, os.EX_OSERR)
 
                 command = 'mkdir -p /etc/opendkim/keys/'

install/installCyberPanel.py

@@ -454,14 +454,14 @@ class InstallCyberPanel:
 
 
 
-                try:
+                # try:
-                    f = open('/etc/resolv.conf', 'w')
+                #     f = open('/etc/resolv.conf', 'a')
-                    f.write('nameserver 8.8.8.8')
+                #     f.write('nameserver 8.8.8.8')
-                    f.close()
+                #     f.close()
-                except IOError as e:
+                # except IOError as e:
-                    InstallCyberPanel.stdOut("[ERROR] Unable to create /etc/resolv.conf: " + str(e) +
+                #     InstallCyberPanel.stdOut("[ERROR] Unable to create /etc/resolv.conf: " + str(e) +
-                                             ".  This may need to be fixed manually as 'echo \"nameserver 8.8.8.8\"> "
+                #                              ".  This may need to be fixed manually as 'echo \"nameserver 8.8.8.8\"> "
-                                             "/etc/resolv.conf'", 1, 1, os.EX_OSERR)
+                #                              "/etc/resolv.conf'", 1, 1, os.EX_OSERR)
 
             if self.distro == centos:
                 command = 'yum -y install epel-release'

plogical/backupSchedule.py

@@ -16,9 +16,12 @@ from websiteFunctions.models import Websites, Backups
 from plogical.processUtilities import ProcessUtilities
 from random import randint
 import json, requests
+from datetime import datetime
+import signal
+
 
 class backupSchedule:
-
+    now = datetime.now()
 
     @staticmethod
     def remoteBackupLogging(fileName, message):
@@ -34,6 +37,8 @@ class backupSchedule:
     def createLocalBackup(virtualHost, backupLogPath):
         try:
 
+            startingTime = datetime.now()
+
             backupSchedule.remoteBackupLogging(backupLogPath, "Starting local backup for: " + virtualHost)
 
             ###
@@ -49,8 +54,12 @@ class backupSchedule:
             tempStoragePath = data['tempStorage']
 
             backupSchedule.remoteBackupLogging(backupLogPath, "Waiting for backup to complete.. ")
+            time.sleep(5)
+            schedulerPath = '/home/cyberpanel/%s-backup.txt' % (virtualHost)
 
             while (1):
+                diff = datetime.now() - startingTime
+
                 backupDomain = virtualHost
                 status = os.path.join("/home", backupDomain, "backup/status")
                 backupFileNamePath = os.path.join("/home", backupDomain, "backup/backupFileName")
@@ -114,6 +123,10 @@ class backupSchedule:
                         except:
                             pass
                         return 0, tempStoragePath
+                    elif os.path.exists(schedulerPath):
+                        os.remove(schedulerPath)
+                        return 0, 'Backup process killed without reporting any error.'
+
         except BaseException as msg:
             logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [119:startBackup]")
             return 0, str(msg)
@@ -247,6 +260,14 @@ class backupSchedule:
 def main():
     backupSchedule.prepare()
 
+def handler(signum, frame):
+    diff = datetime.now() - backupSchedule.now
+    logging.CyberCPLogFileWriter.writeToFile('Signal: %s, time spent: %s' % (str(signum), str(diff.total_seconds())))
+
 
 if __name__ == "__main__":
+    for i in range(1,32):
+        if i == 9 or i == 19 or i == 32:
+            continue
+        signal.signal(i, handler)
     main()

plogical/backupScheduleLocal.py

@@ -4,9 +4,12 @@ import time
 from plogical.backupSchedule import backupSchedule
 from plogical.processUtilities import ProcessUtilities
 from re import match,I,M
+import signal
+from datetime import datetime
 
 class backupScheduleLocal:
     localBackupPath = '/home/cyberpanel/localBackupPath'
+    now = datetime.now()
 
     @staticmethod
     def prepare():
@@ -24,17 +27,22 @@ class backupScheduleLocal:
 
             for virtualHost in os.listdir("/home"):
                 if match(r'([\da-z\.-]+\.[a-z\.]{2,12}|[\d\.]+)([\/:?=&#]{1}[\da-z\.-]+)*[\/\?]?', virtualHost, M | I):
-                    retValues = backupSchedule.createLocalBackup(virtualHost, backupLogPath)
+                    try:
+                        retValues = backupSchedule.createLocalBackup(virtualHost, backupLogPath)
 
-                    if os.path.exists(backupScheduleLocal.localBackupPath):
+                        if os.path.exists(backupScheduleLocal.localBackupPath):
-                        backupPath = retValues[1] + ".tar.gz"
+                            backupPath = retValues[1] + ".tar.gz"
-                        localBackupPath = '%s/%s' % (open(backupScheduleLocal.localBackupPath, 'r').read().rstrip('/'), time.strftime("%b-%d-%Y"))
+                            localBackupPath = '%s/%s' % (open(backupScheduleLocal.localBackupPath, 'r').read().rstrip('/'), time.strftime("%b-%d-%Y"))
 
-                        command = 'mkdir -p %s' % (localBackupPath)
+                            command = 'mkdir -p %s' % (localBackupPath)
-                        ProcessUtilities.normalExecutioner(command)
+                            ProcessUtilities.normalExecutioner(command)
+
+                            command = 'mv %s %s' % (backupPath, localBackupPath)
+                            ProcessUtilities.normalExecutioner(command)
+                    except BaseException, msg:
+                        backupSchedule.remoteBackupLogging(backupLogPath,
+                                                           '[ERROR] Backup failed for %s, error: %s moving on..' % (virtualHost, str(msg)))
 
-                        command = 'mv %s %s' % (backupPath, localBackupPath)
-                        ProcessUtilities.normalExecutioner(command)
 
 
 
@@ -57,5 +65,13 @@ def main():
     backupScheduleLocal.prepare()
 
 
+def handler(signum, frame):
+    diff = datetime.now() - backupScheduleLocal.now
+    logging.CyberCPLogFileWriter.writeToFile('Signal: %s, time spent: %s' % (str(signum), str(diff.total_seconds())))
+
 if __name__ == "__main__":
+    for i in range(1,32):
+        if i == 9 or i == 19 or i == 32:
+            continue
+        signal.signal(i, handler)
     main()

plogical/backupUtilities.py

@@ -263,6 +263,10 @@ class backupUtilities:
             ##### Writing the name of backup file.
 
             ## /home/example.com/backup/backupFileName
+            pidFile = '%sstartBackup' % (backupPath)
+            writeToFile = open(pidFile, 'w')
+            writeToFile.writelines(str(os.getpid()))
+            writeToFile.close()
 
             backupFileNamePath = os.path.join(backupPath,"backupFileName")
             logging.CyberCPLogFileWriter.statusWriter(backupFileNamePath, backupName)
@@ -321,9 +325,17 @@ class backupUtilities:
             logging.CyberCPLogFileWriter.statusWriter(status, "Aborted, "+ str(msg) + ".[365] [5009]")
             print(("Aborted, "+ str(msg) + ".[365] [5009]"))
 
+        os.remove(pidFile)
+
     @staticmethod
     def BackupRoot(tempStoragePath, backupName, backupPath, metaPath=None):
 
+        pidFile = '%sBackupRoot' % (backupPath)
+
+        writeToFile = open(pidFile, 'w')
+        writeToFile.writelines(str(os.getpid()))
+        writeToFile.close()
+
         status = os.path.join(backupPath, 'status')
         metaPathInBackup = os.path.join(tempStoragePath, 'meta.xml')
         backupMetaData = ElementTree.parse(metaPathInBackup)
@@ -422,6 +434,7 @@ class backupUtilities:
                 items.save()
 
         logging.CyberCPLogFileWriter.statusWriter(status, "Completed\n")
+        os.remove(pidFile)
 
     @staticmethod
     def initiateBackup(tempStoragePath,backupName,backupPath):
@@ -1056,16 +1069,24 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
 
         ##
 
+        schedulerPath = '/home/cyberpanel/%s-backup.txt' % (backupDomain)
+
         if not os.path.exists(backupPath) or not os.path.islink(backupPath):
             command = 'mkdir -p %s' % (backupPath)
             ProcessUtilities.executioner(command)
         else:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
         if not os.path.exists(backupPath) or not os.path.islink(backupPath):
             command = 'chown -R %s:%s %s' % (website.externalApp, website.externalApp, backupPath)
             ProcessUtilities.executioner(command)
         else:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
         ##
@@ -1074,12 +1095,18 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
             command = 'mkdir -p %s' % (tempStoragePath)
             ProcessUtilities.executioner(command)
         else:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
         if not os.path.exists(tempStoragePath) or not os.path.islink(tempStoragePath):
             command = 'chown -R %s:%s %s' % (website.externalApp, website.externalApp, tempStoragePath)
             ProcessUtilities.executioner(command)
         else:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
         ##
@@ -1087,12 +1114,18 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
             command = 'touch %s' % (status)
             ProcessUtilities.executioner(command)
         else:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
         if not os.path.exists(status) or not os.path.islink(status):
             command = 'chown cyberpanel:cyberpanel %s' % (status)
             ProcessUtilities.executioner(command)
         else:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
 
@@ -1100,6 +1133,9 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
 
 
         if result[0] == 0:
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             logging.CyberCPLogFileWriter.statusWriter(status, str(result[1]) + ' [1084][5009]')
             return 0
 
@@ -1113,6 +1149,9 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
         output = ProcessUtilities.outputExecutioner(execPath, website.externalApp)
         if output.find('[5009') > -1:
             logging.CyberCPLogFileWriter.writeToFile(output)
+            writeToFile = open(schedulerPath, 'w')
+            writeToFile.writelines('error')
+            writeToFile.close()
             return 0
 
         ## Backing up databases
@@ -1126,6 +1165,9 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
             dbName = database.find('dbName').text
 
             if mysqlUtilities.mysqlUtilities.createDatabaseBackup(dbName, '/home/cyberpanel') == 0:
+                writeToFile = open(schedulerPath, 'w')
+                writeToFile.writelines('error')
+                writeToFile.close()
                 return 0
 
             command = 'mv /home/cyberpanel/%s.sql %s/%s.sql' % (dbName, tempStoragePath, dbName)
@@ -1134,6 +1176,7 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain):
         ##
 
         output = ProcessUtilities.outputExecutioner(execPath, website.externalApp)
+
         if output.find('1,None') > -1:
             execPath = "sudo nice -n 10 /usr/local/CyberCP/bin/python " + virtualHostUtilities.cyberPanel + "/plogical/backupUtilities.py"
             execPath = execPath + " BackupRoot --tempStoragePath " + tempStoragePath + " --backupName " \

test.sh

@@ -0,0 +1 @@
+