CyberPanel/IncBackups/IncBackupsControl.py

#!/usr/local/CyberCP/bin/python2
import os
import os.path
import sys
sys.path.append('/usr/local/CyberCP')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")

import django
try:
    django.setup()
except:
    pass
import threading as multi
from plogical.processUtilities import ProcessUtilities
from IncBackups.models import IncJob, JobSnapshots
from websiteFunctions.models import Websites
import plogical.randomPassword as randomPassword
from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
from xml.etree.ElementTree import Element, SubElement
from xml.etree import ElementTree
from xml.dom import minidom
from backup.models import DBUsers
import plogical.mysqlUtilities as mysqlUtilities
from plogical.backupUtilities import backupUtilities
from plogical.dnsUtilities import DNS
from mailServer.models import Domains as eDomains
from random import randint
try:
    from plogical.virtualHostUtilities import virtualHostUtilities
    from plogical.mailUtilities import mailUtilities
except:
    pass

class IncJobs(multi.Thread):

    def __init__(self, function, extraArgs):
        multi.Thread.__init__(self)
        self.function = function
        self.extraArgs = extraArgs
        self.repoPath = ''
        self.passwordFile = ''
        self.statusPath = ''
        self.website = ''
        self.backupDestinations = ''
        self.jobid = 0
        self.metaPath = ''

    def run(self):

        if self.function == 'createBackup':
            self.createBackup()
        elif self.function == 'restorePoint':
            self.restorePoint()

    def restoreData(self):
        try:

            if self.jobid.destination == 'local':
                repoLocation = '/home/%s/incbackup' % (self.website)
                command = 'restic -r %s restore %s --target / --password-file %s' % (repoLocation, self.jobid.snapshotid, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
            elif self.jobid.destination[:4] == 'sftp':
                repoLocation = '/home/backup/%s' % (self.website)
                command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)

        except BaseException, msg:
            logging.statusWriter(self.statusPath, "%s [46][5009]" % (str(msg)), 1)
            return 0

    def restoreDatabase(self):
        try:

            if self.jobid.destination == 'local':
                repoLocation = '/home/%s/incbackup' % (self.website)
                command = 'restic -r %s restore %s --target / --password-file %s' % (repoLocation, self.jobid.snapshotid, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
            elif self.jobid.destination[:4] == 'sftp':
                repoLocation = '/home/backup/%s' % (self.website)
                command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)

            if mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(self.jobid.type.split(':')[1], '/home/cyberpanel', 'dummy', 'dummy') == 0:
                raise BaseException

            try:
                os.remove('/home/cyberpanel/%s' % (self.jobid.type.split(':')[1]))
            except:
                pass

        except BaseException, msg:
            logging.statusWriter(self.statusPath, "%s [46][5009]" % (str(msg)), 1)
            return 0

    def restoreEmail(self):
        try:

            if self.jobid.destination == 'local':
                repoLocation = '/home/%s/incbackup' % (self.website)
                command = 'restic -r %s restore %s --target / --password-file %s' % (repoLocation, self.jobid.snapshotid, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
            elif self.jobid.destination[:4] == 'sftp':
                repoLocation = '/home/backup/%s' % (self.website)
                command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target / --password-file %s' % (self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)

        except BaseException, msg:
            logging.statusWriter(self.statusPath, "%s [46][5009]" % (str(msg)), 1)
            return 0

    def restorePoint(self):
        try:
            self.statusPath = self.extraArgs['tempPath']
            self.website = self.extraArgs['website']
            jobid = self.extraArgs['jobid']

            self.jobid = JobSnapshots.objects.get(pk=jobid)

            message = 'Starting restore of %s for %s.' % (self.jobid.snapshotid, self.website)
            logging.statusWriter(self.statusPath, message, 1)
            self.passwordFile = '/home/%s/%s' % (self.website, self.website)


            ## Restore Meta first
            metaPathNew = '/home/%s/meta.xml' % (self.website)
            if self.jobid.destination == 'local':
                repoLocation = '/home/%s/incbackup' % (self.website)
                command = 'restic -r %s restore latest --target / --password-file %s --include %s' % (repoLocation, self.passwordFile, metaPathNew)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
            elif self.jobid.destination[:4] == 'sftp':
                repoLocation = '/home/backup/%s' % (self.website)
                command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore latest --target / --password-file %s --include %s' % (self.jobid.destination, repoLocation, self.passwordFile, metaPathNew)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)

            ## Reconstruct configs configs

            if self.extraArgs['reconstruct']:
                execPath = "nice -n 10 /usr/local/CyberCP/bin/python2 " + virtualHostUtilities.cyberPanel + "/IncBackups/restoreMeta.py"
                execPath = execPath + " submitRestore --metaPath %s --statusFile %s" % (metaPathNew, self.statusPath)
                ProcessUtilities.outputExecutioner(execPath)

            ##

            if self.jobid.type[:8] == 'database':
                message = 'Restoring database..'
                logging.statusWriter(self.statusPath, message, 1)
                self.restoreDatabase()
                message = 'Database restored.'
                logging.statusWriter(self.statusPath, message, 1)
            elif self.jobid.type[:4] == 'data':
                self.restoreData()
            elif self.jobid.type[:5] == 'email':
                message = 'Restoring email..'
                logging.statusWriter(self.statusPath, message, 1)
                self.restoreEmail()
                message = 'Emails restored.'
                logging.statusWriter(self.statusPath, message, 1)

            logging.statusWriter(self.statusPath, 'Completed', 1)
        except BaseException, msg:
            logging.statusWriter(self.extraArgs['tempPath'], str(msg), 1)

    def prepareBackupMeta(self):
        try:

            ######### Generating meta

            ## XML Generation

            metaFileXML = Element('metaFile')

            child = SubElement(metaFileXML, 'masterDomain')
            child.text = self.website.domain

            child = SubElement(metaFileXML, 'phpSelection')
            child.text = self.website.phpSelection

            child = SubElement(metaFileXML, 'externalApp')
            child.text = self.website.externalApp

            childDomains = self.website.childdomains_set.all()

            databases = self.website.databases_set.all()

            ## Child domains XML

            childDomainsXML = Element('ChildDomains')

            for items in childDomains:
                childDomainXML = Element('domain')

                child = SubElement(childDomainXML, 'domain')
                child.text = items.domain
                child = SubElement(childDomainXML, 'phpSelection')
                child.text = items.phpSelection
                child = SubElement(childDomainXML, 'path')
                child.text = items.path

                childDomainsXML.append(childDomainXML)

            metaFileXML.append(childDomainsXML)

            ## Databases XML

            databasesXML = Element('Databases')

            for items in databases:
                try:
                    dbuser = DBUsers.objects.get(user=items.dbUser)
                    userToTry = items.dbUser
                except:
                    dbusers = DBUsers.objects.all().filter(user=items.dbUser)
                    userToTry = items.dbUser
                    for it in dbusers:
                        dbuser = it
                        break

                    userToTry = mysqlUtilities.mysqlUtilities.fetchuser(items.dbUser)

                    try:
                        dbuser = DBUsers.objects.get(user=userToTry)
                    except:
                        dbusers = DBUsers.objects.all().filter(user=userToTry)
                        for it in dbusers:
                            dbuser = it
                            break

                databaseXML = Element('database')

                child = SubElement(databaseXML, 'dbName')
                child.text = items.dbName
                child = SubElement(databaseXML, 'dbUser')
                child.text = userToTry
                child = SubElement(databaseXML, 'password')
                child.text = dbuser.password

                databasesXML.append(databaseXML)

            metaFileXML.append(databasesXML)

            ## Get Aliases

            aliasesXML = Element('Aliases')

            aliases = backupUtilities.getAliases(self.website.domain)

            for items in aliases:
                child = SubElement(aliasesXML, 'alias')
                child.text = items

            metaFileXML.append(aliasesXML)

            ## Finish Alias

            ## DNS Records XML

            try:

                dnsRecordsXML = Element("dnsrecords")
                dnsRecords = DNS.getDNSRecords(self.website.domain)

                for items in dnsRecords:
                    dnsRecordXML = Element('dnsrecord')

                    child = SubElement(dnsRecordXML, 'type')
                    child.text = items.type
                    child = SubElement(dnsRecordXML, 'name')
                    child.text = items.name
                    child = SubElement(dnsRecordXML, 'content')
                    child.text = items.content
                    child = SubElement(dnsRecordXML, 'priority')
                    child.text = str(items.prio)

                    dnsRecordsXML.append(dnsRecordXML)

                metaFileXML.append(dnsRecordsXML)

            except BaseException, msg:
                logging.statusWriter(self.statusPath, '%s. [158:prepMeta]' % (str(msg)), 1)

            ## Email accounts XML

            try:
                emailRecordsXML = Element('emails')
                eDomain = eDomains.objects.get(domain=self.website.domain)
                emailAccounts = eDomain.eusers_set.all()

                for items in emailAccounts:
                    emailRecordXML = Element('emailAccount')

                    child = SubElement(emailRecordXML, 'email')
                    child.text = items.email
                    child = SubElement(emailRecordXML, 'password')
                    child.text = items.password

                    emailRecordsXML.append(emailRecordXML)

                metaFileXML.append(emailRecordsXML)
            except BaseException, msg:
                logging.statusWriter(self.statusPath, '%s. [warning:179:prepMeta]' % (str(msg)), 1)

            ## Email meta generated!

            def prettify(elem):
                """Return a pretty-printed XML string for the Element.
                """
                rough_string = ElementTree.tostring(elem, 'utf-8')
                reparsed = minidom.parseString(rough_string)
                return reparsed.toprettyxml(indent="  ")

            ## /home/example.com/backup/backup-example-06-50-03-Thu-Feb-2018/meta.xml -- metaPath

            metaPath = '/home/cyberpanel/%s' % (str(randint(1000, 9999)))

            xmlpretty = prettify(metaFileXML).encode('ascii', 'ignore')
            metaFile = open(metaPath, 'w')
            metaFile.write(xmlpretty)
            metaFile.close()
            os.chmod(metaPath, 0640)

            ## meta generated

            logging.statusWriter(self.statusPath, 'Meta data is ready..', 1)

            metaPathNew = '/home/%s/meta.xml' % (self.website.domain)
            command = 'mv %s %s' % (metaPath, metaPathNew)
            ProcessUtilities.executioner(command)

            return 1

        except BaseException, msg:
            logging.statusWriter(self.statusPath, "%s [207][5009]" % (str(msg)), 1)
            return 0

    def backupData(self):
        try:
            logging.statusWriter(self.statusPath, 'Backing up data..', 1)

            if self.backupDestinations == 'local':
                backupPath = '/home/%s' % (self.website.domain)
                command = 'restic -r %s backup %s --password-file %s --exclude %s' % (self.repoPath, backupPath, self.passwordFile, self.repoPath)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
                snapShotid = result.split(' ')[-2]

                newSnapshot = JobSnapshots(job=self.jobid, type='data:%s' % (backupPath), snapshotid=snapShotid, destination=self.backupDestinations)
                newSnapshot.save()

            elif self.backupDestinations[:4] == 'sftp':
                remotePath = '/home/backup/%s' % (self.website.domain)
                backupPath = '/home/%s' % (self.website.domain)
                command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (self.backupDestinations, remotePath, backupPath, self.passwordFile, self.repoPath)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
                snapShotid = result.split(' ')[-2]
                newSnapshot = JobSnapshots(job=self.jobid, type='data:%s' % (remotePath), snapshotid=snapShotid,
                                           destination=self.backupDestinations)
                newSnapshot.save()

            logging.statusWriter(self.statusPath, 'Data for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
            return 1
        except BaseException, msg:
            logging.statusWriter(self.statusPath,'%s. [IncJobs.backupData.223][5009]' % str(msg), 1)
            return 0

    def backupDatabases(self):
        try:
            logging.statusWriter(self.statusPath, 'Backing up databases..', 1)

            databases = self.website.databases_set.all()

            for items in databases:
                if mysqlUtilities.mysqlUtilities.createDatabaseBackup(items.dbName, '/home/cyberpanel') == 0:
                    return 0

                dbPath = '/home/cyberpanel/%s.sql' % (items.dbName)

                if self.backupDestinations == 'local':
                    command = 'restic -r %s backup %s --password-file %s' % (self.repoPath, dbPath, self.passwordFile)
                    result = ProcessUtilities.outputExecutioner(command)
                    logging.statusWriter(self.statusPath, result, 1)
                    snapShotid = result.split(' ')[-2]

                    newSnapshot = JobSnapshots(job=self.jobid, type='database:%s' % (items.dbName), snapshotid=snapShotid, destination=self.backupDestinations)
                    newSnapshot.save()

                elif self.backupDestinations[:4] == 'sftp':
                    remotePath = '/home/backup/%s' % (self.website.domain)
                    command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (
                    self.backupDestinations, remotePath, dbPath, self.passwordFile, self.repoPath)
                    result = ProcessUtilities.outputExecutioner(command)
                    logging.statusWriter(self.statusPath, result, 1)
                    snapShotid = result.split(' ')[-2]
                    newSnapshot = JobSnapshots(job=self.jobid, type='database:%s' % (items.dbName), snapshotid=snapShotid,
                                               destination=self.backupDestinations)
                    newSnapshot.save()

                try:
                    os.remove('/home/cyberpanel/%s' % (items.dbName))
                except:
                    pass
            return 1
        except BaseException, msg:
            logging.statusWriter(self.statusPath,'%s. [IncJobs.backupDatabases.269][5009]' % str(msg), 1)
            return 0

    def emailBackup(self):
        try:
            logging.statusWriter(self.statusPath, 'Backing up emails..', 1)

            backupPath = '/home/vmail/%s' % (self.website.domain)

            if self.backupDestinations == 'local':
                command = 'restic -r %s backup %s --password-file %s' % (
                    self.repoPath, backupPath, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
                snapShotid = result.split(' ')[-2]

                newSnapshot = JobSnapshots(job=self.jobid, type='email:%s' % (backupPath), snapshotid=snapShotid,
                                           destination=self.backupDestinations)
                newSnapshot.save()

            elif self.backupDestinations[:4] == 'sftp':
                remotePath = '/home/backup/%s' % (self.website.domain)
                command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (
                    self.backupDestinations, remotePath, backupPath, self.passwordFile, self.repoPath)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
                snapShotid = result.split(' ')[-2]
                newSnapshot = JobSnapshots(job=self.jobid, type='email:%s' % (backupPath), snapshotid=snapShotid,
                                           destination=self.backupDestinations)
                newSnapshot.save()


            logging.statusWriter(self.statusPath, 'Emails for %s backed to %s.' % (self.website.domain, self.backupDestinations), 1)
            return 1
        except BaseException, msg:
            logging.statusWriter(self.statusPath,'%s. [IncJobs.emailBackup.269][5009]' % str(msg), 1)
            return 0

    def initiateRepo(self):
        try:
            logging.statusWriter(self.statusPath, 'Will first initiate backup repo..', 1)

            if self.backupDestinations == 'local':
                command = 'restic init --repo %s --password-file %s' % (self.repoPath, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)

            elif self.backupDestinations[:4] == 'sftp':
                remotePath = '/home/backup/%s' % (self.website.domain)
                command = 'export PATH=${PATH}:/usr/bin && restic init --repo %s:%s --password-file %s' % (self.backupDestinations, remotePath, self.passwordFile)
                result = ProcessUtilities.outputExecutioner(command)
                logging.statusWriter(self.statusPath, result, 1)
            else:
                logging.statusWriter(self.statusPath, 'AWS implementation is currently pending. [5009]', 1)
                return 0

            logging.statusWriter(self.statusPath, 'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1)
            return 1
        except BaseException, msg:
            logging.statusWriter(self.statusPath,'%s. [IncJobs.initiateRepo.47][5009]' % str(msg), 1)
            return 0

    def createBackup(self):
        self.statusPath = self.extraArgs['tempPath']
        website = self.extraArgs['website']
        self.backupDestinations = self.extraArgs['backupDestinations']
        websiteData = self.extraArgs['websiteData']
        websiteEmails = self.extraArgs['websiteEmails']
        websiteSSLs = self.extraArgs['websiteSSLs']
        websiteDatabases = self.extraArgs['websiteDatabases']

        self.website = Websites.objects.get(domain=website)

        newJob = IncJob(website=self.website)
        newJob.save()

        self.jobid = newJob

        self.passwordFile = '/home/%s/%s' % (self.website.domain, self.website.domain)

        self.repoPath = '/home/%s/incbackup' % (self.website.domain)

        if not os.path.exists(self.passwordFile):
            password = randomPassword.generate_pass()
            command = 'echo "%s" > %s' % (password, self.passwordFile)
            ProcessUtilities.executioner(command, self.website.externalApp)


        if self.initiateRepo() == 0:
            return

        if self.prepareBackupMeta() == 0:
            return

        if websiteData:
            if self.backupData() == 0:
                return

        if websiteDatabases:
            if self.backupDatabases() == 0:
                return

        if websiteEmails:
            if self.emailBackup() == 0:
                return

        ## meta generated

        logging.statusWriter(self.statusPath, 'Taking backup of meta file, this file can be used to reconstruct at later point..', 1)

        metaPathNew = '/home/%s/meta.xml' % (self.website.domain)

        if self.backupDestinations == 'local':
            command = 'restic -r %s backup %s --password-file %s' % (
                self.repoPath, metaPathNew, self.passwordFile)
            result = ProcessUtilities.outputExecutioner(command)
            logging.statusWriter(self.statusPath, result, 1)
        elif self.backupDestinations[:4] == 'sftp':
            remotePath = '/home/backup/%s' % (self.website.domain)
            command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s' % (
                self.backupDestinations, remotePath, metaPathNew, self.passwordFile, self.repoPath)
            result = ProcessUtilities.outputExecutioner(command)
            logging.statusWriter(self.statusPath, result, 1)

        try:
            os.remove(metaPathNew)
        except:
            pass

        logging.statusWriter(self.statusPath, 'Completed', 1)