fixed backup and anonymization with better-sqlite3

src/routes/api/database.js

@@ -5,13 +5,13 @@ const log = require('../../services/log');
 const backupService = require('../../services/backup');
 const consistencyChecksService = require('../../services/consistency_checks');
 
-function anonymize() {
+async function anonymize() {
-    return backupService.anonymize();
+    return await backupService.anonymize();
 }
 
-function backupDatabase() {
+async function backupDatabase() {
     return {
-        backupFile: backupService.backupNow("now")
+        backupFile: await backupService.backupNow("now")
     };
 }
 

src/routes/routes.js

@@ -97,9 +97,14 @@ function route(method, path, middleware, routeHandler, resultHandler, transactio
             });
 
             if (resultHandler) {
+                if (result && result.then) {
+                    result.then(actualResult => resultHandler(req, res, actualResult))
+                }
+                else {
                     resultHandler(req, res, result);
                 }
             }
+        }
         catch (e) {
             log.error(`${method} ${path} threw exception: ` + e.stack);
 

src/services/backup.js

@@ -10,6 +10,7 @@ const syncMutexService = require('./sync_mutex');
 const attributeService = require('./attributes');
 const cls = require('./cls');
 const utils = require('./utils');
+const Database = require('better-sqlite3');
 
 function regularBackup() {
     periodBackup('lastDailyBackupDate', 'daily', 24 * 3600);
@@ -32,7 +33,7 @@ function periodBackup(optionName, fileName, periodInSeconds) {
 
 const COPY_ATTEMPT_COUNT = 50;
 
-function copyFile(backupFile) {
+async function copyFile(backupFile) {
     const sql = require('./sql');
 
     try {
@@ -40,79 +41,54 @@ function copyFile(backupFile) {
     } catch (e) {
     } // unlink throws exception if the file did not exist
 
-    let success = false;
+    await sql.dbConnection.backup(backupFile);
-    let attemptCount = 0
-
-    for (; attemptCount < COPY_ATTEMPT_COUNT && !success; attemptCount++) {
-        try {
-            sql.executeWithoutTransaction(`VACUUM INTO '${backupFile}'`);
-
-            success = true;
-        } catch (e) {
-            log.info(`Copy DB attempt ${attemptCount + 1} failed with "${e.message}", retrying...`);
-        }
-        // we re-try since VACUUM is very picky and it can't run if there's any other query currently running
-        // which is difficult to guarantee so we just re-try
-    }
-
-    return attemptCount !== COPY_ATTEMPT_COUNT;
 }
 
 async function backupNow(name) {
     // we don't want to backup DB in the middle of sync with potentially inconsistent DB state
-    return await syncMutexService.doExclusively(() => {
+    return await syncMutexService.doExclusively(async () => {
         const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
 
-        const success = copyFile(backupFile);
+        await copyFile(backupFile);
 
-        if (success) {
         log.info("Created backup at " + backupFile);
-        }
-        else {
-            log.error(`Creating backup ${backupFile} failed`);
-        }
 
         return backupFile;
     });
 }
 
-function anonymize() {
+async function anonymize() {
     if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
         fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
     }
 
     const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
 
-    const success = copyFile(anonymizedFile);
+    await copyFile(anonymizedFile);
 
-    if (!success) {
+    const db = new Database(anonymizedFile);
-        return { success: false };
-    }
 
-    const db = sqlite.open({
+    db.prepare("UPDATE api_tokens SET token = 'API token value'").run();
-        filename: anonymizedFile,
+    db.prepare("UPDATE notes SET title = 'title'").run();
-        driver: sqlite3.Database
+    db.prepare("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL").run();
-    });
+    db.prepare("UPDATE note_revisions SET title = 'title'").run();
-
+    db.prepare("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL").run();
-    db.run("UPDATE api_tokens SET token = 'API token value'");
-    db.run("UPDATE notes SET title = 'title'");
-    db.run("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL");
-    db.run("UPDATE note_revisions SET title = 'title'");
-    db.run("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL");
 
     // we want to delete all non-builtin attributes because they can contain sensitive names and values
     // on the other hand builtin/system attrs should not contain any sensitive info
-    const builtinAttrs = attributeService.getBuiltinAttributeNames().map(name => "'" + utils.sanitizeSql(name) + "'").join(', ');
+    const builtinAttrs = attributeService
+        .getBuiltinAttributeNames()
+        .map(name => "'" + utils.sanitizeSql(name) + "'").join(', ');
 
-    db.run(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`);
+    db.prepare(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`).run();
-    db.run(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`);
+    db.prepare(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`).run();
-    db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL");
+    db.prepare("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL").run();
-    db.run(`UPDATE options SET value = 'anonymized' WHERE name IN 
+    db.prepare(`UPDATE options SET value = 'anonymized' WHERE name IN 
                     ('documentId', 'documentSecret', 'encryptedDataKey', 
                      'passwordVerificationHash', 'passwordVerificationSalt', 
                      'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
-                      AND value != ''`);
+                      AND value != ''`).run();
-    db.run("VACUUM");
+    db.prepare("VACUUM").run();
 
     db.close();
 

src/services/import/tar.js

@@ -14,6 +14,7 @@ const commonmark = require('commonmark');
 const TaskContext = require('../task_context.js');
 const protectedSessionService = require('../protected_session');
 const mimeService = require("./mime");
+const sql = require("../sql");
 const treeService = require("../tree");
 
 /**
@@ -166,6 +167,7 @@ function importTar(taskContext, fileBuffer, importRootNote) {
             return;
         }
 
+        sql.transactional(() => {
             ({note} = noteService.createNewNote({
                 parentNoteId: parentNoteId,
                 title: noteTitle,
@@ -179,6 +181,7 @@ function importTar(taskContext, fileBuffer, importRootNote) {
             }));
 
             saveAttributes(note, noteMeta);
+        });
 
         if (!firstNote) {
             firstNote = note;

src/services/options.js

@@ -67,9 +67,7 @@ function getOptions() {
 }
 
 function getOptionsMap() {
-    const options = getOptions();
+    return require('./sql').getMap("SELECT name, value FROM options ORDER BY name");
-
-    return utils.toObject(options, opt => [opt.name, opt.value]);
 }
 
 module.exports = {

src/services/repository.js

@@ -131,7 +131,6 @@ function updateEntity(entity) {
                     eventService.emit(eventService.ENTITY_CREATED, eventPayload);
                 }
 
-                // it seems to be better to handle deletion and update separately
                 eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload);
             }
         }

src/services/source_id.js

@@ -2,7 +2,6 @@ const utils = require('./utils');
 const dateUtils = require('./date_utils');
 const log = require('./log');
 const sql = require('./sql');
-const sqlInit = require('./sql_init');
 const cls = require('./cls');
 
 function saveSourceId(sourceId) {
@@ -49,8 +48,10 @@ const currentSourceId = createSourceId();
 
 // very ugly
 setTimeout(() => {
+    const sqlInit = require('./sql_init');
+
     sqlInit.dbReady.then(cls.wrap(() => saveSourceId(currentSourceId)));
-}, 1000);
+}, 5000);
 
 function getCurrentSourceId() {
     return currentSourceId;

src/services/sql.js

@@ -253,6 +253,7 @@ function transactional(func) {
 }
 
 module.exports = {
+    dbConnection,
     insert,
     replace,
     getValue,