release 0.3.0

migrations/0063__image_table.sql

@@ -0,0 +1,11 @@
+CREATE TABLE images
+(
+  image_id TEXT PRIMARY KEY NOT NULL,
+  format TEXT NOT NULL,
+  checksum TEXT NOT NULL,
+  name TEXT NOT NULL,
+  data BLOB,
+  is_deleted INT NOT NULL DEFAULT 0,
+  date_modified TEXT NOT NULL,
+  date_created TEXT NOT NULL
+);

migrations/0064__add_note_id_to_image_table.sql

@@ -0,0 +1,16 @@
+DROP TABLE images;
+
+CREATE TABLE images
+(
+  image_id TEXT PRIMARY KEY NOT NULL,
+  note_id TEXT NOT NULL,
+  format TEXT NOT NULL,
+  checksum TEXT NOT NULL,
+  name TEXT NOT NULL,
+  data BLOB,
+  is_deleted INT NOT NULL DEFAULT 0,
+  date_modified TEXT NOT NULL,
+  date_created TEXT NOT NULL
+);
+
+CREATE INDEX images_note_id_index ON images (note_id);

migrations/0065__notes_image.sql

@@ -0,0 +1,27 @@
+DROP TABLE images;
+
+CREATE TABLE images
+(
+  image_id TEXT PRIMARY KEY NOT NULL,
+  format TEXT NOT NULL,
+  checksum TEXT NOT NULL,
+  name TEXT NOT NULL,
+  data BLOB,
+  is_deleted INT NOT NULL DEFAULT 0,
+  date_modified TEXT NOT NULL,
+  date_created TEXT NOT NULL
+);
+
+CREATE TABLE notes_image
+(
+  note_image_id TEXT PRIMARY KEY NOT NULL,
+  note_id TEXT NOT NULL,
+  image_id TEXT NOT NULL,
+  is_deleted INT NOT NULL DEFAULT 0,
+  date_modified TEXT NOT NULL,
+  date_created TEXT NOT NULL
+);
+
+CREATE INDEX notes_image_note_id_index ON notes_image (note_id);
+CREATE INDEX notes_image_image_id_index ON notes_image (image_id);
+CREATE INDEX notes_image_note_id_image_id_index ON notes_image (note_id, image_id);

package.json

@@ -1,7 +1,12 @@
 {
   "name": "trilium",
   "description": "Trilium Notes",
-  "version": "0.2.2",
+  "version": "0.3.0",
+  "license": "AGPL-3.0-only",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/zadam/trilium.git"
+  },
   "scripts": {
     "start": "node ./bin/www",
     "test-electron": "xo",
@@ -14,6 +19,7 @@
     "publish-forge": "electron-forge publish"
   },
   "dependencies": {
+    "async-mutex": "^0.1.3",
     "body-parser": "~1.18.2",
     "cookie-parser": "~1.4.3",
     "debug": "~3.1.0",
@@ -23,15 +29,24 @@
     "electron-debug": "^1.0.0",
     "electron-in-page-search": "^1.2.4",
     "express": "~4.16.2",
+    "express-promise-wrap": "^0.2.2",
     "express-session": "^1.15.6",
     "fs-extra": "^4.0.2",
     "helmet": "^3.9.0",
     "html": "^1.0.0",
+    "image-type": "^3.0.0",
+    "imagemin": "^5.3.1",
+    "imagemin-giflossy": "^5.1.10",
+    "imagemin-mozjpeg": "^7.0.0",
+    "imagemin-pngquant": "^5.0.1",
     "ini": "^1.3.4",
+    "jimp": "^0.2.28",
+    "multer": "^1.3.0",
     "rand-token": "^0.4.0",
     "request": "^2.83.0",
     "request-promise": "^4.2.2",
     "rimraf": "^2.6.2",
+    "sanitize-filename": "^1.6.1",
     "scrypt": "^6.0.3",
     "serve-favicon": "~2.4.5",
     "session-file-store": "^1.1.2",

public/javascripts/init.js

@@ -90,6 +90,8 @@ $(document).bind('keydown', "ctrl+shift+down", () => {
     return false;
 });
 
+$("#note-title").bind('keydown', 'return', () => $("#note-detail").focus());
+
 $(window).on('beforeunload', () => {
     // this makes sure that when user e.g. reloads the page or navigates away from the page, the note's content is saved
     // this sends the request asynchronously and doesn't wait for result

public/javascripts/messaging.js

@@ -29,7 +29,9 @@ const messaging = (function() {
 
             const syncData = message.data.filter(sync => sync.source_id !== glob.sourceId);
 
-            if (syncData.some(sync => sync.entity_name === 'notes_tree')) {
+            if (syncData.some(sync => sync.entity_name === 'notes_tree')
+                || syncData.some(sync => sync.entity_name === 'notes')) {
+
                 console.log(now(), "Reloading tree because of background changes");
 
                 noteTree.reload();
@@ -47,6 +49,9 @@ const messaging = (function() {
                 recentNotes.reload();
             }
 
+            // we don't detect image changes here since images themselves are immutable and references should be
+            // updated in note detail as well
+
             changesToPushCountEl.html(message.changesToPushCount);
         }
         else if (message.type === 'sync-hash-check-failed') {

public/javascripts/server.js

@@ -91,6 +91,7 @@ const server = (function() {
         get,
         post,
         put,
-        remove
+        remove,
+        getHeaders
     }
 })();

routes/api/anonymization.js

@@ -4,11 +4,12 @@ const express = require('express');
 const router = express.Router();
 const anonymization = require('../../services/anonymization');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('/anonymize', auth.checkApiAuth, async (req, res, next) => {
+router.post('/anonymize', auth.checkApiAuth, wrap(async (req, res, next) => {
     await anonymization.anonymize();
 
     res.send({});
-});
+}));
 
 module.exports = router;

routes/api/app_info.js

@@ -4,9 +4,10 @@ const express = require('express');
 const router = express.Router();
 const app_info = require('../../services/app_info');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkApiAuth, async (req, res, next) => {
+router.get('', auth.checkApiAuth, wrap(async (req, res, next) => {
     res.send(app_info);
-});
+}));
 
 module.exports = router;

routes/api/cleanup.js

@@ -7,8 +7,9 @@ const utils = require('../../services/utils');
 const sync_table = require('../../services/sync_table');
 const auth = require('../../services/auth');
 const log = require('../../services/log');
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('/cleanup-soft-deleted-items', auth.checkApiAuth, async (req, res, next) => {
+router.post('/cleanup-soft-deleted-items', auth.checkApiAuth, wrap(async (req, res, next) => {
     await sql.doInTransaction(async () => {
         const noteIdsToDelete = await sql.getFirstColumn("SELECT note_id FROM notes WHERE is_deleted = 1");
         const noteIdsSql = noteIdsToDelete
@@ -34,14 +35,14 @@ router.post('/cleanup-soft-deleted-items', auth.checkApiAuth, async (req, res, n
     });
 
     res.send({});
-});
+}));
 
-router.post('/vacuum-database', auth.checkApiAuth, async (req, res, next) => {
+router.post('/vacuum-database', auth.checkApiAuth, wrap(async (req, res, next) => {
     await sql.execute("VACUUM");
 
     log.info("Database has been vacuumed.");
 
     res.send({});
-});
+}));
 
 module.exports = router;

routes/api/event_log.js

@@ -4,14 +4,15 @@ const express = require('express');
 const router = express.Router();
 const sql = require('../../services/sql');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkApiAuth, async (req, res, next) => {
+router.get('', auth.checkApiAuth, wrap(async (req, res, next) => {
     await deleteOld();
 
     const result = await sql.getAll("SELECT * FROM event_log ORDER BY date_added DESC");
 
     res.send(result);
-});
+}));
 
 async function deleteOld() {
     const cutoffId = await sql.getFirstValue("SELECT id FROM event_log ORDER BY id DESC LIMIT 1000, 1");

routes/api/export.js

@@ -8,8 +8,9 @@ const sql = require('../../services/sql');
 const data_dir = require('../../services/data_dir');
 const html = require('html');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/:noteId/to/:directory', auth.checkApiAuth, async (req, res, next) => {
+router.get('/:noteId/to/:directory', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
     const directory = req.params.directory.replace(/[^0-9a-zA-Z_-]/gi, '');
 
@@ -30,7 +31,7 @@ router.get('/:noteId/to/:directory', auth.checkApiAuth, async (req, res, next) =
     await exportNote(noteTreeId, completeExportDir);
 
     res.send({});
-});
+}));
 
 async function exportNote(noteTreeId, dir) {
     const noteTree = await sql.getFirst("SELECT * FROM notes_tree WHERE note_tree_id = ?", [noteTreeId]);

routes/api/image.js

@@ -0,0 +1,142 @@
+"use strict";
+
+const express = require('express');
+const router = express.Router();
+const sql = require('../../services/sql');
+const auth = require('../../services/auth');
+const utils = require('../../services/utils');
+const sync_table = require('../../services/sync_table');
+const multer = require('multer')();
+const imagemin = require('imagemin');
+const imageminMozJpeg = require('imagemin-mozjpeg');
+const imageminPngQuant = require('imagemin-pngquant');
+const imageminGifLossy = require('imagemin-giflossy');
+const jimp = require('jimp');
+const imageType = require('image-type');
+const sanitizeFilename = require('sanitize-filename');
+const wrap = require('express-promise-wrap').wrap;
+
+router.get('/:imageId/:filename', auth.checkApiAuthOrElectron, wrap(async (req, res, next) => {
+    const image = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [req.params.imageId]);
+
+    if (!image) {
+        return res.status(404).send({});
+    }
+
+    res.set('Content-Type', 'image/' + image.format);
+
+    res.send(image.data);
+}));
+
+router.post('', auth.checkApiAuthOrElectron, multer.single('upload'), wrap(async (req, res, next) => {
+    const sourceId = req.headers.source_id;
+    const noteId = req.query.noteId;
+    const file = req.file;
+
+    const note = await sql.getFirst("SELECT * FROM notes WHERE note_id = ?", [noteId]);
+
+    if (!note) {
+        return res.status(404).send(`Note ${noteId} doesn't exist.`);
+    }
+
+    if (!["image/png", "image/jpeg", "image/gif"].includes(file.mimetype)) {
+        return res.status(400).send("Unknown image type: " + file.mimetype);
+    }
+
+    const now = utils.nowDate();
+
+    const resizedImage = await resize(file.buffer);
+    const optimizedImage = await optimize(resizedImage);
+
+    const imageFormat = imageType(optimizedImage);
+
+    const fileNameWithouExtension = file.originalname.replace(/\.[^/.]+$/, "");
+    const fileName = sanitizeFilename(fileNameWithouExtension + "." + imageFormat.ext);
+
+    const imageId = utils.newImageId();
+
+    await sql.doInTransaction(async () => {
+        await sql.insert("images", {
+            image_id: imageId,
+            format: imageFormat.ext,
+            name: fileName,
+            checksum: utils.hash(optimizedImage),
+            data: optimizedImage,
+            is_deleted: 0,
+            date_modified: now,
+            date_created: now
+        });
+
+        await sync_table.addImageSync(imageId, sourceId);
+
+        const noteImageId = utils.newNoteImageId();
+
+        await sql.insert("notes_image", {
+            note_image_id: noteImageId,
+            note_id: noteId,
+            image_id: imageId,
+            is_deleted: 0,
+            date_modified: now,
+            date_created: now
+        });
+
+        await sync_table.addNoteImageSync(noteImageId, sourceId);
+    });
+
+    res.send({
+        uploaded: true,
+        url: `/api/images/${imageId}/${fileName}`
+    });
+}));
+
+const MAX_SIZE = 1000;
+const MAX_BYTE_SIZE = 200000; // images should have under 100 KBs
+
+async function resize(buffer) {
+    const image = await jimp.read(buffer);
+
+    if (image.bitmap.width > image.bitmap.height && image.bitmap.width > MAX_SIZE) {
+        image.resize(MAX_SIZE, jimp.AUTO);
+    }
+    else if (image.bitmap.height > MAX_SIZE) {
+        image.resize(jimp.AUTO, MAX_SIZE);
+    }
+    else if (buffer.byteLength <= MAX_BYTE_SIZE) {
+        return buffer;
+    }
+
+    // we do resizing with max quality which will be trimmed during optimization step next
+    image.quality(100);
+
+    // when converting PNG to JPG we lose alpha channel, this is replaced by white to match Trilium white background
+    image.background(0xFFFFFFFF);
+
+    // getBuffer doesn't support promises so this workaround
+    return await new Promise((resolve, reject) => image.getBuffer(jimp.MIME_JPEG, (err, data) => {
+        if (err) {
+            reject(err);
+        }
+        else {
+            resolve(data);
+        }
+    }));
+}
+
+async function optimize(buffer) {
+    return await imagemin.buffer(buffer, {
+        plugins: [
+            imageminMozJpeg({
+                quality: 50
+            }),
+            imageminPngQuant({
+                quality: "0-70"
+            }),
+            imageminGifLossy({
+                lossy: 80,
+                optimize: '3' // needs to be string
+            })
+        ]
+    });
+}
+
+module.exports = router;

routes/api/import.js

@@ -8,8 +8,9 @@ const data_dir = require('../../services/data_dir');
 const utils = require('../../services/utils');
 const sync_table = require('../../services/sync_table');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/:directory/to/:parentNoteId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/:directory/to/:parentNoteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const directory = req.params.directory.replace(/[^0-9a-zA-Z_-]/gi, '');
     const parentNoteId = req.params.parentNoteId;
 
@@ -18,7 +19,7 @@ router.get('/:directory/to/:parentNoteId', auth.checkApiAuth, async (req, res, n
     await sql.doInTransaction(async () => await importNotes(dir, parentNoteId));
 
     res.send({});
-});
+}));
 
 async function importNotes(dir, parentNoteId) {
     const parent = await sql.getFirst("SELECT * FROM notes WHERE note_id = ?", [parentNoteId]);

routes/api/login.js

@@ -9,8 +9,9 @@ const auth = require('../../services/auth');
 const password_encryption = require('../../services/password_encryption');
 const protected_session = require('../../services/protected_session');
 const app_info = require('../../services/app_info');
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('/sync', async (req, res, next) => {
+router.post('/sync', wrap(async (req, res, next) => {
     const timestampStr = req.body.timestamp;
 
     const timestamp = utils.parseDate(timestampStr);
@@ -44,10 +45,10 @@ router.post('/sync', async (req, res, next) => {
     res.send({
         sourceId: source_id.getCurrentSourceId()
     });
-});
+}));
 
 // this is for entering protected mode so user has to be already logged-in (that's the reason we don't require username)
-router.post('/protected', auth.checkApiAuth, async (req, res, next) => {
+router.post('/protected', auth.checkApiAuth, wrap(async (req, res, next) => {
     const password = req.body.password;
 
     if (!await password_encryption.verifyPassword(password)) {
@@ -67,6 +68,6 @@ router.post('/protected', auth.checkApiAuth, async (req, res, next) => {
         success: true,
         protectedSessionId: protectedSessionId
     });
-});
+}));
 
 module.exports = router;

routes/api/migration.js

@@ -6,20 +6,21 @@ const auth = require('../../services/auth');
 const options = require('../../services/options');
 const migration = require('../../services/migration');
 const app_info = require('../../services/app_info');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkApiAuthForMigrationPage, async (req, res, next) => {
+router.get('', auth.checkApiAuthForMigrationPage, wrap(async (req, res, next) => {
     res.send({
         db_version: parseInt(await options.getOption('db_version')),
         app_db_version: app_info.db_version
     });
-});
+}));
 
-router.post('', auth.checkApiAuthForMigrationPage, async (req, res, next) => {
+router.post('', auth.checkApiAuthForMigrationPage, wrap(async (req, res, next) => {
     const migrations = await migration.migrate();
 
     res.send({
         migrations: migrations
     });
-});
+}));
 
 module.exports = router;

routes/api/note_history.js

@@ -7,8 +7,9 @@ const auth = require('../../services/auth');
 const data_encryption = require('../../services/data_encryption');
 const protected_session = require('../../services/protected_session');
 const sync_table = require('../../services/sync_table');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/:noteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
     const history = await sql.getAll("SELECT * FROM notes_history WHERE note_id = ? order by date_modified_to desc", [noteId]);
 
@@ -22,9 +23,9 @@ router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => {
     }
 
     res.send(history);
-});
+}));
 
-router.put('', auth.checkApiAuth, async (req, res, next) => {
+router.put('', auth.checkApiAuth, wrap(async (req, res, next) => {
     const sourceId = req.headers.source_id;
 
     await sql.doInTransaction(async () => {
@@ -34,6 +35,6 @@ router.put('', auth.checkApiAuth, async (req, res, next) => {
     });
 
     res.send();
-});
+}));
 
 module.exports = router;

routes/api/notes.js

@@ -8,8 +8,9 @@ const notes = require('../../services/notes');
 const log = require('../../services/log');
 const protected_session = require('../../services/protected_session');
 const data_encryption = require('../../services/data_encryption');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/:noteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
 
     const detail = await sql.getFirst("SELECT * FROM notes WHERE note_id = ?", [noteId]);
@@ -30,9 +31,9 @@ router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => {
     res.send({
         detail: detail
     });
-});
+}));
 
-router.post('/:parentNoteId/children', auth.checkApiAuth, async (req, res, next) => {
+router.post('/:parentNoteId/children', auth.checkApiAuth, wrap(async (req, res, next) => {
     const sourceId = req.headers.source_id;
     const parentNoteId = req.params.parentNoteId;
     const note = req.body;
@@ -43,9 +44,9 @@ router.post('/:parentNoteId/children', auth.checkApiAuth, async (req, res, next)
         'note_id': noteId,
         'note_tree_id': noteTreeId
     });
-});
+}));
 
-router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const note = req.body;
     const noteId = req.params.noteId;
     const sourceId = req.headers.source_id;
@@ -54,17 +55,17 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => {
     await notes.updateNote(noteId, note, dataKey, sourceId);
 
     res.send({});
-});
+}));
 
-router.delete('/:noteTreeId', auth.checkApiAuth, async (req, res, next) => {
+router.delete('/:noteTreeId', auth.checkApiAuth, wrap(async (req, res, next) => {
     await sql.doInTransaction(async () => {
         await notes.deleteNote(req.params.noteTreeId, req.headers.source_id);
     });
 
     res.send({});
-});
+}));
 
-router.get('/', auth.checkApiAuth, async (req, res, next) => {
+router.get('/', auth.checkApiAuth, wrap(async (req, res, next) => {
     const search = '%' + req.query.search + '%';
 
     const result = await sql.getAll("SELECT note_id FROM notes WHERE note_title LIKE ? OR note_text LIKE ?", [search, search]);
@@ -76,6 +77,6 @@ router.get('/', auth.checkApiAuth, async (req, res, next) => {
     }
 
     res.send(noteIdList);
-});
+}));
 
 module.exports = router;

routes/api/notes_move.js

@@ -6,13 +6,14 @@ const sql = require('../../services/sql');
 const auth = require('../../services/auth');
 const utils = require('../../services/utils');
 const sync_table = require('../../services/sync_table');
+const wrap = require('express-promise-wrap').wrap;
 
 /**
  * Code in this file deals with moving and cloning note tree rows. Relationship between note and parent note is unique
  * for not deleted note trees. There may be multiple deleted note-parent note relationships.
  */
 
-router.put('/:noteTreeId/move-to/:parentNoteId', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/move-to/:parentNoteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
     const parentNoteId = req.params.parentNoteId;
     const sourceId = req.headers.source_id;
@@ -36,9 +37,9 @@ router.put('/:noteTreeId/move-to/:parentNoteId', auth.checkApiAuth, async (req,
     });
 
     res.send({ success: true });
-});
+}));
 
-router.put('/:noteTreeId/move-before/:beforeNoteTreeId', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/move-before/:beforeNoteTreeId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
     const beforeNoteTreeId = req.params.beforeNoteTreeId;
     const sourceId = req.headers.source_id;
@@ -67,9 +68,9 @@ router.put('/:noteTreeId/move-before/:beforeNoteTreeId', auth.checkApiAuth, asyn
     });
 
     res.send({ success: true });
-});
+}));
 
-router.put('/:noteTreeId/move-after/:afterNoteTreeId', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/move-after/:afterNoteTreeId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
     const afterNoteTreeId = req.params.afterNoteTreeId;
     const sourceId = req.headers.source_id;
@@ -96,9 +97,9 @@ router.put('/:noteTreeId/move-after/:afterNoteTreeId', auth.checkApiAuth, async
     });
 
     res.send({ success: true });
-});
+}));
 
-router.put('/:childNoteId/clone-to/:parentNoteId', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:childNoteId/clone-to/:parentNoteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const parentNoteId = req.params.parentNoteId;
     const childNoteId = req.params.childNoteId;
     const prefix = req.body.prefix;
@@ -131,9 +132,9 @@ router.put('/:childNoteId/clone-to/:parentNoteId', auth.checkApiAuth, async (req
     });
 
     res.send({ success: true });
-});
+}));
 
-router.put('/:noteId/clone-after/:afterNoteTreeId', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteId/clone-after/:afterNoteTreeId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
     const afterNoteTreeId = req.params.afterNoteTreeId;
     const sourceId = req.headers.source_id;
@@ -168,7 +169,7 @@ router.put('/:noteId/clone-after/:afterNoteTreeId', auth.checkApiAuth, async (re
     });
 
     res.send({ success: true });
-});
+}));
 
 async function loadSubTreeNoteIds(parentNoteId, subTreeNoteIds) {
     subTreeNoteIds.push(parentNoteId);
@@ -190,7 +191,7 @@ async function validateParentChild(res, parentNoteId, childNoteId, noteTreeId =
     if (existing && (noteTreeId === null || existing.note_tree_id !== noteTreeId)) {
         res.send({
             success: false,
-            message: 'This note already exists in target parent note.'
+            message: 'This note already exists in the target.'
         });
 
         return false;
@@ -246,7 +247,7 @@ async function checkTreeCycle(parentNoteId, childNoteId) {
     return await checkTreeCycleInner(parentNoteId);
 }
 
-router.put('/:noteTreeId/expanded/:expanded', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/expanded/:expanded', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
     const expanded = req.params.expanded;
 
@@ -257,6 +258,6 @@ router.put('/:noteTreeId/expanded/:expanded', auth.checkApiAuth, async (req, res
     });
 
     res.send({});
-});
+}));
 
 module.exports = router;

routes/api/password.js

@@ -5,11 +5,12 @@ const router = express.Router();
 const sql = require('../../services/sql');
 const changePassword = require('../../services/change_password');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('/change', auth.checkApiAuth, async (req, res, next) => {
+router.post('/change', auth.checkApiAuth, wrap(async (req, res, next) => {
     const result = await changePassword.changePassword(req.body['current_password'], req.body['new_password'], req);
 
     res.send(result);
-});
+}));
 
 module.exports = router;

routes/api/recent_changes.js

@@ -4,8 +4,9 @@ const express = require('express');
 const router = express.Router();
 const sql = require('../../services/sql');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/', auth.checkApiAuth, async (req, res, next) => {
+router.get('/', auth.checkApiAuth, wrap(async (req, res, next) => {
     const recentChanges = await sql.getAll(
         `SELECT 
             notes.is_deleted AS current_is_deleted,
@@ -19,6 +20,6 @@ router.get('/', auth.checkApiAuth, async (req, res, next) => {
         LIMIT 1000`);
 
     res.send(recentChanges);
-});
+}));
 
 module.exports = router;

routes/api/recent_notes.js

@@ -7,12 +7,13 @@ const auth = require('../../services/auth');
 const utils = require('../../services/utils');
 const sync_table = require('../../services/sync_table');
 const options = require('../../services/options');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkApiAuth, async (req, res, next) => {
+router.get('', auth.checkApiAuth, wrap(async (req, res, next) => {
     res.send(await getRecentNotes());
-});
+}));
 
-router.put('/:noteTreeId/:notePath', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/:notePath', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
     const notePath = req.params.notePath;
     const sourceId = req.headers.source_id;
@@ -31,7 +32,7 @@ router.put('/:noteTreeId/:notePath', auth.checkApiAuth, async (req, res, next) =
     });
 
     res.send(await getRecentNotes());
-});
+}));
 
 async function getRecentNotes() {
     return await sql.getAll(`

routes/api/settings.js

@@ -5,24 +5,25 @@ const router = express.Router();
 const sql = require('../../services/sql');
 const options = require('../../services/options');
 const auth = require('../../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
 // options allowed to be updated directly in settings dialog
 const ALLOWED_OPTIONS = ['protected_session_timeout', 'history_snapshot_time_interval'];
 
-router.get('/all', auth.checkApiAuth, async (req, res, next) => {
+router.get('/all', auth.checkApiAuth, wrap(async (req, res, next) => {
     const settings = await sql.getMap("SELECT opt_name, opt_value FROM options");
 
     res.send(settings);
-});
+}));
 
-router.get('/', auth.checkApiAuth, async (req, res, next) => {
+router.get('/', auth.checkApiAuth, wrap(async (req, res, next) => {
     const settings = await sql.getMap("SELECT opt_name, opt_value FROM options WHERE opt_name IN ("
         + ALLOWED_OPTIONS.map(x => '?').join(",") + ")", ALLOWED_OPTIONS);
 
     res.send(settings);
-});
+}));
 
-router.post('/', auth.checkApiAuth, async (req, res, next) => {
+router.post('/', auth.checkApiAuth, wrap(async (req, res, next) => {
     const body = req.body;
     const sourceId = req.headers.source_id;
 
@@ -38,6 +39,6 @@ router.post('/', auth.checkApiAuth, async (req, res, next) => {
     else {
         res.send("not allowed option to set");
     }
-});
+}));
 
 module.exports = router;

routes/api/setup.js

@@ -8,8 +8,9 @@ const sql = require('../../services/sql');
 const utils = require('../../services/utils');
 const my_scrypt = require('../../services/my_scrypt');
 const password_encryption = require('../../services/password_encryption');
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('', auth.checkAppNotInitialized, async (req, res, next) => {
+router.post('', auth.checkAppNotInitialized, wrap(async (req, res, next) => {
     const { username, password } = req.body;
 
     await sql.doInTransaction(async () => {
@@ -27,6 +28,6 @@ router.post('', auth.checkAppNotInitialized, async (req, res, next) => {
     sql.setDbReadyAsResolved();
 
     res.send({});
-});
+}));
 
 module.exports = router;

routes/api/sql.js

@@ -4,8 +4,9 @@ const express = require('express');
 const router = express.Router();
 const auth = require('../../services/auth');
 const sql = require('../../services/sql');
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('/execute', auth.checkApiAuth, async (req, res, next) => {
+router.post('/execute', auth.checkApiAuth, wrap(async (req, res, next) => {
     const query = req.body.query;
 
     try {
@@ -20,6 +21,6 @@ router.post('/execute', auth.checkApiAuth, async (req, res, next) => {
             error: e.message
         });
     }
-});
+}));
 
 module.exports = router;

routes/api/sync.js

@@ -10,19 +10,20 @@ const sql = require('../../services/sql');
 const options = require('../../services/options');
 const content_hash = require('../../services/content_hash');
 const log = require('../../services/log');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/check', auth.checkApiAuth, async (req, res, next) => {
+router.get('/check', auth.checkApiAuth, wrap(async (req, res, next) => {
     res.send({
         'hashes': await content_hash.getHashes(),
         'max_sync_id': await sql.getFirstValue('SELECT MAX(id) FROM sync')
     });
-});
+}));
 
-router.post('/now', auth.checkApiAuth, async (req, res, next) => {
+router.post('/now', auth.checkApiAuth, wrap(async (req, res, next) => {
     res.send(await sync.sync());
-});
+}));
 
-router.post('/fill-sync-rows', auth.checkApiAuth, async (req, res, next) => {
+router.post('/fill-sync-rows', auth.checkApiAuth, wrap(async (req, res, next) => {
     await sql.doInTransaction(async () => {
         await sync_table.fillAllSyncRows();
     });
@@ -30,9 +31,9 @@ router.post('/fill-sync-rows', auth.checkApiAuth, async (req, res, next) => {
     log.info("Sync rows have been filled.");
 
     res.send({});
-});
+}));
 
-router.post('/force-full-sync', auth.checkApiAuth, async (req, res, next) => {
+router.post('/force-full-sync', auth.checkApiAuth, wrap(async (req, res, next) => {
     await sql.doInTransaction(async () => {
         await options.setOption('last_synced_pull', 0);
         await options.setOption('last_synced_push', 0);
@@ -44,9 +45,9 @@ router.post('/force-full-sync', auth.checkApiAuth, async (req, res, next) => {
     sync.sync();
 
     res.send({});
-});
+}));
 
-router.post('/force-note-sync/:noteId', auth.checkApiAuth, async (req, res, next) => {
+router.post('/force-note-sync/:noteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
 
     await sql.doInTransaction(async () => {
@@ -68,35 +69,35 @@ router.post('/force-note-sync/:noteId', auth.checkApiAuth, async (req, res, next
     sync.sync();
 
     res.send({});
-});
+}));
 
-router.get('/changed', auth.checkApiAuth, async (req, res, next) => {
+router.get('/changed', auth.checkApiAuth, wrap(async (req, res, next) => {
     const lastSyncId = parseInt(req.query.lastSyncId);
 
     res.send(await sql.getAll("SELECT * FROM sync WHERE id > ?", [lastSyncId]));
-});
+}));
 
-router.get('/notes/:noteId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/notes/:noteId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
 
     res.send({
         entity: await sql.getFirst("SELECT * FROM notes WHERE note_id = ?", [noteId])
     });
-});
+}));
 
-router.get('/notes_tree/:noteTreeId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/notes_tree/:noteTreeId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
 
     res.send(await sql.getFirst("SELECT * FROM notes_tree WHERE note_tree_id = ?", [noteTreeId]));
-});
+}));
 
-router.get('/notes_history/:noteHistoryId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/notes_history/:noteHistoryId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteHistoryId = req.params.noteHistoryId;
 
     res.send(await sql.getFirst("SELECT * FROM notes_history WHERE note_history_id = ?", [noteHistoryId]));
-});
+}));
 
-router.get('/options/:optName', auth.checkApiAuth, async (req, res, next) => {
+router.get('/options/:optName', auth.checkApiAuth, wrap(async (req, res, next) => {
     const optName = req.params.optName;
 
     if (!options.SYNCED_OPTIONS.includes(optName)) {
@@ -105,57 +106,86 @@ router.get('/options/:optName', auth.checkApiAuth, async (req, res, next) => {
     else {
         res.send(await sql.getFirst("SELECT * FROM options WHERE opt_name = ?", [optName]));
     }
-});
+}));
 
-router.get('/notes_reordering/:noteTreeParentId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/notes_reordering/:noteTreeParentId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeParentId = req.params.noteTreeParentId;
 
     res.send({
         parent_note_id: noteTreeParentId,
         ordering: await sql.getMap("SELECT note_tree_id, note_position FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0", [noteTreeParentId])
     });
-});
+}));
 
-router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, async (req, res, next) => {
+router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
 
     res.send(await sql.getFirst("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId]));
-});
+}));
 
-router.put('/notes', auth.checkApiAuth, async (req, res, next) => {
+router.get('/images/:imageId', auth.checkApiAuth, wrap(async (req, res, next) => {
+    const imageId = req.params.imageId;
+    const entity = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [imageId]);
+
+    if (entity && entity.data !== null) {
+        entity.data = entity.data.toString('base64');
+    }
+
+    res.send(entity);
+}));
+
+router.get('/notes_image/:noteImageId', auth.checkApiAuth, wrap(async (req, res, next) => {
+    const noteImageId = req.params.noteImageId;
+
+    res.send(await sql.getFirst("SELECT * FROM notes_image WHERE note_image_id = ?", [noteImageId]));
+}));
+
+router.put('/notes', auth.checkApiAuth, wrap(async (req, res, next) => {
     await syncUpdate.updateNote(req.body.entity, req.body.sourceId);
 
     res.send({});
-});
+}));
 
-router.put('/notes_tree', auth.checkApiAuth, async (req, res, next) => {
+router.put('/notes_tree', auth.checkApiAuth, wrap(async (req, res, next) => {
     await syncUpdate.updateNoteTree(req.body.entity, req.body.sourceId);
 
     res.send({});
-});
+}));
 
-router.put('/notes_history', auth.checkApiAuth, async (req, res, next) => {
+router.put('/notes_history', auth.checkApiAuth, wrap(async (req, res, next) => {
     await syncUpdate.updateNoteHistory(req.body.entity, req.body.sourceId);
 
     res.send({});
-});
+}));
 
-router.put('/notes_reordering', auth.checkApiAuth, async (req, res, next) => {
+router.put('/notes_reordering', auth.checkApiAuth, wrap(async (req, res, next) => {
     await syncUpdate.updateNoteReordering(req.body.entity, req.body.sourceId);
 
     res.send({});
-});
+}));
 
-router.put('/options', auth.checkApiAuth, async (req, res, next) => {
+router.put('/options', auth.checkApiAuth, wrap(async (req, res, next) => {
     await syncUpdate.updateOptions(req.body.entity, req.body.sourceId);
 
     res.send({});
-});
+}));
 
-router.put('/recent_notes', auth.checkApiAuth, async (req, res, next) => {
+router.put('/recent_notes', auth.checkApiAuth, wrap(async (req, res, next) => {
     await syncUpdate.updateRecentNotes(req.body.entity, req.body.sourceId);
 
     res.send({});
-});
+}));
+
+router.put('/images', auth.checkApiAuth, wrap(async (req, res, next) => {
+    await syncUpdate.updateImage(req.body.entity, req.body.sourceId);
+
+    res.send({});
+}));
+
+router.put('/notes_image', auth.checkApiAuth, wrap(async (req, res, next) => {
+    await syncUpdate.updateNoteImage(req.body.entity, req.body.sourceId);
+
+    res.send({});
+}));
 
 module.exports = router;

routes/api/tree.js

@@ -10,8 +10,9 @@ const protected_session = require('../../services/protected_session');
 const data_encryption = require('../../services/data_encryption');
 const notes = require('../../services/notes');
 const sync_table = require('../../services/sync_table');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('/', auth.checkApiAuth, async (req, res, next) => {
+router.get('/', auth.checkApiAuth, wrap(async (req, res, next) => {
     const notes = await sql.getAll(`
       SELECT 
         notes_tree.*, 
@@ -39,9 +40,9 @@ router.get('/', auth.checkApiAuth, async (req, res, next) => {
         notes: notes,
         start_note_path: await options.getOption('start_note_path')
     });
-});
+}));
 
-router.put('/:noteId/protect-sub-tree/:isProtected', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteId/protect-sub-tree/:isProtected', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteId = req.params.noteId;
     const isProtected = !!parseInt(req.params.isProtected);
     const dataKey = protected_session.getDataKey(req);
@@ -52,9 +53,9 @@ router.put('/:noteId/protect-sub-tree/:isProtected', auth.checkApiAuth, async (r
     });
 
     res.send({});
-});
+}));
 
-router.put('/:noteTreeId/set-prefix', auth.checkApiAuth, async (req, res, next) => {
+router.put('/:noteTreeId/set-prefix', auth.checkApiAuth, wrap(async (req, res, next) => {
     const noteTreeId = req.params.noteTreeId;
     const sourceId = req.headers.source_id;
     const prefix = utils.isEmptyOrWhitespace(req.body.prefix) ? null : req.body.prefix;
@@ -66,6 +67,6 @@ router.put('/:noteTreeId/set-prefix', auth.checkApiAuth, async (req, res, next)
     });
 
     res.send({});
-});
+}));
 
 module.exports = router;

routes/index.js

@@ -5,12 +5,13 @@ const router = express.Router();
 const auth = require('../services/auth');
 const source_id = require('../services/source_id');
 const sql = require('../services/sql');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkAuth, async (req, res, next) => {
+router.get('', auth.checkAuth, wrap(async (req, res, next) => {
     res.render('index', {
         sourceId: await source_id.generateSourceId(),
         maxSyncIdAtLoad: await sql.getFirstValue("SELECT MAX(id) FROM sync")
     });
-});
+}));
 
 module.exports = router;

routes/login.js

@@ -5,12 +5,13 @@ const router = express.Router();
 const utils = require('../services/utils');
 const options = require('../services/options');
 const my_scrypt = require('../services/my_scrypt');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', (req, res, next) => {
+router.get('', wrap(async (req, res, next) => {
     res.render('login', { 'failedAuth': false });
-});
+}));
 
-router.post('', async (req, res, next) => {
+router.post('', wrap(async (req, res, next) => {
     const userName = await options.getOption('username');
 
     const guessedPassword = req.body.password;
@@ -32,7 +33,7 @@ router.post('', async (req, res, next) => {
     else {
         res.render('login', {'failedAuth': true});
     }
-});
+}));
 
 
 async function verifyPassword(guessed_password) {

routes/logout.js

@@ -2,14 +2,15 @@
 
 const express = require('express');
 const router = express.Router();
+const wrap = require('express-promise-wrap').wrap;
 
-router.post('', async (req, res, next) => {
+router.post('', wrap(async (req, res, next) => {
     req.session.regenerate(() => {
         req.session.loggedIn = false;
 
         res.redirect('/');
     });
 
-});
+}));
 
 module.exports = router;

routes/migration.js

@@ -3,9 +3,10 @@
 const express = require('express');
 const router = express.Router();
 const auth = require('../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkAuthForMigrationPage, (req, res, next) => {
+router.get('', auth.checkAuthForMigrationPage, wrap(async (req, res, next) => {
     res.render('migration', {});
-});
+}));
 
 module.exports = router;

routes/routes.js

@@ -24,6 +24,7 @@ const setupApiRoute = require('./api/setup');
 const sqlRoute = require('./api/sql');
 const anonymizationRoute = require('./api/anonymization');
 const cleanupRoute = require('./api/cleanup');
+const imageRoute = require('./api/image');
 
 function register(app) {
     app.use('/', indexRoute);
@@ -51,6 +52,7 @@ function register(app) {
     app.use('/api/sql', sqlRoute);
     app.use('/api/anonymization', anonymizationRoute);
     app.use('/api/cleanup', cleanupRoute);
+    app.use('/api/images', imageRoute);
 }
 
 module.exports = {

routes/setup.js

@@ -3,9 +3,10 @@
 const express = require('express');
 const router = express.Router();
 const auth = require('../services/auth');
+const wrap = require('express-promise-wrap').wrap;
 
-router.get('', auth.checkAppNotInitialized, (req, res, next) => {
+router.get('', auth.checkAppNotInitialized, wrap(async (req, res, next) => {
     res.render('setup', {});
-});
+}));
 
 module.exports = router;

services/app_info.js

@@ -3,7 +3,7 @@
 const build = require('./build');
 const packageJson = require('../package');
 
-const APP_DB_VERSION = 62;
+const APP_DB_VERSION = 65;
 
 module.exports = {
     app_version: packageJson.version,

services/auth.js

@@ -28,6 +28,20 @@ async function checkAuthForMigrationPage(req, res, next) {
     }
 }
 
+// for electron things which need network stuff
+// currently we're doing that for file upload because handling form data seems to be difficult
+async function checkApiAuthOrElectron(req, res, next) {
+    if (!req.session.loggedIn && !utils.isElectron()) {
+        res.status(401).send("Not authorized");
+    }
+    else if (await sql.isDbUpToDate()) {
+        next();
+    }
+    else {
+        res.status(409).send("Mismatched app versions"); // need better response than that
+    }
+}
+
 async function checkApiAuth(req, res, next) {
     if (!req.session.loggedIn) {
         res.status(401).send("Not authorized");
@@ -63,5 +77,6 @@ module.exports = {
     checkAuthForMigrationPage,
     checkApiAuth,
     checkApiAuthForMigrationPage,
-    checkAppNotInitialized
+    checkAppNotInitialized,
+    checkApiAuthOrElectron
 };

services/backup.js

@@ -6,6 +6,7 @@ const fs = require('fs-extra');
 const dataDir = require('./data_dir');
 const log = require('./log');
 const sql = require('./sql');
+const sync_mutex = require('./sync_mutex');
 
 async function regularBackup() {
     const now = new Date();
@@ -21,17 +22,25 @@ async function regularBackup() {
 }
 
 async function backupNow() {
-    const now = utils.nowDate();
+    // we don't want to backup DB in the middle of sync with potentially inconsistent DB state
+    const releaseMutex = await sync_mutex.acquire();
 
-    const backupFile = dataDir.BACKUP_DIR + "/" + "backup-" + utils.getDateTimeForFile() + ".db";
+    try {
+        const now = utils.nowDate();
 
-    fs.copySync(dataDir.DOCUMENT_PATH, backupFile);
+        const backupFile = dataDir.BACKUP_DIR + "/" + "backup-" + utils.getDateTimeForFile() + ".db";
 
-    log.info("Created backup at " + backupFile);
+        fs.copySync(dataDir.DOCUMENT_PATH, backupFile);
 
-    await sql.doInTransaction(async () => {
+        log.info("Created backup at " + backupFile);
-        await options.setOption('last_backup_date', now);
+
-    });
+        await sql.doInTransaction(async () => {
+            await options.setOption('last_backup_date', now);
+        });
+    }
+    finally {
+        releaseMutex();
+    }
 }
 
 async function cleanupOldBackups() {

services/build.js

@@ -1 +1 @@
-module.exports = { build_date:"2018-01-03T23:05:00-05:00", build_revision: "f2aaf8b0a3b761fb6a1ec79e7c6b95e3eb9e4db0" };
+module.exports = { build_date:"2018-01-07T10:04:46-05:00", build_revision: "488e657cc43fab879662a6da23b5a69dd7591b06" };

services/consistency_checks.js

@@ -3,8 +3,12 @@
 const sql = require('./sql');
 const log = require('./log');
 const messaging = require('./messaging');
+const sync_mutex = require('./sync_mutex');
+const utils = require('./utils');
 
 async function runCheck(query, errorText, errorList) {
+    utils.assertArguments(query, errorText, errorList);
+
     const result = await sql.getFirstColumn(query);
 
     if (result.length > 0) {
@@ -80,11 +84,9 @@ async function runSyncRowChecks(table, key, errorList) {
         `Missing ${table} records for existing sync rows`, errorList);
 }
 
-async function runChecks() {
+async function runAllChecks() {
     const errorList = [];
 
-    const startTime = new Date();
-
     await runCheck(`
           SELECT 
             note_id 
@@ -139,7 +141,7 @@ async function runChecks() {
           WHERE
             (SELECT COUNT(*) FROM notes_tree WHERE notes.note_id = notes_tree.note_id AND notes_tree.is_deleted = 0) = 0
             AND notes.is_deleted = 0
-    `, );
+    `, 'No undeleted note trees for note IDs', errorList);
 
     await runCheck(`
           SELECT 
@@ -175,10 +177,22 @@ async function runChecks() {
             COUNT(*) > 1`,
         "Duplicate undeleted parent note <-> note relationship - parent note ID > note ID", errorList);
 
+    await runCheck(`
+          SELECT 
+            images.image_id
+          FROM 
+            images
+            LEFT JOIN notes_image ON notes_image.image_id = images.image_id
+          WHERE 
+            notes_image.note_image_id IS NULL`,
+        "Image with no note relation", errorList);
+
     await runSyncRowChecks("notes", "note_id", errorList);
     await runSyncRowChecks("notes_history", "note_history_id", errorList);
     await runSyncRowChecks("notes_tree", "note_tree_id", errorList);
     await runSyncRowChecks("recent_notes", "note_tree_id", errorList);
+    await runSyncRowChecks("images", "image_id", errorList);
+    await runSyncRowChecks("notes_image", "note_image_id", errorList);
 
     if (errorList.length === 0) {
         // we run this only if basic checks passed since this assumes basic data consistency
@@ -186,7 +200,24 @@ async function runChecks() {
         await checkTreeCycles(errorList);
     }
 
-    const elapsedTimeMs = new Date().getTime() - startTime.getTime();
+    return errorList;
+}
+
+async function runChecks() {
+    let errorList;
+    let elapsedTimeMs;
+    const releaseMutex = await sync_mutex.acquire();
+
+    try {
+        const startTime = new Date();
+
+        errorList = await runAllChecks();
+
+        elapsedTimeMs = new Date().getTime() - startTime.getTime();
+    }
+    finally {
+        releaseMutex();
+    }
 
     if (errorList.length > 0) {
         log.info(`Consistency checks failed (took ${elapsedTimeMs}ms) with these errors: ` + JSON.stringify(errorList));

services/content_hash.js

@@ -19,51 +19,70 @@ async function getHashes() {
     const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(',');
 
     const hashes = {
-        notes: getHash(await sql.getAll(`SELECT
+        notes: getHash(await sql.getAll(`
-                                                  note_id,
+            SELECT
-                                                  note_title,
+              note_id,
-                                                  note_text,
+              note_title,
-                                                  date_modified,
+              note_text,
-                                                  is_protected,
+              date_modified,
-                                                  is_deleted
+              is_protected,
-                                                FROM notes
+              is_deleted
-                                                ORDER BY note_id`)),
+            FROM notes
+            ORDER BY note_id`)),
 
-        notes_tree: getHash(await sql.getAll(`SELECT
+        notes_tree: getHash(await sql.getAll(`
-                                                       note_tree_id,
+            SELECT
-                                                       note_id,
+               note_tree_id,
-                                                       parent_note_id,
+               note_id,
-                                                       note_position,
+               parent_note_id,
-                                                       date_modified,
+               note_position,
-                                                       is_deleted,
+               date_modified,
-                                                       prefix
+               is_deleted,
-                                                     FROM notes_tree
+               prefix
-                                                     ORDER BY note_tree_id`)),
+             FROM notes_tree
+             ORDER BY note_tree_id`)),
 
-        notes_history: getHash(await sql.getAll(`SELECT
+        notes_history: getHash(await sql.getAll(`
-                                                          note_history_id,
+            SELECT
-                                                          note_id,
+              note_history_id,
-                                                          note_title,
+              note_id,
-                                                          note_text,
+              note_title,
-                                                          date_modified_from,
+              note_text,
-                                                          date_modified_to
+              date_modified_from,
-                                                        FROM notes_history
+              date_modified_to
-                                                        ORDER BY note_history_id`)),
+            FROM notes_history
+            ORDER BY note_history_id`)),
 
-        recent_notes: getHash(await sql.getAll(`SELECT
+        recent_notes: getHash(await sql.getAll(`
-                                                         note_tree_id,
+           SELECT
-                                                         note_path,
+             note_tree_id,
-                                                         date_accessed,
+             note_path,
-                                                         is_deleted
+             date_accessed,
-                                                       FROM recent_notes
+             is_deleted
-                                                       ORDER BY note_path`)),
+           FROM recent_notes
+           ORDER BY note_path`)),
 
-        options: getHash(await sql.getAll(`SELECT 
+        options: getHash(await sql.getAll(`
-                                                    opt_name,
+           SELECT 
-                                                    opt_value 
+             opt_name,
-                                                  FROM options 
+             opt_value 
-                                                  WHERE opt_name IN (${optionsQuestionMarks}) 
+           FROM options 
-                                                  ORDER BY opt_name`, options.SYNCED_OPTIONS))
+           WHERE opt_name IN (${optionsQuestionMarks}) 
+           ORDER BY opt_name`, options.SYNCED_OPTIONS)),
+
+        // we don't include image data on purpose because they are quite large, checksum is good enough
+        // to represent the data anyway
+        images: getHash(await sql.getAll(`
+          SELECT 
+            image_id,
+            format,
+            checksum,
+            name,
+            is_deleted,
+            date_modified,
+            date_created
+          FROM images  
+          ORDER BY image_id`))
     };
 
     const elapseTimeMs = new Date().getTime() - startTime.getTime();

services/notes.js

@@ -135,6 +135,76 @@ async function protectNoteHistory(noteId, dataKey, protect, sourceId) {
     }
 }
 
+async function saveNoteHistory(noteId, dataKey, sourceId, nowStr) {
+    const oldNote = await sql.getFirst("SELECT * FROM notes WHERE note_id = ?", [noteId]);
+
+    if (oldNote.is_protected) {
+        decryptNote(oldNote, dataKey);
+    }
+
+    const newNoteHistoryId = utils.newNoteHistoryId();
+
+    await sql.insert('notes_history', {
+        note_history_id: newNoteHistoryId,
+        note_id: noteId,
+        // title and text should be decrypted now
+        note_title: oldNote.note_title,
+        note_text: oldNote.note_text,
+        is_protected: 0, // will be fixed in the protectNoteHistory() call
+        date_modified_from: oldNote.date_modified,
+        date_modified_to: nowStr
+    });
+
+    await sync_table.addNoteHistorySync(newNoteHistoryId, sourceId);
+}
+
+async function saveNoteImages(noteId, noteText, sourceId) {
+    const existingNoteImages = await sql.getAll("SELECT * FROM notes_image WHERE note_id = ?", [noteId]);
+    const foundImageIds = [];
+    const now = utils.nowDate();
+    const re = /src="\/api\/images\/([a-zA-Z0-9]+)\//g;
+    let match;
+
+    while (match = re.exec(noteText)) {
+        const imageId = match[1];
+        const existingNoteImage = existingNoteImages.find(ni => ni.image_id === imageId);
+
+        if (!existingNoteImage) {
+            const noteImageId = utils.newNoteImageId();
+
+            await sql.insert("notes_image", {
+                note_image_id: noteImageId,
+                note_id: noteId,
+                image_id: imageId,
+                is_deleted: 0,
+                date_modified: now,
+                date_created: now
+            });
+
+            await sync_table.addNoteImageSync(noteImageId, sourceId);
+        }
+        else if (existingNoteImage.is_deleted) {
+            await sql.execute("UPDATE notes_image SET is_deleted = 0, date_modified = ? WHERE note_image_id = ?",
+                [now, existingNoteImage.note_image_id]);
+
+            await sync_table.addNoteImageSync(existingNoteImage.note_image_id, sourceId);
+        }
+        // else we don't need to do anything
+
+        foundImageIds.push(imageId);
+    }
+
+    // marking note images as deleted if they are not present on the page anymore
+    const unusedNoteImages = existingNoteImages.filter(ni => !foundImageIds.includes(ni.image_id));
+
+    for (const unusedNoteImage of unusedNoteImages) {
+        await sql.execute("UPDATE notes_image SET is_deleted = 1, date_modified = ? WHERE note_image_id = ?",
+            [now, unusedNoteImage.note_image_id]);
+
+        await sync_table.addNoteImageSync(unusedNoteImage.note_image_id, sourceId);
+    }
+}
+
 async function updateNote(noteId, newNote, dataKey, sourceId) {
     if (newNote.detail.is_protected) {
         await encryptNote(newNote, dataKey);
@@ -154,28 +224,11 @@ async function updateNote(noteId, newNote, dataKey, sourceId) {
         const msSinceDateCreated = now.getTime() - utils.parseDate(newNote.detail.date_created).getTime();
 
         if (!existingNoteHistoryId && msSinceDateCreated >= historySnapshotTimeInterval * 1000) {
-            const oldNote = await sql.getFirst("SELECT * FROM notes WHERE note_id = ?", [noteId]);
+            await saveNoteHistory(noteId, dataKey, sourceId, nowStr);
-
-            if (oldNote.is_protected) {
-                decryptNote(oldNote, dataKey);
-            }
-
-            const newNoteHistoryId = utils.newNoteHistoryId();
-
-            await sql.insert('notes_history', {
-                note_history_id: newNoteHistoryId,
-                note_id: noteId,
-                // title and text should be decrypted now
-                note_title: oldNote.note_title,
-                note_text: oldNote.note_text,
-                is_protected: 0, // will be fixed in the protectNoteHistory() call
-                date_modified_from: oldNote.date_modified,
-                date_modified_to: nowStr
-            });
-
-            await sync_table.addNoteHistorySync(newNoteHistoryId, sourceId);
         }
 
+        await saveNoteImages(noteId, newNote.detail.note_text, sourceId);
+
         await protectNoteHistory(noteId, dataKey, newNote.detail.is_protected);
 
         await sql.execute("UPDATE notes SET note_title = ?, note_text = ?, is_protected = ?, date_modified = ? WHERE note_id = ?", [

services/sync.js

@@ -14,22 +14,13 @@ const fs = require('fs');
 const app_info = require('./app_info');
 const messaging = require('./messaging');
 const sync_setup = require('./sync_setup');
+const sync_mutex = require('./sync_mutex');
 
-let syncInProgress = false;
 let proxyToggle = true;
 let syncServerCertificate = null;
 
 async function sync() {
-    if (syncInProgress) {
+    const releaseMutex = await sync_mutex.acquire();
-        log.info("Sync already in progress");
-
-        return {
-            success: false,
-            message: "Sync already in progress"
-        };
-    }
-
-    syncInProgress = true;
 
     try {
         if (!await sql.isDbUpToDate()) {
@@ -74,7 +65,7 @@ async function sync() {
         }
     }
     finally {
-        syncInProgress = false;
+        releaseMutex();
     }
 }
 
@@ -152,6 +143,12 @@ async function pullSync(syncContext) {
         else if (sync.entity_name === 'recent_notes') {
             await syncUpdate.updateRecentNotes(resp, syncContext.sourceId);
         }
+        else if (sync.entity_name === 'images') {
+            await syncUpdate.updateImage(resp, syncContext.sourceId);
+        }
+        else if (sync.entity_name === 'notes_image') {
+            await syncUpdate.updateNoteImage(resp, syncContext.sourceId);
+        }
         else {
             throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
         }
@@ -223,6 +220,16 @@ async function pushEntity(sync, syncContext) {
     else if (sync.entity_name === 'recent_notes') {
         entity = await sql.getFirst('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]);
     }
+    else if (sync.entity_name === 'images') {
+        entity = await sql.getFirst('SELECT * FROM images WHERE image_id = ?', [sync.entity_id]);
+
+        if (entity.data !== null) {
+            entity.data = entity.data.toString('base64');
+        }
+    }
+    else if (sync.entity_name === 'notes_image') {
+        entity = await sql.getFirst('SELECT * FROM notes_image WHERE note_image_id = ?', [sync.entity_id]);
+    }
     else {
         throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
     }

services/sync_mutex.js

@@ -0,0 +1,8 @@
+/**
+ * Sync makes process can make data intermittently inconsistent. Processes which require strong data consistency
+ * (like consistency checks) can use this mutex to make sure sync isn't currently running.
+ */
+
+const Mutex = require('async-mutex').Mutex;
+
+module.exports = new Mutex();

services/sync_table.js

@@ -28,6 +28,14 @@ async function addRecentNoteSync(noteTreeId, sourceId) {
     await addEntitySync("recent_notes", noteTreeId, sourceId);
 }
 
+async function addImageSync(imageId, sourceId) {
+    await addEntitySync("images", imageId, sourceId);
+}
+
+async function addNoteImageSync(noteImageId, sourceId) {
+    await addEntitySync("notes_image", noteImageId, sourceId);
+}
+
 async function addEntitySync(entityName, entityId, sourceId) {
     await sql.replace("sync", {
         entity_name: entityName,
@@ -78,6 +86,8 @@ async function fillAllSyncRows() {
     await fillSyncRows("notes_tree", "note_tree_id");
     await fillSyncRows("notes_history", "note_history_id");
     await fillSyncRows("recent_notes", "note_tree_id");
+    await fillSyncRows("images", "image_id");
+    await fillSyncRows("notes_image", "note_image_id");
 }
 
 module.exports = {
@@ -87,6 +97,8 @@ module.exports = {
     addNoteHistorySync,
     addOptionsSync,
     addRecentNoteSync,
+    addImageSync,
+    addNoteImageSync,
     cleanupSyncRowsForMissingEntities,
     fillAllSyncRows
 };

services/sync_update.js

@@ -92,11 +92,45 @@ async function updateRecentNotes(entity, sourceId) {
     }
 }
 
+async function updateImage(entity, sourceId) {
+    if (entity.data !== null) {
+        entity.data = Buffer.from(entity.data, 'base64');
+    }
+
+    const origImage = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [entity.image_id]);
+
+    if (!origImage || origImage.date_modified <= entity.date_modified) {
+        await sql.doInTransaction(async () => {
+            await sql.replace("images", entity);
+
+            await sync_table.addImageSync(entity.image_id, sourceId);
+        });
+
+        log.info("Update/sync image " + entity.image_id);
+    }
+}
+
+async function updateNoteImage(entity, sourceId) {
+    const origNoteImage = await sql.getFirst("SELECT * FROM notes_image WHERE note_image_id = ?", [entity.note_image_id]);
+
+    if (!origNoteImage || origNoteImage.date_modified <= entity.date_modified) {
+        await sql.doInTransaction(async () => {
+            await sql.replace("notes_image", entity);
+
+            await sync_table.addNoteImageSync(entity.note_image_id, sourceId);
+        });
+
+        log.info("Update/sync note image " + entity.note_image_id);
+    }
+}
+
 module.exports = {
     updateNote,
     updateNoteTree,
     updateNoteHistory,
     updateNoteReordering,
     updateOptions,
-    updateRecentNotes
+    updateRecentNotes,
+    updateImage,
+    updateNoteImage
 };

services/utils.js

@@ -15,6 +15,14 @@ function newNoteHistoryId() {
     return randomString(12);
 }
 
+function newImageId() {
+    return randomString(12);
+}
+
+function newNoteImageId() {
+    return randomString(12);
+}
+
 function randomString(length) {
     return randtoken.generate(length);
 }
@@ -79,6 +87,14 @@ function sanitizeSql(str) {
     return str.replace(/'/g, "\\'");
 }
 
+function assertArguments() {
+    for (const i in arguments) {
+        if (!arguments[i]) {
+            throw new Error(`Argument idx#${i} should not be falsy: ${arguments[i]}`);
+        }
+    }
+}
+
 module.exports = {
     randomSecureToken,
     randomString,
@@ -88,6 +104,8 @@ module.exports = {
     newNoteId,
     newNoteTreeId,
     newNoteHistoryId,
+    newImageId,
+    newNoteImageId,
     toBase64,
     fromBase64,
     hmac,
@@ -95,5 +113,6 @@ module.exports = {
     hash,
     isEmptyOrWhitespace,
     getDateTimeForFile,
-    sanitizeSql
+    sanitizeSql,
+    assertArguments
 };