we now remember past source ids so app restart won't change anything in sync operation

app.js

@@ -9,36 +9,8 @@ const helmet = require('helmet');
 const session = require('express-session');
 const FileStore = require('session-file-store')(session);
 const os = require('os');
-const options = require('./services/options');
-const utils = require('./services/utils');
-const sql = require('./services/sql');
-const dataDir = require('./services/data_dir');
 const sessionSecret = require('./services/session_secret');
 
-
-const db = require('sqlite');
-
-db.open(dataDir.DOCUMENT_PATH, { Promise }).then(async () => {
-    const tableResults = await sql.getResults("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'");
-
-    if (tableResults.length !== 1) {
-        console.log("No connection to initialized DB.");
-        process.exit(1);
-    }
-
-    if (!await options.getOption('document_id')) {
-        await options.setOption('document_id', utils.randomString(32));
-    }
-
-    if (!await options.getOption('document_secret')) {
-        await options.setOption('document_secret', utils.randomSecureToken(32));
-    }
-})
-    .catch(e => {
-        console.log("Error connecting to DB.", e);
-        process.exit(1);
-    });
-
 const app = express();
 
 // view engine setup

migrations/0034__source_ids.sql

@@ -0,0 +1,5 @@
+CREATE TABLE `source_ids` (
+	`source_id`	TEXT NOT NULL,
+	`date_created`	INTEGER NOT NULL,
+	PRIMARY KEY(`source_id`)
+);

routes/api/login.js

@@ -5,7 +5,7 @@ const router = express.Router();
 const options = require('../../services/options');
 const utils = require('../../services/utils');
 const migration = require('../../services/migration');
-const SOURCE_ID = require('../../services/source_id');
+const source_id = require('../../services/source_id');
 const auth = require('../../services/auth');
 const password_encryption = require('../../services/password_encryption');
 const protected_session = require('../../services/protected_session');
@@ -40,7 +40,7 @@ router.post('/sync', async (req, res, next) => {
     req.session.loggedIn = true;
 
     res.send({
-        sourceId: SOURCE_ID
+        sourceId: source_id.currentSourceId
     });
 });
 

routes/api/note_history.js

@@ -6,6 +6,7 @@ const sql = require('../../services/sql');
 const auth = require('../../services/auth');
 const data_encryption = require('../../services/data_encryption');
 const protected_session = require('../../services/protected_session');
+const sync_table = require('../../services/sync_table');
 
 router.get('/:noteId', auth.checkApiAuth, async (req, res, next) => {
     const noteId = req.params.noteId;
@@ -27,7 +28,7 @@ router.put('', auth.checkApiAuth, async (req, res, next) => {
     await sql.doInTransaction(async () => {
         await sql.replace("notes_history", req.body);
 
-        await sql.addNoteHistorySync(req.body.note_history_id);
+        await sync_table.addNoteHistorySync(req.body.note_history_id);
     });
 
     res.send();

routes/api/notes_move.js

@@ -6,6 +6,7 @@ const sql = require('../../services/sql');
 const utils = require('../../services/utils');
 const audit_category = require('../../services/audit_category');
 const auth = require('../../services/auth');
+const sync_table = require('../../services/sync_table');
 
 router.put('/:noteId/moveTo/:parentId', auth.checkApiAuth, async (req, res, next) => {
     let noteId = req.params.noteId;
@@ -26,7 +27,7 @@ router.put('/:noteId/moveTo/:parentId', auth.checkApiAuth, async (req, res, next
         await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?",
             [parentId, newNotePos, now, noteId]);
 
-        await sql.addNoteTreeSync(noteId);
+        await sync_table.addNoteTreeSync(noteId);
         await sql.addAudit(audit_category.CHANGE_PARENT, utils.browserId(req), noteId, null, parentId);
     });
 
@@ -50,8 +51,8 @@ router.put('/:noteId/moveBefore/:beforeNoteId', async (req, res, next) => {
             await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?",
                 [beforeNote['note_pid'], beforeNote['note_pos'], now, noteId]);
 
-            await sql.addNoteTreeSync(noteId);
+            await sync_table.addNoteTreeSync(noteId);
-            await sql.addNoteReorderingSync(beforeNote['note_pid']);
+            await sync_table.addNoteReorderingSync(beforeNote['note_pid']);
             await sql.addAudit(audit_category.CHANGE_POSITION, utils.browserId(req), beforeNote['note_pid']);
         });
     }
@@ -76,8 +77,8 @@ router.put('/:noteId/moveAfter/:afterNoteId', async (req, res, next) => {
             await sql.execute("update notes_tree set note_pid = ?, note_pos = ?, date_modified = ? where note_id = ?",
                 [afterNote['note_pid'], afterNote['note_pos'] + 1, now, noteId]);
 
-            await sql.addNoteTreeSync(noteId);
+            await sync_table.addNoteTreeSync(noteId);
-            await sql.addNoteReorderingSync(afterNote['note_pid']);
+            await sync_table.addNoteReorderingSync(afterNote['note_pid']);
             await sql.addAudit(audit_category.CHANGE_POSITION, utils.browserId(req), afterNote['note_pid']);
         });
     }

routes/api/recent_notes.js

@@ -5,6 +5,7 @@ const router = express.Router();
 const sql = require('../../services/sql');
 const auth = require('../../services/auth');
 const utils = require('../../services/utils');
+const sync_table = require('../../services/sync_table');
 
 router.get('', auth.checkApiAuth, async (req, res, next) => {
     res.send(await getRecentNotes());
@@ -17,7 +18,7 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => {
         is_deleted: 0
     });
 
-    await sql.addRecentNoteSync(req.params.noteId);
+    await sync_table.addRecentNoteSync(req.params.noteId);
 
     res.send(await getRecentNotes());
 });
@@ -25,7 +26,7 @@ router.put('/:noteId', auth.checkApiAuth, async (req, res, next) => {
 router.delete('/:noteId', auth.checkApiAuth, async (req, res, next) => {
     await sql.execute('UPDATE recent_notes SET is_deleted = 1 WHERE note_id = ?', [req.params.noteId]);
 
-    await sql.addRecentNoteSync(req.params.noteId);
+    await sync_table.addRecentNoteSync(req.params.noteId);
 
     res.send(await getRecentNotes());
 });

routes/api/sync.js

@@ -14,9 +14,8 @@ router.post('/now', auth.checkApiAuth, async (req, res, next) => {
 
 router.get('/changed', auth.checkApiAuth, async (req, res, next) => {
     const lastSyncId = parseInt(req.query.lastSyncId);
-    const sourceId = req.query.sourceId;
 
-    res.send(await sql.getResults("SELECT * FROM sync WHERE id > ? AND source_id != ?", [lastSyncId, sourceId]));
+    res.send(await sql.getResults("SELECT * FROM sync WHERE id > ?", [lastSyncId]));
 });
 
 router.get('/notes/:noteId', auth.checkApiAuth, async (req, res, next) => {

services/migration.js

@@ -4,7 +4,7 @@ const options = require('./options');
 const fs = require('fs-extra');
 const log = require('./log');
 
-const APP_DB_VERSION = 33;
+const APP_DB_VERSION = 34;
 const MIGRATIONS_DIR = "migrations";
 
 async function migrate() {

services/notes.js

@@ -4,6 +4,7 @@ const utils = require('./utils');
 const notes = require('./notes');
 const audit_category = require('./audit_category');
 const data_encryption = require('./data_encryption');
+const sync_table = require('./sync_table');
 
 async function createNewNote(parentNoteId, note, browserId) {
     const noteId = utils.newNoteId();
@@ -38,8 +39,8 @@ async function createNewNote(parentNoteId, note, browserId) {
 
     await sql.doInTransaction(async () => {
         await sql.addAudit(audit_category.CREATE_NOTE, browserId, noteId);
-        await sql.addNoteTreeSync(noteId);
+        await sync_table.addNoteTreeSync(noteId);
-        await sql.addNoteSync(noteId);
+        await sync_table.addNoteSync(noteId);
 
         const now = utils.nowTimestamp();
 
@@ -105,7 +106,7 @@ async function protectNote(note, dataKey, protect) {
         await sql.execute("UPDATE notes SET note_title = ?, note_text = ?, is_protected = ? WHERE note_id = ?",
             [note.note_title, note.note_text, note.is_protected, note.note_id]);
 
-        await sql.addNoteSync(note.note_id);
+        await sync_table.addNoteSync(note.note_id);
     }
 
     await protectNoteHistory(note.note_id, dataKey, protect);
@@ -129,7 +130,7 @@ async function protectNoteHistory(noteId, dataKey, protect) {
         await sql.execute("UPDATE notes_history SET note_title = ?, note_text = ?, is_protected = ? WHERE note_history_id = ?",
             [history.note_title, history.note_text, history.is_protected, history.note_history_id]);
 
-        await sql.addNoteHistorySync(history.note_history_id);
+        await sync_table.addNoteHistorySync(history.note_history_id);
     }
 }
 
@@ -166,7 +167,7 @@ async function updateNote(noteId, newNote, ctx) {
                 now
             ]);
 
-            await sql.addNoteHistorySync(newNoteHistoryId);
+            await sync_table.addNoteHistorySync(newNoteHistoryId);
         }
 
         await protectNoteHistory(noteId, ctx.getDataKey(), newNote.detail.is_protected);
@@ -194,8 +195,8 @@ async function updateNote(noteId, newNote, ctx) {
             await sql.insert("links", link);
         }
 
-        await sql.addNoteTreeSync(noteId);
+        await sync_table.addNoteTreeSync(noteId);
-        await sql.addNoteSync(noteId);
+        await sync_table.addNoteSync(noteId);
     });
 }
 
@@ -232,8 +233,8 @@ async function deleteNote(noteId, browserId) {
     await sql.execute("update notes_tree set is_deleted = 1, date_modified = ? where note_id = ?", [now, noteId]);
     await sql.execute("update notes set is_deleted = 1, date_modified = ? where note_id = ?", [now, noteId]);
 
-    await sql.addNoteTreeSync(noteId);
+    await sync_table.addNoteTreeSync(noteId);
-    await sql.addNoteSync(noteId);
+    await sync_table.addNoteSync(noteId);
 
     await sql.addAudit(audit_category.DELETE_NOTE, browserId, noteId);
 }

services/options.js

@@ -1,5 +1,6 @@
 const sql = require('./sql');
 const utils = require('./utils');
+const sync_table = require('./sync_table');
 
 const SYNCED_OPTIONS = [ 'username', 'password_verification_hash', 'encrypted_data_key', 'protected_session_timeout',
     'history_snapshot_time_interval' ];
@@ -20,7 +21,7 @@ async function setOptionInTransaction(optName, optValue) {
 
 async function setOption(optName, optValue) {
     if (SYNCED_OPTIONS.includes(optName)) {
-        await sql.addOptionsSync(optName);
+        await sync_table.addOptionsSync(optName);
     }
 
     await setOptionNoSync(optName, optValue);
@@ -32,6 +33,16 @@ async function setOptionNoSync(optName, optValue) {
     await sql.execute("UPDATE options SET opt_value = ?, date_modified = ? WHERE opt_name = ?", [optValue, now, optName]);
 }
 
+sql.dbReady.then(async () => {
+    if (!await getOption('document_id')) {
+        await setOption('document_id', utils.randomSecureToken(16));
+    }
+
+    if (!await getOption('document_secret')) {
+        await setOption('document_secret', utils.randomSecureToken(16));
+    }
+});
+
 module.exports = {
     getOption,
     setOption,

services/source_id.js

@@ -1,8 +1,27 @@
 const utils = require('./utils');
 const log = require('./log');
+const sql = require('./sql');
 
-const sourceId = utils.randomString(16);
+const currentSourceId = utils.randomString(12);
 
-log.info("Using sourceId=" + sourceId);
+log.info("Using sourceId=" + currentSourceId);
 
-module.exports = sourceId;
+let allSourceIds = [];
+
+sql.dbReady.then(async () => {
+    sql.insert("source_ids", {
+        source_id: currentSourceId,
+        date_created: utils.nowTimestamp()
+    });
+
+    allSourceIds = await sql.getFlattenedResults("source_id", "SELECT source_id FROM source_ids");
+});
+
+function isLocalSourceId(srcId) {
+    return allSourceIds.includes(srcId);
+}
+
+module.exports = {
+    currentSourceId,
+    isLocalSourceId
+};

services/sql.js

@@ -3,7 +3,7 @@
 const db = require('sqlite');
 const utils = require('./utils');
 const log = require('./log');
-const SOURCE_ID = require('./source_id');
+const dataDir = require('./data_dir');
 
 async function insert(table_name, rec, replace = false) {
     const keys = Object.keys(rec);
@@ -116,39 +116,6 @@ async function deleteRecentAudits(category, browserId, noteId) {
             [category, browserId, noteId, deleteCutoff])
 }
 
-async function addNoteSync(noteId, sourceId) {
-    await addEntitySync("notes", noteId, sourceId)
-}
-
-async function addNoteTreeSync(noteId, sourceId) {
-    await addEntitySync("notes_tree", noteId, sourceId)
-}
-
-async function addNoteReorderingSync(noteId, sourceId) {
-    await addEntitySync("notes_reordering", noteId, sourceId)
-}
-
-async function addNoteHistorySync(noteHistoryId, sourceId) {
-    await addEntitySync("notes_history", noteHistoryId, sourceId);
-}
-
-async function addOptionsSync(optName, sourceId) {
-    await addEntitySync("options", optName, sourceId);
-}
-
-async function addRecentNoteSync(noteId, sourceId) {
-    await addEntitySync("recent_notes", noteId, sourceId);
-}
-
-async function addEntitySync(entityName, entityId, sourceId) {
-    await replace("sync", {
-        entity_name: entityName,
-        entity_id: entityId,
-        sync_date: utils.nowTimestamp(),
-        source_id: sourceId || SOURCE_ID
-    });
-}
-
 async function wrap(func) {
     const error = new Error();
 
@@ -182,7 +149,24 @@ async function doInTransaction(func) {
     }
 }
 
+const dbReady = db.open(dataDir.DOCUMENT_PATH, { Promise });
+
+dbReady
+    .then(async () => {
+        const tableResults = await getResults("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'");
+
+        if (tableResults.length !== 1) {
+            console.log("No connection to initialized DB.");
+            process.exit(1);
+        }
+    })
+    .catch(e => {
+        console.log("Error connecting to DB.", e);
+        process.exit(1);
+    });
+
 module.exports = {
+    dbReady,
     insert,
     replace,
     getSingleValue,
@@ -196,11 +180,5 @@ module.exports = {
     addAudit,
     deleteRecentAudits,
     remove,
-    doInTransaction,
+    doInTransaction
-    addNoteSync,
-    addNoteTreeSync,
-    addNoteReorderingSync,
-    addNoteHistorySync,
-    addOptionsSync,
-    addRecentNoteSync
 };

services/sync.js

@@ -7,7 +7,7 @@ const options = require('./options');
 const migration = require('./migration');
 const utils = require('./utils');
 const config = require('./config');
-const SOURCE_ID = require('./source_id');
+const source_id = require('./source_id');
 const notes = require('./notes');
 const syncUpdate = require('./sync_update');
 
@@ -100,13 +100,19 @@ async function login() {
 async function pullSync(syncContext) {
     const lastSyncedPull = parseInt(await options.getOption('last_synced_pull'));
 
-    const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull + "&sourceId=" + SOURCE_ID;
+    const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
 
     const syncRows = await syncRequest(syncContext, 'GET', changesUri);
 
     log.info("Pulled " + syncRows.length + " changes from " + changesUri);
 
     for (const sync of syncRows) {
+        if (source_id.isLocalSourceId(sync.source_id)) {
+            log.info("Skipping " + sync.entity_name + " " + sync.entity_id + " because it has local source id.");
+
+            continue;
+        }
+
         const resp = await syncRequest(syncContext, 'GET', "/api/sync/" + sync.entity_name + "/" + sync.entity_id);
 
         if (sync.entity_name === 'notes') {
@@ -204,7 +210,7 @@ async function readAndPushEntity(sync, syncContext) {
 
 async function sendEntity(syncContext, entity, entityName) {
     const payload = {
-        sourceId: SOURCE_ID,
+        sourceId: source_id.currentSourceId,
         entity: entity
     };
 

services/sync_table.js

@@ -0,0 +1,45 @@
+const sql = require('./sql');
+const source_id = require('./source_id');
+const utils = require('./utils');
+
+async function addNoteSync(noteId, sourceId) {
+    await addEntitySync("notes", noteId, sourceId)
+}
+
+async function addNoteTreeSync(noteId, sourceId) {
+    await addEntitySync("notes_tree", noteId, sourceId)
+}
+
+async function addNoteReorderingSync(noteId, sourceId) {
+    await addEntitySync("notes_reordering", noteId, sourceId)
+}
+
+async function addNoteHistorySync(noteHistoryId, sourceId) {
+    await addEntitySync("notes_history", noteHistoryId, sourceId);
+}
+
+async function addOptionsSync(optName, sourceId) {
+    await addEntitySync("options", optName, sourceId);
+}
+
+async function addRecentNoteSync(noteId, sourceId) {
+    await addEntitySync("recent_notes", noteId, sourceId);
+}
+
+async function addEntitySync(entityName, entityId, sourceId) {
+    await sql.replace("sync", {
+        entity_name: entityName,
+        entity_id: entityId,
+        sync_date: utils.nowTimestamp(),
+        source_id: sourceId || source_id.currentSourceId
+    });
+}
+
+module.exports = {
+    addNoteSync,
+    addNoteTreeSync,
+    addNoteReorderingSync,
+    addNoteHistorySync,
+    addOptionsSync,
+    addRecentNoteSync
+};

services/sync_update.js

@@ -5,6 +5,7 @@ const utils = require('./utils');
 const audit_category = require('./audit_category');
 const eventLog = require('./event_log');
 const notes = require('./notes');
+const sync_table = require('./sync_table');
 
 async function updateNote(entity, links, sourceId) {
     const origNote = await sql.getSingleResult("select * from notes where note_id = ?", [entity.note_id]);
@@ -21,7 +22,7 @@ async function updateNote(entity, links, sourceId) {
                 await sql.insert('link', link);
             }
 
-            await sql.addNoteSync(entity.note_id, sourceId);
+            await sync_table.addNoteSync(entity.note_id, sourceId);
             await notes.addNoteAudits(origNote, entity, sourceId);
             await eventLog.addNoteEvent(entity.note_id, "Synced note <note>");
         });
@@ -42,7 +43,7 @@ async function updateNoteTree(entity, sourceId) {
 
             await sql.replace('notes_tree', entity);
 
-            await sql.addNoteTreeSync(entity.note_id, sourceId);
+            await sync_table.addNoteTreeSync(entity.note_id, sourceId);
 
             await sql.addAudit(audit_category.UPDATE_TITLE, sourceId, entity.note_id);
         });
@@ -61,7 +62,7 @@ async function updateNoteHistory(entity, sourceId) {
         await sql.doInTransaction(async () => {
             await sql.replace('notes_history', entity);
 
-            await sql.addNoteHistorySync(entity.note_history_id, sourceId);
+            await sync_table.addNoteHistorySync(entity.note_history_id, sourceId);
         });
 
         log.info("Update/sync note history " + entity.note_history_id);
@@ -77,7 +78,7 @@ async function updateNoteReordering(entity, sourceId) {
             await sql.execute("UPDATE notes_tree SET note_pos = ? WHERE note_id = ?", [entity.ordering[key], key]);
         });
 
-        await sql.addNoteReorderingSync(entity.note_pid, sourceId);
+        await sync_table.addNoteReorderingSync(entity.note_pid, sourceId);
         await sql.addAudit(audit_category.CHANGE_POSITION, sourceId, entity.note_pid);
     });
 }
@@ -93,7 +94,7 @@ async function updateOptions(entity, sourceId) {
         await sql.doInTransaction(async () => {
             await sql.replace('options', entity);
 
-            await sql.addOptionsSync(entity.opt_name, sourceId);
+            await sync_table.addOptionsSync(entity.opt_name, sourceId);
         });
 
         await eventLog.addEvent("Synced option " + entity.opt_name);
@@ -110,7 +111,7 @@ async function updateRecentNotes(entity, sourceId) {
         await sql.doInTransaction(async () => {
             await sql.replace('recent_notes', entity);
 
-            await sql.addRecentNoteSync(entity.note_id, sourceId);
+            await sync_table.addRecentNoteSync(entity.note_id, sourceId);
         });
     }
 }