release 0.37.7

(cherry picked from commit dc063983ea)

.idea/dataSources/a2c75661-f9e2-478f-a69f-6a9409e69997.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <dataSource name="document.db">
-  <database-model serializer="dbm" dbms="SQLITE" family-id="SQLITE" format-version="4.16">
+  <database-model serializer="dbm" dbms="SQLITE" family-id="SQLITE" format-version="4.17">
     <root id="1">
       <ServerVersion>3.25.1</ServerVersion>
     </root>

db/migrations/0152__add_contentLength_to_note.sql

@@ -20,7 +20,7 @@ SELECT noteId, title, -1, isProtected, type, mime, hash, isDeleted, isErased, da
 DROP TABLE notes;
 ALTER TABLE notes_mig RENAME TO notes;
 
-UPDATE notes SET contentLength = (SELECT COALESCE(LENGTH(content), 0) FROM note_contents WHERE note_contents.noteId = notes.noteId);
+UPDATE notes SET contentLength = COALESCE((SELECT COALESCE(LENGTH(content), 0) FROM note_contents WHERE note_contents.noteId = notes.noteId), -1);
 
 CREATE INDEX `IDX_notes_isDeleted` ON `notes` (`isDeleted`);
 CREATE INDEX `IDX_notes_title` ON `notes` (`title`);

libraries/springy.js

@@ -498,7 +498,7 @@
 				t._started = false;
 				onRenderStop();
 			} else {
-				setImmediate(step);
+				requestIdleCallback(step, { timeout: 10 });
 			}
 		}
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.36.5",
+  "version": "0.37.6",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -2,7 +2,7 @@
   "name": "trilium",
   "productName": "Trilium Notes",
   "description": "Trilium Notes",
-  "version": "0.37.2",
+  "version": "0.37.7",
   "license": "AGPL-3.0-only",
   "main": "electron.js",
   "bin": {

src/entities/note.js

@@ -334,6 +334,11 @@ class Note extends Entity {
         // we order by noteId so that attributes from same note stay together. Actual noteId ordering doesn't matter.
 
         const filteredAttributes = attributes.filter((attr, index) => {
+            // if this exact attribute already appears then don't include it (can happen via cloning)
+            if (attributes.findIndex(it => it.attributeId === attr.attributeId) !== index) {
+                return false;
+            }
+
             if (attr.isDefinition()) {
                 const firstDefinitionIndex = attributes.findIndex(el => el.type === attr.type && el.name === attr.name);
 
@@ -788,6 +793,7 @@ class Note extends Entity {
         delete pojo.isContentAvailable;
         delete pojo.__attributeCache;
         delete pojo.content;
+        /** zero references to contentHash, probably can be removed */
         delete pojo.contentHash;
     }
 }

src/public/javascripts/dialogs/export.js

@@ -74,7 +74,7 @@ $form.on('submit', () => {
 function exportBranch(branchId, type, format, version) {
     taskId = utils.randomString(10);
 
-    const url = utils.getHost() + `/api/notes/${branchId}/export/${type}/${format}/${version}/${taskId}`;
+    const url = utils.getUrlForDownload(`api/notes/${branchId}/export/${type}/${format}/${version}/${taskId}`);
 
     utils.download(url);
 }

src/public/javascripts/dialogs/note_revisions.js

@@ -102,7 +102,9 @@ async function setContentPane() {
     const $downloadButton = $('<button class="btn btn-sm btn-primary" type="button">Download</button>');
 
     $downloadButton.on('click', () => {
-        utils.download(utils.getHost() + `/api/notes/${revisionItem.noteId}/revisions/${revisionItem.noteRevisionId}/download`);
+        const url = utils.getUrlForDownload(`api/notes/${revisionItem.noteId}/revisions/${revisionItem.noteRevisionId}/download`);
+
+        utils.download(url);
     });
 
     $titleButtons.append($downloadButton);

src/public/javascripts/dialogs/options/sidebar.js

@@ -93,7 +93,7 @@ export default class SidebarOptions {
         this.$sidebarMinWidth.val(options.sidebarMinWidth);
         this.$sidebarWidthPercent.val(options.sidebarWidthPercent);
 
-        if (parseInt(options.showSidebarInNewTab)) {
+        if (options.showSidebarInNewTab === 'true') {
             this.$showSidebarInNewTab.attr("checked", "checked");
         }
         else {

src/public/javascripts/services/clipboard.js

@@ -4,83 +4,97 @@ import cloningService from "./cloning.js";
 import toastService from "./toast.js";
 import hoistedNoteService from "./hoisted_note.js";
 
-let clipboardIds = [];
+/*
+ * Clipboard contains node keys which are not stable. If a (part of the) tree is reloaded,
+ * node keys in the clipboard might not exist anymore. Code here should then be ready to deal
+ * with this.
+ */
+
+let clipboardNodeKeys = [];
 let clipboardMode = null;
 
-async function pasteAfter(node) {
+async function pasteAfter(afterNode) {
+    if (isClipboardEmpty()) {
+        return;
+    }
+
     if (clipboardMode === 'cut') {
-        const nodes = clipboardIds.map(nodeKey => treeUtils.getNodeByKey(nodeKey));
+        const nodes = clipboardNodeKeys.map(nodeKey => treeUtils.getNodeByKey(nodeKey));
 
-        await treeChangesService.moveAfterNode(nodes, node);
+        await treeChangesService.moveAfterNode(nodes, afterNode);
 
-        clipboardIds = [];
+        clipboardNodeKeys = [];
         clipboardMode = null;
     }
     else if (clipboardMode === 'copy') {
-        for (const noteId of clipboardIds) {
-            await cloningService.cloneNoteAfter(noteId, node.data.branchId);
+        for (const nodeKey of clipboardNodeKeys) {
+            const clipNode = treeUtils.getNodeByKey(nodeKey);
+
+            await cloningService.cloneNoteAfter(clipNode.data.noteId, afterNode.data.branchId);
         }
 
         // copy will keep clipboardIds and clipboardMode so it's possible to paste into multiple places
     }
-    else if (clipboardIds.length === 0) {
-        // just do nothing
-    }
     else {
         toastService.throwError("Unrecognized clipboard mode=" + clipboardMode);
     }
 }
 
-async function pasteInto(node) {
+async function pasteInto(parentNode) {
+    if (isClipboardEmpty()) {
+        return;
+    }
+
     if (clipboardMode === 'cut') {
-        const nodes = clipboardIds.map(nodeKey => treeUtils.getNodeByKey(nodeKey));
+        const nodes = clipboardNodeKeys.map(nodeKey => treeUtils.getNodeByKey(nodeKey));
 
-        await treeChangesService.moveToNode(nodes, node);
+        await treeChangesService.moveToNode(nodes, parentNode);
 
-        await node.setExpanded(true);
+        await parentNode.setExpanded(true);
 
-        clipboardIds = [];
+        clipboardNodeKeys = [];
         clipboardMode = null;
     }
     else if (clipboardMode === 'copy') {
-        for (const noteId of clipboardIds) {
-            await cloningService.cloneNoteTo(noteId, node.data.noteId);
+        for (const nodeKey of clipboardNodeKeys) {
+            const clipNode = treeUtils.getNodeByKey(nodeKey);
+
+            await cloningService.cloneNoteTo(clipNode.data.noteId, parentNode.data.noteId);
         }
 
-        await node.setExpanded(true);
+        await parentNode.setExpanded(true);
 
         // copy will keep clipboardIds and clipboardMode so it's possible to paste into multiple places
     }
-    else if (clipboardIds.length === 0) {
-        // just do nothing
-    }
     else {
         toastService.throwError("Unrecognized clipboard mode=" + mode);
     }
 }
 
 function copy(nodes) {
-    clipboardIds = nodes.map(node => node.data.noteId);
+    clipboardNodeKeys = nodes.map(node => node.key);
     clipboardMode = 'copy';
 
     toastService.showMessage("Note(s) have been copied into clipboard.");
 }
 
 function cut(nodes) {
-    clipboardIds = nodes
+    clipboardNodeKeys = nodes
         .filter(node => node.data.noteId !== hoistedNoteService.getHoistedNoteNoPromise())
         .filter(node => node.getParent().data.noteType !== 'search')
-        .map(node => node.data.noteId);
+        .map(node => node.key);
 
-    if (clipboardIds.length > 0) {
+    if (clipboardNodeKeys.length > 0) {
         clipboardMode = 'cut';
 
         toastService.showMessage("Note(s) have been cut into clipboard.");
     }
 }
 
-function isEmpty() {
-    return clipboardIds.length === 0;
+function isClipboardEmpty() {
+    clipboardNodeKeys = clipboardNodeKeys.filter(key => !!treeUtils.getNodeByKey(key));
+
+    return clipboardNodeKeys.length === 0;
 }
 
 export default {
@@ -88,5 +102,5 @@ export default {
     pasteInto,
     cut,
     copy,
-    isEmpty
+    isClipboardEmpty
 }

src/public/javascripts/services/note_detail.js

@@ -273,7 +273,9 @@ async function filterTabs(noteId) {
 
 async function noteDeleted(noteId) {
     for (const tc of tabContexts) {
-        if (tc.notePath && tc.notePath.split("/").includes(noteId)) {
+        // not removing active even if it contains deleted note since that one will move to another note (handled by deletion logic)
+        // and we would lose tab context state (e.g. sidebar visibility)
+        if (!tc.isActive() && tc.notePath && tc.notePath.split("/").includes(noteId)) {
             await tabRow.removeTab(tc.$tab[0]);
         }
     }

src/public/javascripts/services/note_detail_book.js

@@ -185,8 +185,7 @@ class NoteDetailBook {
         }
         else if (type === 'file') {
             function getFileUrl() {
-                // electron needs absolute URL so we extract current host, port, protocol
-                return utils.getHost() + "/api/notes/" + note.noteId + "/download";
+                return utils.getUrlForDownload("api/notes/" + note.noteId + "/download");
             }
 
             const $downloadButton = $('<button class="file-download btn btn-primary" type="button">Download</button>');

src/public/javascripts/services/note_detail_file.js

@@ -87,8 +87,7 @@ class NoteDetailFile {
     }
 
     getFileUrl() {
-        // electron needs absolute URL so we extract current host, port, protocol
-        return utils.getHost() + "/api/notes/" + this.ctx.note.noteId + "/download";
+        return utils.getUrlForDownload("api/notes/" + this.ctx.note.noteId + "/download");
     }
 
     show() {}

src/public/javascripts/services/note_detail_image.js

@@ -98,8 +98,7 @@ class NoteDetailImage {
     }
 
     getFileUrl() {
-        // electron needs absolute URL so we extract current host, port, protocol
-        return utils.getHost() + `/api/notes/${this.ctx.note.noteId}/download`;
+        return utils.getUrlForDownload(`api/notes/${this.ctx.note.noteId}/download`);
     }
 
     show() {}

src/public/javascripts/services/search_notes.js

@@ -142,6 +142,12 @@ async function refreshSearch() {
     toastService.showMessage("Saved search note refreshed.");
 }
 
+function searchInSubtree(noteId) {
+    showSearch();
+
+    $searchInput.val(`@in=${noteId} @text*=*`);
+}
+
 function init() {
     const hashValue = document.location.hash ? document.location.hash.substr(1) : ""; // strip initial #
 
@@ -178,5 +184,6 @@ export default {
     refreshSearch,
     doSearch,
     init,
+    searchInSubtree,
     getHelpText: () => helpText
 };

src/public/javascripts/services/tab_context.js

@@ -246,11 +246,15 @@ class TabContext {
     }
 
     setCurrentNotePathToHash() {
-        if (this.$tab[0] === this.tabRow.activeTabEl) {
+        if (this.isActive()) {
             document.location.hash = (this.notePath || "") + "-" + this.tabId;
         }
     }
 
+    isActive() {
+        return this.$tab[0] === this.tabRow.activeTabEl;
+    }
+
     setupClasses() {
         for (const clazz of Array.from(this.$tab[0].classList)) { // create copy to safely iterate over while removing classes
             if (clazz !== 'note-tab') {

src/public/javascripts/services/tree_context_menu.js

@@ -9,6 +9,7 @@ import hoistedNoteService from './hoisted_note.js';
 import noteDetailService from './note_detail.js';
 import clipboard from './clipboard.js';
 import protectedSessionHolder from "./protected_session_holder.js";
+import searchNotesService from "./search_notes.js";
 
 class TreeContextMenu {
     constructor(node) {
@@ -41,7 +42,7 @@ class TreeContextMenu {
                 || (selNodes.length === 1 && selNodes[0] === this.node);
 
         const notSearch = note.type !== 'search';
-        const parentNotSearch = parentNote.type !== 'search';
+        const parentNotSearch = !parentNote || parentNote.type !== 'search';
         const insertNoteAfterEnabled = isNotRoot && !isHoisted && parentNotSearch;
 
         return [
@@ -55,6 +56,8 @@ class TreeContextMenu {
             { title: "Delete <kbd>Delete</kbd>", cmd: "delete", uiIcon: "trash",
                 enabled: isNotRoot && !isHoisted && parentNotSearch },
             { title: "----" },
+            { title: "Search in subtree <kbd>Ctrl+Shift+S</kbd>", cmd: "searchInSubtree", uiIcon: "search",
+                enabled: notSearch && noSelectedNotes },
             isHoisted ? null : { title: "Hoist note <kbd>Ctrl-H</kbd>", cmd: "hoist", uiIcon: "empty", enabled: noSelectedNotes && notSearch },
             !isHoisted || !isNotRoot ? null : { title: "Unhoist note <kbd>Ctrl-H</kbd>", cmd: "unhoist", uiIcon: "arrow-up" },
             { title: "Edit branch prefix <kbd>F2</kbd>", cmd: "editBranchPrefix", uiIcon: "empty",
@@ -72,11 +75,11 @@ class TreeContextMenu {
             { title: "Move to ... <kbd>Ctrl+Shift+X</kbd>", cmd: "moveTo", uiIcon: "empty",
                 enabled: isNotRoot && !isHoisted && parentNotSearch },
             { title: "Paste into <kbd>Ctrl+V</kbd>", cmd: "pasteInto", uiIcon: "paste",
-                enabled: !clipboard.isEmpty() && notSearch && noSelectedNotes },
+                enabled: !clipboard.isClipboardEmpty() && notSearch && noSelectedNotes },
             { title: "Paste after", cmd: "pasteAfter", uiIcon: "paste",
-                enabled: !clipboard.isEmpty() && isNotRoot && parentNotSearch && noSelectedNotes },
+                enabled: !clipboard.isClipboardEmpty() && isNotRoot && !isHoisted && parentNotSearch && noSelectedNotes },
             { title: "Duplicate note here", cmd: "duplicateNote", uiIcon: "empty",
-                enabled: noSelectedNotes && parentNotSearch && (!note.isProtected || protectedSessionHolder.isProtectedSessionAvailable()) },
+                enabled: noSelectedNotes && parentNotSearch && isNotRoot && !isHoisted && (!note.isProtected || protectedSessionHolder.isProtectedSessionAvailable()) },
             { title: "----" },
             { title: "Export", cmd: "export", uiIcon: "empty",
                 enabled: notSearch && noSelectedNotes },
@@ -177,6 +180,9 @@ class TreeContextMenu {
 
             treeService.duplicateNote(this.node.data.noteId, branch.parentNoteId);
         }
+        else if (cmd === "searchInSubtree") {
+            searchNotesService.searchInSubtree(this.node.data.noteId);
+        }
         else {
             ws.logError("Unknown command: " + cmd);
         }

src/public/javascripts/services/tree_keybindings.js

@@ -4,6 +4,7 @@ import treeService from "./tree.js";
 import hoistedNoteService from "./hoisted_note.js";
 import clipboard from "./clipboard.js";
 import treeCache from "./tree_cache.js";
+import searchNoteService from "./search_notes.js";
 
 const keyBindings = {
     "del": node => {
@@ -167,6 +168,11 @@ const keyBindings = {
     "down": node => {
         node.navigate($.ui.keyCode.DOWN, true).then(treeService.clearSelectedNodes);
 
+        return false;
+    },
+    "ctrl+shift+s": node => {
+        searchNoteService.searchInSubtree(node.data.noteId);
+
         return false;
     }
 };

src/public/javascripts/services/utils.js

@@ -214,6 +214,20 @@ async function clearBrowserCache() {
     }
 }
 
+/**
+ * @param url - should be without initial slash!!!
+ */
+function getUrlForDownload(url) {
+    if (isElectron()) {
+        // electron needs absolute URL so we extract current host, port, protocol
+        return getHost() + '/' + url;
+    }
+    else {
+        // web server can be deployed on subdomain so we need to use relative path
+        return url;
+    }
+}
+
 export default {
     reloadApp,
     parseDate,
@@ -230,7 +244,6 @@ export default {
     escapeHtml,
     stopWatch,
     formatLabel,
-    getHost,
     download,
     toObject,
     randomString,
@@ -245,5 +258,6 @@ export default {
     getMimeTypeClass,
     closeActiveDialog,
     isHtmlEmpty,
-    clearBrowserCache
+    clearBrowserCache,
+    getUrlForDownload
 };

src/public/javascripts/services/ws.js

@@ -127,11 +127,13 @@ async function consumeSyncData() {
 }
 
 function connectWebSocket() {
-    const protocol = document.location.protocol === 'https:' ? 'wss' : 'ws';
+    const loc = window.location;
+    const webSocketUri = (loc.protocol === "https:" ? "wss:" : "ws:")
+                       + "//" + loc.host + loc.pathname;
 
     // use wss for secure messaging
-    const ws = new WebSocket(protocol + "://" + location.host);
-    ws.onopen = () => console.debug(utils.now(), "Connected to server with WebSocket");
+    const ws = new WebSocket(webSocketUri);
+    ws.onopen = () => console.debug(utils.now(), `Connected to server ${webSocketUri} with WebSocket`);
     ws.onmessage = handleMessage;
     // we're not handling ws.onclose here because reconnection is done in sendPing()
 

src/public/javascripts/setup.js

@@ -76,12 +76,12 @@ function SetupModel() {
             }
 
             // not using server.js because it loads too many dependencies
-            $.post('/api/setup/new-document', {
+            $.post('api/setup/new-document', {
                 username: username,
                 password: password1,
                 theme: theme
             }).then(() => {
-                window.location.replace("/");
+                window.location.replace("./");
             });
         }
         else if (this.setupType() === 'sync-from-server') {
@@ -128,10 +128,10 @@ function SetupModel() {
 }
 
 async function checkOutstandingSyncs() {
-    const { stats, initialized } = await $.get('/api/sync/stats');
+    const { stats, initialized } = await $.get('api/sync/stats');
 
     if (initialized) {
-        window.location.replace("/");
+        window.location.replace("./");
     }
 
     const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;

src/public/stylesheets/desktop.css

@@ -89,6 +89,11 @@ body {
     font-size: inherit;
 }
 
+#context-menu-container {
+    max-height: 100vh;
+    overflow: auto; /* make it scrollable when exceeding total height of the window */
+}
+
 #context-menu-container, #context-menu-container .dropdown-menu {
     padding: 3px 0 0;
     z-index: 1111;

src/public/stylesheets/style.css

@@ -411,6 +411,10 @@ div.ui-tooltip {
     height: 150px;
 }
 
+#sql-console-query .CodeMirror-scroll {
+    min-height: inherit !important;
+}
+
 .btn {
     border-radius: var(--button-border-radius);
 }

src/routes/api/recent_changes.js

@@ -17,8 +17,8 @@ async function getRecentChanges() {
             FROM 
                 note_revisions
                 JOIN notes USING(noteId)
-            ORDER BY 
-                utcDateCreated DESC
+            ORDER BY
+                note_revisions.utcDateCreated DESC
             LIMIT 1000
         )
         UNION ALL SELECT * FROM (

src/services/anonymization.js

@@ -18,7 +18,10 @@ async function anonymize() {
 
     await db.run("UPDATE notes SET title = 'title'");
     await db.run("UPDATE note_contents SET content = 'text'");
-    await db.run("UPDATE note_revisions SET title = 'title', content = 'text'");
+    await db.run("UPDATE note_revisions SET title = 'title'");
+    await db.run("UPDATE note_revision_contents SET content = 'title'");
+    await db.run("UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label'");
+    await db.run("UPDATE attributes SET name = 'name' WHERE type = 'relation'");
     await db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL");
     await db.run(`UPDATE options SET value = 'anonymized' WHERE name IN 
                     ('documentSecret', 'encryptedDataKey', 'passwordVerificationHash', 

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2019-11-18T23:04:09+01:00", buildRevision: "834e1f7253186922d2b5df2f6a01c34f7c2d7fe4" };
+module.exports = { buildDate:"2019-12-02T23:09:42+01:00", buildRevision: "6ba3e5ab7f866ac93ecbb48883dabf61723cde98" };

src/services/consistency_checks.js

@@ -311,6 +311,25 @@ async function findLogicIssues() {
             }
         });
 
+    await findAndFixIssues(`
+                SELECT notes.noteId
+                FROM notes
+                  LEFT JOIN note_contents USING(noteId)
+                WHERE
+                  note_contents.noteId IS NULL`,
+        async ({noteId}, autoFix) => {
+            if (autoFix) {
+                const note = await repository.getNote(noteId);
+                // empty string might be wrong choice for some note types (and protected notes) but it's a best guess
+                await note.setContent(note.isErased ? null : '');
+
+                logFix(`Note ${noteId} content was set to empty string since there was no corresponding row`);
+            }
+            else {
+                logError(`Note ${noteId} content row does not exist`);
+            }
+        });
+
     await findAndFixIssues(`
           SELECT noteId
           FROM notes
@@ -321,6 +340,7 @@ async function findLogicIssues() {
         async ({noteId}, autoFix) => {
             if (autoFix) {
                 const note = await repository.getNote(noteId);
+                // empty string might be wrong choice for some note types (and protected notes) but it's a best guess
                 await note.setContent('');
 
                 logFix(`Note ${noteId} content was set to empty string since it was null even though it is not deleted`);
@@ -360,6 +380,25 @@ async function findLogicIssues() {
             }
         });
 
+    await findAndFixIssues(`
+                SELECT note_revisions.noteRevisionId
+                FROM note_revisions
+                LEFT JOIN note_revision_contents USING(noteRevisionId)
+                WHERE note_revision_contents.noteRevisionId IS NULL`,
+        async ({noteRevisionId}, autoFix) => {
+            if (autoFix) {
+                const noteRevision = await repository.getNoteRevision(noteRevisionId);
+                await noteRevision.setContent(null);
+                noteRevision.isErased = true;
+                await noteRevision.save();
+
+                logFix(`Note revision content ${noteRevisionId} was created and set to erased since it did not exist.`);
+            }
+            else {
+                logError(`Note revision content ${noteRevisionId} does not exist`);
+            }
+        });
+
     await findAndFixIssues(`
           SELECT noteRevisionId
           FROM note_revisions
@@ -587,12 +626,31 @@ async function runAllChecks() {
     return !unrecoveredConsistencyErrors;
 }
 
+async function showEntityStat(name, query) {
+    const map = await sql.getMap(query);
+
+    map[0] = map[0] || 0;
+    map[1] = map[1] || 0;
+
+    log.info(`${name} deleted: ${map[1]}, not deleted ${map[0]}`);
+}
+
+async function runDbDiagnostics() {
+    await showEntityStat("Notes", `SELECT isDeleted, count(noteId) FROM notes GROUP BY isDeleted`);
+    await showEntityStat("Note revisions", `SELECT isErased, count(noteRevisionId) FROM note_revisions GROUP BY isErased`);
+    await showEntityStat("Branches", `SELECT isDeleted, count(branchId) FROM branches GROUP BY isDeleted`);
+    await showEntityStat("Attributes", `SELECT isDeleted, count(attributeId) FROM attributes GROUP BY isDeleted`);
+    await showEntityStat("API tokens", `SELECT isDeleted, count(apiTokenId) FROM api_tokens GROUP BY isDeleted`);
+}
+
 async function runChecks() {
     let elapsedTimeMs;
 
     await syncMutexService.doExclusively(async () => {
         const startTime = new Date();
 
+        await runDbDiagnostics();
+
         await runAllChecks();
 
         elapsedTimeMs = Date.now() - startTime.getTime();
@@ -624,7 +682,7 @@ sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(runChecks), 60 * 60 * 1000);
 
     // kickoff checks soon after startup (to not block the initial load)
-    setTimeout(cls.wrap(runChecks), 10 * 1000);
+    setTimeout(cls.wrap(runChecks), 20 * 1000);
 });
 
 module.exports = {};

src/services/content_hash.js

@@ -56,6 +56,8 @@ async function checkContentHashes(otherHashes) {
         if (hashes[key] !== otherHashes[key]) {
             allChecksPassed = false;
 
+            log.info(`Content hash check for ${key} FAILED. Local is ${hashes[key]}, remote is ${otherHashes[key]}`);
+
             if (key !== 'recent_notes') {
                 // let's not get alarmed about recent notes which get updated often and can cause failures in race conditions
                 ws.sendMessageToAllClients({type: 'sync-hash-check-failed'});

src/services/handlers.js

@@ -81,7 +81,7 @@ eventService.subscribe(eventService.CHILD_NOTE_CREATED, async ({ parentNote, chi
 async function processInverseRelations(entityName, entity, handler) {
     if (entityName === 'attributes' && entity.type === 'relation') {
         const note = await entity.getNote();
-        const attributes = (await note.getAttributes(entity.name)).filter(relation => relation.type === 'relation-definition');
+        const attributes = (await note.getOwnedAttributes(entity.name)).filter(relation => relation.type === 'relation-definition');
 
         for (const attribute of attributes) {
             const definition = attribute.value;

src/services/image.js

@@ -12,9 +12,7 @@ const imageminGifLossy = require('imagemin-giflossy');
 const jimp = require('jimp');
 const imageType = require('image-type');
 const sanitizeFilename = require('sanitize-filename');
-const dateUtils = require('./date_utils');
 const noteRevisionService = require('./note_revisions.js');
-const NoteRevision = require("../entities/note_revision");
 
 async function processImage(uploadBuffer, originalName, shrinkImageSwitch) {
     const origImageFormat = imageType(uploadBuffer);

src/services/import/enex.js

@@ -3,6 +3,7 @@ const fileType = require('file-type');
 const stream = require('stream');
 const log = require("../log");
 const utils = require("../utils");
+const sql = require("../sql");
 const noteService = require("../notes");
 const imageService = require("../image");
 const protectedSessionService = require('../protected_session');
@@ -11,7 +12,7 @@ const protectedSessionService = require('../protected_session');
 function parseDate(text) {
     // insert - and : to make it ISO format
     text = text.substr(0, 4) + "-" + text.substr(4, 2) + "-" + text.substr(6, 2)
-        + "T" + text.substr(9, 2) + ":" + text.substr(11, 2) + ":" + text.substr(13, 2) + "Z";
+        + " " + text.substr(9, 2) + ":" + text.substr(11, 2) + ":" + text.substr(13, 2) + ".000Z";
 
     return text;
 }
@@ -150,7 +151,7 @@ async function importEnex(taskContext, file, parentNote) {
             } else if (currentTag === 'created') {
                 note.utcDateCreated = parseDate(text);
             } else if (currentTag === 'updated') {
-                // updated is currently ignored since utcDateModified is updated automatically with each save
+                note.utcDateModified = parseDate(text);
             } else if (currentTag === 'tag') {
                 note.attributes.push({
                     type: 'label',
@@ -187,9 +188,27 @@ async function importEnex(taskContext, file, parentNote) {
         }
     });
 
+    async function updateDates(noteId, utcDateCreated, utcDateModified) {
+        // it's difficult to force custom dateCreated and dateModified to Note entity so we do it post-creation with SQL
+        await sql.execute(`
+                UPDATE notes 
+                SET dateCreated = ?, 
+                    utcDateCreated = ?,
+                    dateModified = ?,
+                    utcDateModified = ?
+                WHERE noteId = ?`,
+            [utcDateCreated, utcDateCreated, utcDateModified, utcDateModified, noteId]);
+
+        await sql.execute(`
+                UPDATE note_contents
+                SET utcDateModified = ?
+                WHERE noteId = ?`,
+            [utcDateModified, noteId]);
+    }
+
     async function saveNote() {
         // make a copy because stream continues with the next async call and note gets overwritten
-        let {title, content, attributes, resources, utcDateCreated} = note;
+        let {title, content, attributes, resources, utcDateCreated, utcDateModified} = note;
 
         content = extractContent(content);
 
@@ -201,6 +220,10 @@ async function importEnex(taskContext, file, parentNote) {
             isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
         })).note;
 
+        utcDateCreated = utcDateCreated || noteEntity.utcDateCreated;
+        // sometime date modified is not present in ENEX, then use date created
+        utcDateModified = utcDateModified || utcDateCreated;
+
         taskContext.increaseProgressCount();
 
         let noteContent = await noteEntity.getContent();
@@ -224,6 +247,8 @@ async function importEnex(taskContext, file, parentNote) {
                     isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
                 })).note;
 
+                await updateDates(resourceNote.noteId, utcDateCreated, utcDateModified);
+
                 taskContext.increaseProgressCount();
 
                 const resourceLink = `<a href="#root/${resourceNote.noteId}">${utils.escapeHtml(resource.title)}</a>`;
@@ -235,7 +260,9 @@ async function importEnex(taskContext, file, parentNote) {
                 try {
                     const originalName = "image." + resource.mime.substr(6);
 
-                    const {url} = await imageService.saveImage(noteEntity.noteId, resource.content, originalName, taskContext.data.shrinkImages);
+                    const {url, note: imageNote} = await imageService.saveImage(noteEntity.noteId, resource.content, originalName, taskContext.data.shrinkImages);
+
+                    await updateDates(imageNote.noteId, utcDateCreated, utcDateModified);
 
                     const imageLink = `<img src="${url}">`;
 
@@ -257,6 +284,10 @@ async function importEnex(taskContext, file, parentNote) {
 
         // save updated content with links to files/images
         await noteEntity.setContent(noteContent);
+
+        await noteService.scanForLinks(noteEntity.noteId);
+
+        await updateDates(noteEntity.noteId, utcDateCreated, utcDateModified);
     }
 
     saxStream.on("closetag", async tag => {

src/services/import/tar.js

@@ -258,7 +258,8 @@ async function importTar(taskContext, fileBuffer, importRootNote) {
             content = content.toString("UTF-8");
         }
 
-        if ((noteMeta && noteMeta.format === 'markdown') || (!noteMeta && ['text/markdown', 'text/x-markdown'].includes(mime))) {
+        if ((noteMeta && noteMeta.format === 'markdown')
+            || (!noteMeta && taskContext.data.textImportedAsText && ['text/markdown', 'text/x-markdown'].includes(mime))) {
             const parsed = mdReader.parse(content);
             content = mdWriter.render(parsed);
         }

src/services/migration.js

@@ -43,9 +43,6 @@ async function migrate() {
         try {
             log.info("Attempting migration to version " + mig.dbVersion);
 
-            // needs to happen outside of the transaction (otherwise it's a NO-OP)
-            await sql.execute("PRAGMA foreign_keys = OFF");
-
             await sql.transactional(async () => {
                 if (mig.type === 'sql') {
                     const migrationSql = fs.readFileSync(resourceDir.MIGRATIONS_DIR + "/" + mig.file).toString('utf8');
@@ -76,10 +73,6 @@ async function migrate() {
 
             utils.crash();
         }
-        finally {
-            // make sure foreign keys are enabled even if migration script disables them
-            await sql.execute("PRAGMA foreign_keys = ON");
-        }
     }
 
     if (await sqlInit.isDbUpToDate()) {

src/services/note_revisions.js

@@ -7,10 +7,6 @@ const dateUtils = require('../services/date_utils');
  * @param {Note} note
  */
 async function protectNoteRevisions(note) {
-    if (await note.hasLabel('disableVersioning')) {
-        return;
-    }
-
     for (const revision of await note.getRevisions()) {
         if (note.isProtected !== revision.isProtected) {
             const content = await revision.getContent();
@@ -30,6 +26,10 @@ async function protectNoteRevisions(note) {
  * @return {NoteRevision}
  */
 async function createNoteRevision(note) {
+    if (await note.hasLabel("disableVersioning")) {
+        return;
+    }
+
     const noteRevision = await new NoteRevision({
         noteId: note.noteId,
         // title and text should be decrypted now

src/services/notes.js

@@ -117,7 +117,7 @@ async function createNewNote(parentNoteId, noteData) {
         isExpanded: !!noteData.isExpanded
     }).save();
 
-    for (const attr of await parentNote.getAttributes()) {
+    for (const attr of await parentNote.getOwnedAttributes()) {
         if (attr.name.startsWith("child:")) {
             await new Attribute({
                noteId: note.noteId,
@@ -308,8 +308,7 @@ async function saveLinks(note, content) {
 }
 
 async function saveNoteRevision(note) {
-    // files and images are immutable, they can't be updated
-    // but we don't even version titles which is probably not correct
+    // files and images are versioned separately
     if (note.type === 'file' || note.type === 'image' || await note.hasLabel('disableVersioning')) {
         return;
     }
@@ -480,7 +479,7 @@ async function eraseDeletedNotes() {
         SET content = NULL,
             utcDateModified = '${utcNowDateTime}'
         WHERE noteRevisionId IN 
-            (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN ((???)))`, noteIdsToErase);
+            (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase);
 
     await sql.executeMany(`
         UPDATE note_revisions 
@@ -514,7 +513,7 @@ async function duplicateNote(noteId, parentNoteId) {
         notePosition: origBranch ? origBranch.notePosition + 1 : null
     }).save();
 
-    for (const attribute of await origNote.getAttributes()) {
+    for (const attribute of await origNote.getOwnedAttributes()) {
         const attr = new Attribute(attribute);
         attr.attributeId = undefined; // force creation of new attribute
         attr.noteId = newNote.noteId;

src/services/protected_session.js

@@ -37,7 +37,7 @@ function isProtectedSessionAvailable() {
 function decryptNotes(notes) {
     for (const note of notes) {
         if (note.isProtected) {
-            note.title = decrypt(note.title);
+            note.title = decryptString(note.title);
         }
     }
 }

src/services/search.js

@@ -35,9 +35,9 @@ async function searchForNoteIds(searchString) {
             }
         }
 
-        const isInFilter = filters.find(filter => filter.name.toLowerCase() === 'in');
+        const isInFilters = filters.filter(filter => filter.name.toLowerCase() === 'in');
 
-        if (isInFilter) {
+        for (const isInFilter of isInFilters) {
             if (isInFilter.operator === '=') {
                 noteIds = noteIds.filter(noteId => noteCacheService.isInAncestor(noteId, isInFilter.value));
             }

src/services/sql_init.js

@@ -57,8 +57,6 @@ async function initDbConnection() {
             return;
         }
 
-        await sql.execute("PRAGMA foreign_keys = ON");
-
         const currentDbVersion = await getDbVersion();
 
         if (currentDbVersion > appInfo.dbVersion) {
@@ -175,9 +173,11 @@ async function isDbUpToDate() {
 }
 
 async function dbInitialized() {
-    await optionService.setOption('initialized', 'true');
+    if (!await isDbInitialized()) {
+        await optionService.setOption('initialized', 'true');
 
-    await initDbConnection();
+        await initDbConnection();
+    }
 }
 
 dbReady.then(async () => {

src/tools/generate_document.js

@@ -1,48 +1,32 @@
-const fs = require('fs');
-const dataDir = require('../services/data_dir');
-
-fs.unlinkSync(dataDir.DOCUMENT_PATH);
+/**
+ * Usage: node src/tools/generate_document.js 1000
+ * will create 1000 new notes and some clones into a current document.db
+ */
 
 require('../entities/entity_constructor');
-const optionService = require('../services/options');
 const sqlInit = require('../services/sql_init');
-const myScryptService = require('../services/my_scrypt');
-const passwordEncryptionService = require('../services/password_encryption');
-const utils = require('../services/utils');
 const noteService = require('../services/notes');
+const attributeService = require('../services/attributes');
 const cls = require('../services/cls');
 const cloningService = require('../services/cloning');
-const loremIpsum = require('lorem-ipsum');
-
-async function setUserNamePassword() {
-    const username = "test";
-    const password = "test";
-
-    await optionService.setOption('username', username);
-
-    await optionService.setOption('passwordVerificationSalt', utils.randomSecureToken(32));
-    await optionService.setOption('passwordDerivedKeySalt', utils.randomSecureToken(32));
-
-    const passwordVerificationKey = utils.toBase64(await myScryptService.getVerificationHash(password));
-    await optionService.setOption('passwordVerificationHash', passwordVerificationKey);
-
-    await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16));
-
-    await sqlInit.initDbConnection();
-}
+const loremIpsum = require('lorem-ipsum').loremIpsum;
 
 const noteCount = parseInt(process.argv[2]);
+
+if (!noteCount) {
+    console.error(`Please enter number of notes as program parameter.`);
+    process.exit(1);
+}
+
 const notes = ['root'];
 
-function getRandomParentNoteId() {
+function getRandomNoteId() {
     const index = Math.floor(Math.random() * notes.length);
 
     return notes[index];
 }
 
 async function start() {
-    await setUserNamePassword();
-
     for (let i = 0; i < noteCount; i++) {
         const title = loremIpsum({ count: 1, units: 'sentences', sentenceLowerBound: 1, sentenceUpperBound: 10 });
 
@@ -50,17 +34,17 @@ async function start() {
         const content = loremIpsum({ count: paragraphCount, units: 'paragraphs', sentenceLowerBound: 1, sentenceUpperBound: 15,
             paragraphLowerBound: 3, paragraphUpperBound: 10, format: 'html' });
 
-        const {note} = await noteService.createNote(getRandomParentNoteId(), title, content);
+        const {note} = await noteService.createNote(getRandomNoteId(), title, content);
 
         console.log(`Created note ${i}: ${title}`);
 
         notes.push(note.noteId);
     }
 
-    // we'll create clones for 20% of notes
-    for (let i = 0; i < (noteCount / 50); i++) {
-        const noteIdToClone = getRandomParentNoteId();
-        const parentNoteId = getRandomParentNoteId();
+    // we'll create clones for 4% of notes
+    for (let i = 0; i < (noteCount / 25); i++) {
+        const noteIdToClone = getRandomNoteId();
+        const parentNoteId = getRandomNoteId();
         const prefix = Math.random() > 0.8 ? "prefix" : null;
 
         const result = await cloningService.cloneNoteToParent(noteIdToClone, parentNoteId, prefix);
@@ -68,6 +52,30 @@ async function start() {
         console.log(`Cloning ${i}:`, result.success ? "succeeded" : "FAILED");
     }
 
+    for (let i = 0; i < noteCount; i++) {
+        await attributeService.createAttribute({
+            noteId: getRandomNoteId(),
+            type: 'label',
+            name: 'label',
+            value: 'value',
+            isInheritable: Math.random() > 0.1 // 10% are inheritable
+        });
+
+        console.log(`Creating label ${i}`);
+    }
+
+    for (let i = 0; i < noteCount; i++) {
+        await attributeService.createAttribute({
+            noteId: getRandomNoteId(),
+            type: 'relation',
+            name: 'relation',
+            value: getRandomNoteId(),
+            isInheritable: Math.random() > 0.1 // 10% are inheritable
+        });
+
+        console.log(`Creating relation ${i}`);
+    }
+
     process.exit(0);
 }
 

src/views/setup.ejs

@@ -138,7 +138,6 @@
 </div>
 
 <script type="text/javascript">
-    const baseApiUrl = 'api/';
     const glob = {
         sourceId: ''
     };