release 0.45.8

config-sample.ini

@@ -1,5 +1,5 @@
 [General]
-# Instance name can be used to distinguish between different instances
+# Instance name can be used to distinguish between different instances using backend api.getInstanceName()
 instanceName=
 
 # set to true to allow using Trilium without authentication (makes sense for server build only, desktop build doesn't need password)

libraries/ckeditor/ckeditor-content.css

@@ -1,12 +1,12 @@
 /* !!!!!! TRILIUM CUSTOM CHANGES !!!!!! */
 
-.ck-widget__type-around { /* gets rid of triangles: https://github.com/zadam/trilium/issues/1129 */
+.printed-content .ck-widget__selection-handle, .printed-content .ck-widget__type-around { /* gets rid of triangles: https://github.com/zadam/trilium/issues/1129 */
     display: none;
 }
 
 /*
- * CKEditor 5 (v22.0.0) content styles.
+ * CKEditor 5 (v23.1.0) content styles.
- * Generated on Thu, 27 Aug 2020 12:13:06 GMT.
+ * Generated on Thu, 29 Oct 2020 12:17:48 GMT.
  * For more information, check out https://ckeditor.com/docs/ckeditor5/latest/builds/guides/integration/content-styles.html
  */
 
@@ -23,32 +23,6 @@
     --ck-todo-list-checkmark-size: 16px;
 }
 
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-yellow {
-    background-color: var(--ck-highlight-marker-yellow);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-green {
-    background-color: var(--ck-highlight-marker-green);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-pink {
-    background-color: var(--ck-highlight-marker-pink);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-blue {
-    background-color: var(--ck-highlight-marker-blue);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .pen-red {
-    color: var(--ck-highlight-pen-red);
-    background-color: transparent;
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .pen-green {
-    color: var(--ck-highlight-pen-green);
-    background-color: transparent;
-}
 /* ckeditor5-image/theme/imagestyle.css */
 .ck-content .image-style-side {
     float: right;
@@ -84,6 +58,17 @@
     max-width: 100%;
     min-width: 50px;
 }
+/* ckeditor5-image/theme/imagecaption.css */
+.ck-content .image > figcaption {
+    display: table-caption;
+    caption-side: bottom;
+    word-break: break-word;
+    color: hsl(0, 0%, 20%);
+    background-color: hsl(0, 0%, 97%);
+    padding: .6em;
+    font-size: .75em;
+    outline-offset: -1px;
+}
 /* ckeditor5-image/theme/imageresize.css */
 .ck-content .image.image_resized {
     max-width: 100%;
@@ -98,22 +83,31 @@
 .ck-content .image.image_resized > figcaption {
     display: block;
 }
-/* ckeditor5-image/theme/imagecaption.css */
+/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .image > figcaption {
+.ck-content .marker-yellow {
-    display: table-caption;
+    background-color: var(--ck-highlight-marker-yellow);
-    caption-side: bottom;
-    word-break: break-word;
-    color: hsl(0, 0%, 20%);
-    background-color: hsl(0, 0%, 97%);
-    padding: .6em;
-    font-size: .75em;
-    outline-offset: -1px;
 }
-/* ckeditor5-basic-styles/theme/code.css */
+/* ckeditor5-highlight/theme/highlight.css */
-.ck-content code {
+.ck-content .marker-green {
-    background-color: hsla(0, 0%, 78%, 0.3);
+    background-color: var(--ck-highlight-marker-green);
-    padding: .15em;
+}
-    border-radius: 2px;
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-pink {
+    background-color: var(--ck-highlight-marker-pink);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-blue {
+    background-color: var(--ck-highlight-marker-blue);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .pen-red {
+    color: var(--ck-highlight-pen-red);
+    background-color: transparent;
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .pen-green {
+    color: var(--ck-highlight-pen-green);
+    background-color: transparent;
 }
 /* ckeditor5-font/theme/fontsize.css */
 .ck-content .text-tiny {
@@ -146,6 +140,12 @@
     border-left: 0;
     border-right: solid 5px hsl(0, 0%, 80%);
 }
+/* ckeditor5-basic-styles/theme/code.css */
+.ck-content code {
+    background-color: hsla(0, 0%, 78%, 0.3);
+    padding: .15em;
+    border-radius: 2px;
+}
 /* ckeditor5-table/theme/table.css */
 .ck-content .table {
     margin: 1em auto;
@@ -215,13 +215,6 @@
     -ms-user-select: none;
     user-select: none;
 }
-/* ckeditor5-media-embed/theme/mediaembed.css */
-.ck-content .media {
-    clear: both;
-    margin: 1em 0;
-    display: block;
-    min-width: 15em;
-}
 /* ckeditor5-list/theme/todolist.css */
 .ck-content .todo-list {
     list-style: none;
@@ -289,6 +282,18 @@
 .ck-content .todo-list .todo-list__label .todo-list__label__description {
     vertical-align: middle;
 }
+/* ckeditor5-media-embed/theme/mediaembed.css */
+.ck-content .media {
+    clear: both;
+    margin: 1em 0;
+    display: block;
+    min-width: 15em;
+}
+/* ckeditor5-html-embed/theme/htmlembed.css */
+.ck-content .raw-html-embed {
+    margin: 1em auto;
+    min-width: 15em;
+}
 /* ckeditor5-horizontal-line/theme/horizontalline.css */
 .ck-content hr {
     margin: 15px 0;
@@ -330,4 +335,4 @@
     .ck-content .page-break::after {
         display: none;
     }
-}
+}

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.45.3",
+  "version": "0.45.6",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -2654,9 +2654,9 @@
       }
     },
     "electron": {
-      "version": "9.3.4",
+      "version": "9.3.5",
-      "resolved": "https://registry.npmjs.org/electron/-/electron-9.3.4.tgz",
+      "resolved": "https://registry.npmjs.org/electron/-/electron-9.3.5.tgz",
-      "integrity": "sha512-OHP8qMKgW8D8GtH+altB22WJw/lBOyyVdoz5e8D0/iPBmJU3Jm93vO4z4Eh/9DvdSXlH8bMHUCMLL9PVW6f+tw==",
+      "integrity": "sha512-EPmDsp7sO0UPtw7nLD1ufse/nBskP+ifXzBgUg9psCUlapkzuwYi6pmLAzKLW/bVjwgyUKwh1OKWILWfOeLGcQ==",
       "dev": true,
       "requires": {
         "@electron/get": "^1.0.1",
@@ -4838,6 +4838,11 @@
         "type-check": "~0.3.2"
       }
     },
+    "limiter": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz",
+      "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA=="
+    },
     "line-column": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz",
@@ -6913,6 +6918,22 @@
       "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
       "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
     },
+    "stream-throttle": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz",
+      "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=",
+      "requires": {
+        "commander": "^2.2.0",
+        "limiter": "^1.0.5"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "2.20.3",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+          "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        }
+      }
+    },
     "streamsearch": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",

package.json

@@ -2,7 +2,7 @@
   "name": "trilium",
   "productName": "Trilium Notes",
   "description": "Trilium Notes",
-  "version": "0.45.4",
+  "version": "0.45.8",
   "license": "AGPL-3.0-only",
   "main": "electron.js",
   "bin": {
@@ -65,6 +65,7 @@
     "semver": "7.3.2",
     "serve-favicon": "2.5.0",
     "session-file-store": "1.5.0",
+    "stream-throttle": "^0.1.3",
     "striptags": "3.1.1",
     "tmp": "^0.2.1",
     "turndown": "7.0.0",
@@ -76,7 +77,7 @@
   },
   "devDependencies": {
     "cross-env": "7.0.2",
-    "electron": "9.3.4",
+    "electron": "9.3.5",
     "electron-builder": "22.9.1",
     "electron-packager": "15.1.0",
     "electron-rebuild": "2.3.2",

src/entities/branch.js

@@ -38,7 +38,7 @@ class Branch extends Entity {
     }
 
     beforeSaving() {
-        if (this.notePosition === undefined) {
+        if (this.notePosition === undefined || this.notePosition === null) {
             const maxNotePos = sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [this.parentNoteId]);
             this.notePosition = maxNotePos === null ? 0 : maxNotePos + 10;
         }

src/public/app/dialogs/options/other.js

@@ -51,6 +51,12 @@ const TPL = `
         <label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label>
         <input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0">
     </div>
+    
+    <p>You can also trigger erasing manually:</p>
+    
+    <button id="erase-deleted-notes-now-button" class="btn">Erase deleted notes now</button>
+    
+    <br/><br/>
 </div>
 
 <div>
@@ -117,6 +123,13 @@ export default class ProtectedSessionOptions {
             return false;
         });
 
+        this.$eraseDeletedNotesButton = $("#erase-deleted-notes-now-button");
+        this.$eraseDeletedNotesButton.on('click', () => {
+            server.post('notes/erase-deleted-notes-now').then(() => {
+                toastService.showMessage("Deleted notes have been erased.");
+            });
+        });
+
         this.$protectedSessionTimeout = $("#protected-session-timeout-in-seconds");
 
         this.$protectedSessionTimeout.on('change', () => {

src/public/app/entities/note_short.js

@@ -75,14 +75,16 @@ class NoteShort {
         this.parentToBranch[parentNoteId] = branchId;
     }
 
-    addChild(childNoteId, branchId) {
+    addChild(childNoteId, branchId, sort = true) {
         if (!this.children.includes(childNoteId)) {
             this.children.push(childNoteId);
         }
 
         this.childToBranch[childNoteId] = branchId;
 
-        this.sortChildren();
+        if (sort) {
+            this.sortChildren();
+        }
     }
 
     sortChildren() {

src/public/app/services/frontend_script_api.js

@@ -274,7 +274,10 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
      *
      * @method
      * @param {string} notePath (or noteId)
-     * @param {string} [noteTitle] - if not present we'll use note title
+     * @param {object} [params]
+     * @param {boolean} [params.showTooltip=true] - enable/disable tooltip on the link
+     * @param {boolean} [params.showNotePath=false] - show also whole note's path as part of the link
+     * @param {string} [title=] - custom link tile with note's title as default
      */
     this.createNoteLink = linkService.createNoteLink;
 

src/public/app/services/link.js

@@ -130,7 +130,7 @@ function linkContextMenu(e) {
                 appContext.tabManager.openTabWithNote(notePath);
             }
             else if (command === 'openNoteInNewWindow') {
-                appContext.openInNewWindow(notePath);
+                appContext.triggerCommand('openInWindow', {notePath});
             }
         }
     });

src/public/app/services/note_create.js

@@ -88,7 +88,7 @@ function parseSelectedHtml(selectedHtml) {
     }
 }
 
-async function duplicateNote(noteId, parentNoteId) {
+async function duplicateSubtree(noteId, parentNoteId) {
     const {note} = await server.post(`notes/${noteId}/duplicate/${parentNoteId}`);
 
     await ws.waitForMaxKnownEntityChangeId();
@@ -102,5 +102,5 @@ async function duplicateNote(noteId, parentNoteId) {
 export default {
     createNote,
     createNewTopLevelNote,
-    duplicateNote
+    duplicateSubtree
 };

src/public/app/services/sync.js

@@ -8,8 +8,8 @@ async function syncNow() {
         toastService.showMessage("Sync finished successfully.");
     }
     else {
-        if (result.message.length > 100) {
+        if (result.message.length > 200) {
-            result.message = result.message.substr(0, 100);
+            result.message = result.message.substr(0, 200) + "...";
         }
 
         toastService.showError("Sync failed: " + result.message);

src/public/app/services/tree_cache.js

@@ -87,6 +87,8 @@ class TreeCache {
         const branchRows = resp.branches;
         const attributeRows = resp.attributes;
 
+        const noteIdsToSort = new Set();
+
         for (const noteRow of noteRows) {
             const {noteId} = noteRow;
 
@@ -153,7 +155,9 @@ class TreeCache {
             const parentNote = this.notes[branch.parentNoteId];
 
             if (parentNote) {
-                parentNote.addChild(branch.noteId, branch.branchId);
+                parentNote.addChild(branch.noteId, branch.branchId, false);
+
+                noteIdsToSort.add(parentNote.noteId);
             }
         }
 
@@ -178,6 +182,11 @@ class TreeCache {
                 }
             }
         }
+
+        // sort all of them at once, this avoids repeated sorts (#1480)
+        for (const noteId of noteIdsToSort) {
+            this.notes[noteId].sortChildren();
+        }
     }
 
     async reloadNotes(noteIds) {

src/public/app/services/tree_context_menu.js

@@ -2,9 +2,9 @@ import treeService from './tree.js';
 import treeCache from "./tree_cache.js";
 import hoistedNoteService from './hoisted_note.js';
 import clipboard from './clipboard.js';
-import protectedSessionHolder from "./protected_session_holder.js";
 import noteCreateService from "./note_create.js";
 import contextMenu from "./context_menu.js";
+import appContext from "./app_context.js";
 
 class TreeContextMenu {
     /**
@@ -95,7 +95,7 @@ class TreeContextMenu {
                 enabled: !clipboard.isClipboardEmpty() && notSearch && noSelectedNotes },
             { title: 'Paste after', command: "pasteNotesAfterFromClipboard", uiIcon: "paste",
                 enabled: !clipboard.isClipboardEmpty() && isNotRoot && !isHoisted && parentNotSearch && noSelectedNotes },
-            { title: "Duplicate note(s) here", command: "duplicateNote", uiIcon: "empty",
+            { title: `Duplicate subtree <kbd data-command="duplicateSubtree">`, command: "duplicateSubtree", uiIcon: "empty",
                 enabled: parentNotSearch && isNotRoot && !isHoisted },
             { title: "----" },
             { title: "Export", command: "exportNote", uiIcon: "empty",
@@ -110,14 +110,7 @@ class TreeContextMenu {
         const notePath = treeService.getNotePath(this.node);
 
         if (command === 'openInTab') {
-
+            appContext.tabManager.openTabWithNote(notePath);
-            const start = Date.now();
-
-            await this.node.load(true);
-
-            console.log("Reload took", Date.now() - start, "ms");
-
-//            appContext.tabManager.openTabWithNote(notePath);
         }
         else if (command === "insertNoteAfter") {
             const parentNoteId = this.node.data.parentNoteId;

src/public/app/services/ws.js

@@ -8,8 +8,6 @@ import options from "./options.js";
 import treeCache from "./tree_cache.js";
 import noteAttributeCache from "./note_attribute_cache.js";
 
-const $outstandingSyncsCount = $("#outstanding-syncs-count");
-
 const messageHandlers = [];
 
 let ws;
@@ -64,8 +62,6 @@ async function handleMessage(event) {
         let syncRows = message.data;
         lastPingTs = Date.now();
 
-        $outstandingSyncsCount.html(message.outstandingSyncs);
-
         if (syncRows.length > 0) {
             logRows(syncRows);
 

src/public/app/setup.js

@@ -130,7 +130,7 @@ function SetupModel() {
 }
 
 async function checkOutstandingSyncs() {
-    const { stats, initialized } = await $.get('api/sync/stats');
+    const { outstandingPullCount, initialized } = await $.get('api/sync/stats');
 
     if (initialized) {
         if (utils.isElectron()) {
@@ -143,9 +143,7 @@ async function checkOutstandingSyncs() {
         }
     }
     else {
-        const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;
+        $("#outstanding-syncs").html(outstandingPullCount);
-
-        $("#outstanding-syncs").html(totalOutstandingSyncs);
     }
 }
 

src/public/app/widgets/attribute_list.js

@@ -214,7 +214,8 @@ export default class AttributeListWidget extends TabAwareWidget {
                         noteId: attribute.noteId,
                         type: attribute.type,
                         name: attribute.name,
-                        value: attribute.value
+                        value: attribute.value,
+                        isInheritable: attribute.isInheritable
                     },
                     isOwned: false,
                     x: e.pageX,

src/public/app/widgets/global_menu.js

@@ -41,7 +41,7 @@ const TPL = `
 
             <a class="dropdown-item sync-now-button" title="Trigger sync">
                 <span class="bx bx-refresh"></span>
-                Sync now (<span id="outstanding-syncs-count">0</span>)
+                Sync now
             </a>
 
             <a class="dropdown-item" data-trigger-command="openNewWindow">

src/public/app/widgets/note_detail.js

@@ -248,13 +248,22 @@ export default class NoteDetailWidget extends TabAwareWidget {
 
         this.$widget.find('.note-detail-printable:visible').printThis({
             header: $("<h2>").text(this.note && this.note.title).prop('outerHTML'),
-            footer: "<script>document.body.className += ' ck-content';</script>",
+            footer: `
+<script src="libraries/katex/katex.min.js"></script>
+<script src="libraries/katex/auto-render.min.js"></script>
+<script>
+    document.body.className += ' ck-content printed-content';
+    
+    renderMathInElement(document.body, {});
+</script>
+`,
             importCSS: false,
             loadCSS: [
                 "libraries/codemirror/codemirror.css",
                 "libraries/ckeditor/ckeditor-content.css",
                 "libraries/ckeditor/ckeditor-content.css",
                 "libraries/bootstrap/css/bootstrap.min.css",
+                "libraries/katex/katex.min.css",
                 "stylesheets/print.css",
                 "stylesheets/relation_map.css",
                 "stylesheets/themes.css"

src/public/app/widgets/note_tree.js

@@ -1341,7 +1341,7 @@ export default class NoteTreeWidget extends TabAwareWidget {
         protectedSessionService.protectNote(node.data.noteId, false, true);
     }
 
-    duplicateNoteCommand({node}) {
+    duplicateSubtreeCommand({node}) {
         const nodesToDuplicate = this.getSelectedOrActiveNodes(node);
 
         for (const nodeToDuplicate of nodesToDuplicate) {
@@ -1353,7 +1353,7 @@ export default class NoteTreeWidget extends TabAwareWidget {
 
             const branch = treeCache.getBranch(nodeToDuplicate.data.branchId);
 
-            noteCreateService.duplicateNote(nodeToDuplicate.data.noteId, branch.parentNoteId);
+            noteCreateService.duplicateSubtree(nodeToDuplicate.data.noteId, branch.parentNoteId);
         }
     }
 }

src/public/app/widgets/type_widgets/editable_text.js

@@ -38,7 +38,7 @@ const TPL = `
         cursor: text !important;
     }
     
-    .note-detail-editable-text *:first-child {
+    .note-detail-editable-text *:not(figure):first-child {
         margin-top: 0 !important;
     }
     

src/public/stylesheets/themes.css

@@ -143,6 +143,11 @@ body {
     --ck-color-dropdown-panel-background: var(--accented-background-color);
     --ck-color-dropdown-panel-border: var(--main-border-color);
 
+    /* -- Overrides the default .ck-splitbutton class colors. ----------------------------------- */
+
+    --ck-color-split-button-hover-background: var(--ck-color-button-default-hover-background);
+    --ck-color-split-button-hover-border: var(--main-border-color);
+
     /* -- Overrides the default .ck-input class colors. ----------------------------------------- */
 
     --ck-color-input-background: var(--accented-background-color);
@@ -199,6 +204,9 @@ body {
     --ck-color-engine-placeholder-text: var(--muted-text-color);
 
     --ck-z-modal: 10000;
+
+    --ck-color-widget-type-around-button: var(--main-border-color);
+    --ck-color-widget-type-around-button-hover: var(--main-border-color);
 }
 
 body {

src/routes/api/export.js

@@ -23,11 +23,7 @@ function exportBranch(req, res) {
 
     try {
         if (type === 'subtree' && (format === 'html' || format === 'markdown')) {
-            const start = Date.now();
-
             zipExportService.exportToZip(taskContext, branch, format, res);
-
-            console.log("Export took", Date.now() - start, "ms");
         }
         else if (type === 'single') {
             singleExportService.exportSingleNote(taskContext, branch, format, res);

src/routes/api/link_map.js

@@ -3,15 +3,33 @@
 const sql = require('../../services/sql');
 
 function getRelations(noteIds) {
-    return (sql.getManyRows(`
+    noteIds = Array.from(noteIds);
-        SELECT noteId, name, value AS targetNoteId
+
-        FROM attributes
+    return [
-        WHERE (noteId IN (???) OR value IN (???))
+        // first read all non-image relations
-          AND type = 'relation'
+        ...sql.getManyRows(`
-          AND isDeleted = 0
+            SELECT noteId, name, value AS targetNoteId
-          AND noteId != ''
+            FROM attributes
-          AND value != ''
+            WHERE (noteId IN (???) OR value IN (???))
-    `, Array.from(noteIds)));
+              AND type = 'relation'
+              AND name != 'imageLink'
+              AND isDeleted = 0
+              AND noteId != ''
+              AND value != ''`, noteIds),
+        // ... then read only imageLink relations which are not connecting parent and child
+        // this is done to not show image links in the trivial case where they are direct children of the note to which they are included. Same heuristic as in note tree
+        ...sql.getManyRows(`
+            SELECT rel.noteId, rel.name, rel.value AS targetNoteId
+            FROM attributes AS rel
+            LEFT JOIN branches ON branches.parentNoteId = rel.noteId AND branches.noteId = rel.value AND branches.isDeleted = 0 
+            WHERE (rel.noteId IN (???) OR rel.value IN (???))
+              AND rel.type = 'relation'
+              AND rel.name = 'imageLink'
+              AND rel.isDeleted = 0
+              AND rel.noteId != ''
+              AND rel.value != ''
+              AND branches.branchId IS NULL`, noteIds)
+    ];
 }
 
 function getLinkMap(req) {

src/routes/api/notes.js

@@ -187,10 +187,14 @@ function changeTitle(req) {
     return note;
 }
 
-function duplicateNote(req) {
+function duplicateSubtree(req) {
     const {noteId, parentNoteId} = req.params;
 
-    return noteService.duplicateNote(noteId, parentNoteId);
+    return noteService.duplicateSubtree(noteId, parentNoteId);
+}
+
+function eraseDeletedNotesNow() {
+    noteService.eraseDeletedNotesNow();
 }
 
 module.exports = {
@@ -204,5 +208,6 @@ module.exports = {
     setNoteTypeMime,
     getRelationMap,
     changeTitle,
-    duplicateNote
+    duplicateSubtree,
+    eraseDeletedNotesNow
 };

src/routes/api/recent_changes.js

@@ -31,19 +31,36 @@ function getRecentChanges(req) {
         }
     }
 
+    // now we need to also collect date points not represented in note revisions:
+    // 1. creation for all notes (dateCreated)
+    // 2. deletion for deleted notes (dateModified)
     const notes = sql.getRows(`
-        SELECT
+            SELECT
-            notes.noteId,
+                notes.noteId,
-            notes.isDeleted AS current_isDeleted,
+                notes.isDeleted AS current_isDeleted,
-            notes.deleteId AS current_deleteId,
+                notes.deleteId AS current_deleteId,
-            notes.isErased AS current_isErased,
+                notes.isErased AS current_isErased,
-            notes.title AS current_title,
+                notes.title AS current_title,
-            notes.isProtected AS current_isProtected,
+                notes.isProtected AS current_isProtected,
-            notes.title,
+                notes.title,
-            notes.utcDateCreated AS utcDate,
+                notes.utcDateCreated AS utcDate,
-            notes.dateCreated AS date
+                notes.dateCreated AS date
-        FROM
+            FROM
-            notes`);
+                notes
+        UNION ALL
+            SELECT
+                notes.noteId,
+                notes.isDeleted AS current_isDeleted,
+                notes.deleteId AS current_deleteId,
+                notes.isErased AS current_isErased,
+                notes.title AS current_title,
+                notes.isProtected AS current_isProtected,
+                notes.title,
+                notes.utcDateModified AS utcDate,
+                notes.dateModified AS date
+            FROM
+                notes
+            WHERE notes.isDeleted = 1 AND notes.isErased = 0`);
 
     for (const note of notes) {
         if (noteCacheService.isInAncestor(note.noteId, ancestorNoteId)) {

src/routes/api/sender.js

@@ -4,6 +4,7 @@ const imageType = require('image-type');
 const imageService = require('../../services/image');
 const dateNoteService = require('../../services/date_notes');
 const noteService = require('../../services/notes');
+const attributeService = require('../../services/attributes');
 
 function uploadImage(req) {
     const file = req.file;
@@ -37,7 +38,7 @@ function saveNote(req) {
 
     if (req.body.labels) {
         for (const {name, value} of req.body.labels) {
-            note.setLabel(name, value);
+            note.setLabel(attributeService.sanitizeAttributeName(name), value);
         }
     }
 

src/routes/api/setup.js

@@ -38,6 +38,8 @@ function saveSyncSeed(req) {
         }]
     }
 
+    log.info("Saved sync seed.");
+
     sqlInit.createDatabaseForSync(options);
 }
 

src/routes/api/sync.js

@@ -13,13 +13,13 @@ const dateUtils = require('../../services/date_utils');
 const entityConstructor = require('../../entities/entity_constructor');
 const utils = require('../../services/utils');
 
-function testSync() {
+async function testSync() {
     try {
         if (!syncOptions.isSyncSetup()) {
             return { success: false, message: "Sync server host is not configured. Please configure sync first." };
         }
 
-        syncService.login();
+        await syncService.login();
 
         // login was successful so we'll kick off sync now
         // this is important in case when sync server has been just initialized
@@ -43,7 +43,7 @@ function getStats() {
 
     const stats = {
         initialized: optionService.getOption('initialized') === 'true',
-        stats: syncService.stats
+        outstandingPullCount: syncService.getOutstandingPullCount()
     };
 
     log.info(`Returning sync stats: ${JSON.stringify(stats)}`);

src/routes/api/tree.js

@@ -57,7 +57,7 @@ function getTree(req) {
     const noteIds = sql.getColumn(`
         WITH RECURSIVE
             treeWithDescendants(noteId, isExpanded) AS (
-                SELECT noteId, 1 FROM branches WHERE parentNoteId = ? AND isDeleted = 0
+                SELECT noteId, isExpanded FROM branches WHERE parentNoteId = ? AND isDeleted = 0
                 UNION
                 SELECT branches.noteId, branches.isExpanded FROM branches
                   JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId

src/routes/routes.js

@@ -153,8 +153,9 @@ function register(app) {
     route(GET, '/api/notes/:noteId/revisions/:noteRevisionId/download', [auth.checkApiAuthOrElectron], noteRevisionsApiRoute.downloadNoteRevision);
     apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision);
     apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap);
+    apiRoute(POST, '/api/notes/erase-deleted-notes-now', notesApiRoute.eraseDeletedNotesNow);
     apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle);
-    apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateNote);
+    apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree);
 
     apiRoute(GET, '/api/edited-notes/:date', noteRevisionsApiRoute.getEditedNotesOnDate);
 

src/services/attributes.js

@@ -2,7 +2,6 @@
 
 const repository = require('./repository');
 const sql = require('./sql');
-const utils = require('./utils');
 const Attribute = require('../entities/attribute');
 
 const ATTRIBUTE_TYPES = [ 'label', 'relation' ];
@@ -146,6 +145,20 @@ function getBuiltinAttributeNames() {
         ]);
 }
 
+function sanitizeAttributeName(origName) {
+    let fixedName;
+
+    if (origName === '') {
+        fixedName = "unnamed";
+    }
+    else {
+        // any not allowed character should be replaced with underscore
+        fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
+    }
+
+    return fixedName;
+}
+
 module.exports = {
     getNotesWithLabel,
     getNotesWithLabels,
@@ -156,5 +169,6 @@ module.exports = {
     getAttributeNames,
     isAttributeType,
     isAttributeDangerous,
-    getBuiltinAttributeNames
+    getBuiltinAttributeNames,
+    sanitizeAttributeName
 };

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2020-11-12T22:15:23+01:00", buildRevision: "6c57b2220ff05059d7460369b195d281fcd9cbb6" };
+module.exports = { buildDate:"2021-01-11T22:29:31+01:00", buildRevision: "369274ead75947a68bf7bbb5ab1e784e81521030" };

src/services/consistency_checks.js

@@ -11,6 +11,7 @@ const entityChangesService = require('./entity_changes.js');
 const optionsService = require('./options');
 const Branch = require('../entities/branch');
 const dateUtils = require('./date_utils');
+const attributeService = require('./attributes');
 
 class ConsistencyChecks {
     constructor(autoFix) {
@@ -607,20 +608,10 @@ class ConsistencyChecks {
     findWronglyNamedAttributes() {
         const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`);
 
-        const attrNameMatcher = new RegExp("^[\\p{L}\\p{N}_:]+$", "u");
-
         for (const origName of attrNames) {
-            if (!attrNameMatcher.test(origName)) {
+            const fixedName = attributeService.sanitizeAttributeName(origName);
-                let fixedName;
-
-                if (origName === '') {
-                    fixedName = "unnamed";
-                }
-                else {
-                    // any not allowed character should be replaced with underscore
-                    fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
-                }
 
+            if (fixedName !== origName) {
                 if (this.autoFix) {
                     // there isn't a good way to update this:
                     // - just SQL query will fix it in DB but not notify frontend (or other caches) that it has been fixed
@@ -659,7 +650,7 @@ class ConsistencyChecks {
         // root branch should always be expanded
         sql.execute("UPDATE branches SET isExpanded = 1 WHERE branchId = 'root'");
 
-        if (this.unrecoveredConsistencyErrors) {
+        if (!this.unrecoveredConsistencyErrors) {
             // we run this only if basic checks passed since this assumes basic data consistency
 
             this.checkTreeCycles();

src/services/data_encryption.js

@@ -52,6 +52,10 @@ function encrypt(key, plainText, ivLength = 13) {
 }
 
 function decrypt(key, cipherText, ivLength = 13) {
+    if (cipherText === null) {
+        return null;
+    }
+
     if (!key) {
         return "[protected]";
     }
@@ -93,6 +97,10 @@ function decrypt(key, cipherText, ivLength = 13) {
 function decryptString(dataKey, cipherText) {
     const buffer = decrypt(dataKey, cipherText);
 
+    if (buffer === null) {
+        return null;
+    }
+
     const str = buffer.toString('utf-8');
 
     if (str === 'false') {
@@ -108,4 +116,4 @@ module.exports = {
     encrypt,
     decrypt,
     decryptString
-};
+};

src/services/export/zip.js

@@ -143,7 +143,7 @@ function exportToZip(taskContext, branch, format, res) {
         const available = !note.isProtected || protectedSessionService.isProtectedSessionAvailable();
 
         // if it's a leaf then we'll export it even if it's empty
-        if (available && ((note.getContent()).length > 0 || childBranches.length === 0)) {
+        if (available && (note.getContent().length > 0 || childBranches.length === 0)) {
             meta.dataFileName = getDataFileName(note, baseFileName, existingFileNames);
         }
 
@@ -234,7 +234,7 @@ function exportToZip(taskContext, branch, format, res) {
     <link rel="stylesheet" href="${cssUrl}">
     <base target="_parent">
 </head>
-<body>
+<body class="ck-content">
   <h1>${utils.escapeHtml(title)}</h1>
 ${content}
 </body>
@@ -433,14 +433,13 @@ ${content}
     }
 
     const note = branch.getNote();
-    const zipFileName = (branch.prefix ? (branch.prefix + " - ") : "") + note.title + ".zip";
+    const zipFileName = (branch.prefix ? `${branch.prefix} - ` : "") + note.title + ".zip";
 
     res.setHeader('Content-Disposition', utils.getContentDisposition(zipFileName));
     res.setHeader('Content-Type', 'application/zip');
 
-    zipFile.end();
-
     zipFile.outputStream.pipe(res);
+    zipFile.end();
 
     taskContext.taskSucceeded();
 }

src/services/handlers.js

@@ -1,7 +1,7 @@
 const eventService = require('./events');
 const scriptService = require('./script');
 const treeService = require('./tree');
-const log = require('./log');
+const noteService = require('./notes');
 const repository = require('./repository');
 const Attribute = require('../entities/attribute');
 
@@ -58,17 +58,25 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) =>
                 return;
             }
 
-            const targetNote = repository.getNote(entity.value);
+            const templateNote = repository.getNote(entity.value);
 
-            if (!targetNote || !targetNote.isStringNote()) {
+            if (!templateNote) {
                 return;
             }
 
-            const targetNoteContent = targetNote.getContent();
+            if (templateNote.isStringNote()) {
+                const templateNoteContent = templateNote.getContent();
 
-            if (targetNoteContent) {
+                if (templateNoteContent) {
-                note.setContent(targetNoteContent);
+                    note.setContent(templateNoteContent);
+                }
+
+                note.type = templateNote.type;
+                note.mime = templateNote.mime;
+                note.save();
             }
+
+            noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId);
         }
         else if (entity.type === 'label' && entity.name === 'sorted') {
             treeService.sortNotesAlphabetically(entity.noteId);
@@ -86,10 +94,10 @@ eventService.subscribe(eventService.CHILD_NOTE_CREATED, ({ parentNote, childNote
 function processInverseRelations(entityName, entity, handler) {
     if (entityName === 'attributes' && entity.type === 'relation') {
         const note = entity.getNote();
-        const attributes = (note.getOwnedAttributes(entity.name)).filter(relation => relation.type === 'relation-definition');
+        const relDefinitions = note.getLabels('relation:' + entity.name);
 
-        for (const attribute of attributes) {
+        for (const relDefinition of relDefinitions) {
-            const definition = attribute.value;
+            const definition = relDefinition.getDefinition();
 
             if (definition.inverseRelation && definition.inverseRelation.trim()) {
                 const targetNote = entity.getTargetNote();

src/services/image.js

@@ -37,7 +37,7 @@ function getImageType(buffer) {
         }
     }
     else {
-        return imageType(buffer);
+        return imageType(buffer) || "jpg"; // optimistic JPG default
     }
 }
 

src/services/import/enex.js

@@ -1,5 +1,6 @@
 const sax = require("sax");
 const stream = require('stream');
+const {Throttle} = require('stream-throttle');
 const log = require("../log");
 const utils = require("../utils");
 const sql = require("../sql");
@@ -7,6 +8,7 @@ const noteService = require("../notes");
 const imageService = require("../image");
 const protectedSessionService = require('../protected_session');
 const htmlSanitizer = require("../html_sanitizer");
+const attributeService = require("../attributes");
 
 // date format is e.g. 20181121T193703Z
 function parseDate(text) {
@@ -37,10 +39,6 @@ function importEnex(taskContext, file, parentNote) {
         isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
     })).note;
 
-    // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished
-    // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully.
-    const saveNotePromises = [];
-
     function extractContent(content) {
         const openingNoteIndex = content.indexOf('<en-note>');
 
@@ -105,9 +103,17 @@ function importEnex(taskContext, file, parentNote) {
         const previousTag = getPreviousTag();
 
         if (previousTag === 'note-attributes') {
+            let labelName = currentTag;
+
+            if (labelName === 'source-url') {
+                labelName = 'sourceUrl';
+            }
+
+            labelName = attributeService.sanitizeAttributeName(labelName);
+
             note.attributes.push({
                 type: 'label',
-                name: currentTag,
+                name: labelName,
                 value: text
             });
         }
@@ -149,7 +155,7 @@ function importEnex(taskContext, file, parentNote) {
             } else if (currentTag === 'tag') {
                 note.attributes.push({
                     type: 'label',
-                    name: text,
+                    name: attributeService.sanitizeAttributeName(text),
                     value: ''
                 })
             }
@@ -227,6 +233,10 @@ function importEnex(taskContext, file, parentNote) {
         taskContext.increaseProgressCount();
 
         for (const resource of resources) {
+            if (!resource.content) {
+                continue;
+            }
+
             const hash = utils.md5(resource.content);
 
             const mediaRegex = new RegExp(`<en-media hash="${hash}"[^>]*>`, 'g');
@@ -304,7 +314,7 @@ function importEnex(taskContext, file, parentNote) {
         path.pop();
 
         if (tag === 'note') {
-            saveNotePromises.push(saveNote());
+            saveNote();
         }
     });
 
@@ -323,12 +333,15 @@ function importEnex(taskContext, file, parentNote) {
     return new Promise((resolve, reject) =>
     {
         // resolve only when we parse the whole document AND saving of all notes have been finished
-        saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) });
+        saxStream.on("end", () => resolve(rootNote));
 
         const bufferStream = new stream.PassThrough();
         bufferStream.end(file.buffer);
 
-        bufferStream.pipe(saxStream);
+        bufferStream
+            // rate limiting to improve responsiveness during / after import
+            .pipe(new Throttle({rate: 500000}))
+            .pipe(saxStream);
     });
 }
 

src/services/import/zip.js

@@ -158,7 +158,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             }
 
             if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) {
-                attr.name = 'disabled-' + attr.name;
+                attr.name = 'disabled:' + attr.name;
             }
 
             attributes.push(attr);

src/services/keyboard_actions.js

@@ -184,6 +184,12 @@ const DEFAULT_KEYBOARD_ACTIONS = [
         description: "Add note above to the selection",
         scope: "note-tree"
     },
+    {
+        actionName: "duplicateSubtree",
+        defaultShortcuts: [],
+        description: "Duplicate subtree",
+        scope: "note-tree"
+    },
 
 
     {
@@ -307,6 +313,12 @@ const DEFAULT_KEYBOARD_ACTIONS = [
         description: "Cuts the selection from the current note and creates subnote with the selected text",
         scope: "text-detail"
     },
+    {
+        actionName: "addIncludeNoteToText",
+        defaultShortcuts: [],
+        description: "Opens the dialog to include a note",
+        scope: "text-detail"
+    },
 
     {
         separator: "Attributes (labels & relations)"

src/services/note_cache/entities/note.js

@@ -338,7 +338,7 @@ class Note {
 
     decrypt() {
         if (this.isProtected && !this.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) {
-            this.title = protectedSessionService.decryptString(note.title);
+            this.title = protectedSessionService.decryptString(this.title);
 
             this.isDecrypted = true;
         }

src/services/note_revisions.js

@@ -2,6 +2,7 @@
 
 const NoteRevision = require('../entities/note_revision');
 const dateUtils = require('../services/date_utils');
+const log = require('../services/log');
 
 /**
  * @param {Note} note
@@ -9,14 +10,21 @@ const dateUtils = require('../services/date_utils');
 function protectNoteRevisions(note) {
     for (const revision of note.getRevisions()) {
         if (note.isProtected !== revision.isProtected) {
-            const content = revision.getContent();
+            try {
+                const content = revision.getContent();
 
-            revision.isProtected = note.isProtected;
+                revision.isProtected = note.isProtected;
 
-            // this will force de/encryption
+                // this will force de/encryption
-            revision.setContent(content);
+                revision.setContent(content);
 
-            revision.save();
+                revision.save();
+            }
+            catch (e) {
+                log.error("Could not un/protect note revision ID = " + revision.noteRevisionId);
+
+                throw e;
+            }
         }
     }
 }

src/services/notes.js

@@ -185,18 +185,25 @@ function protectNoteRecursively(note, protect, includingSubTree, taskContext) {
 }
 
 function protectNote(note, protect) {
-    if (protect !== note.isProtected) {
+    try {
-        const content = note.getContent();
+        if (protect !== note.isProtected) {
+            const content = note.getContent();
 
-        note.isProtected = protect;
+            note.isProtected = protect;
 
-        // this will force de/encryption
+            // this will force de/encryption
-        note.setContent(content);
+            note.setContent(content);
 
-        note.save();
+            note.save();
+        }
+
+        noteRevisionService.protectNoteRevisions(note);
     }
+    catch (e) {
+        log.error("Could not un/protect note ID = " + note.noteId);
 
-    noteRevisionService.protectNoteRevisions(note);
+        throw e;
+    }
 }
 
 function findImageLinks(content, foundLinks) {
@@ -668,8 +675,10 @@ function scanForLinks(note) {
     }
 }
 
-function eraseDeletedNotes() {
+function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) {
-    const eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
+    if (eraseNotesAfterTimeInSeconds === null) {
+        eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
+    }
 
     const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000);
 
@@ -719,26 +728,75 @@ function eraseDeletedNotes() {
     log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
 }
 
-function duplicateNote(noteId, parentNoteId) {
+function eraseDeletedNotesNow() {
-    const origNote = repository.getNote(noteId);
+    eraseDeletedNotes(0);
+}
 
+// do a replace in str - all keys should be replaced by the corresponding values
+function replaceByMap(str, mapObj) {
+    const re = new RegExp(Object.keys(mapObj).join("|"),"g");
+
+    return str.replace(re, matched => mapObj[matched]);
+}
+
+function duplicateSubtree(origNoteId, newParentNoteId) {
+    if (origNoteId === 'root') {
+        throw new Error('Duplicating root is not possible');
+    }
+
+    const origNote = repository.getNote(origNoteId);
+    // might be null if orig note is not in the target newParentNoteId
+    const origBranch = origNote.getBranches().find(branch => branch.parentNoteId === newParentNoteId);
+
+    const noteIdMapping = getNoteIdMapping(origNote);
+
+    const res = duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapping);
+
+    if (!res.note.title.endsWith('(dup)')) {
+        res.note.title += " (dup)";
+    }
+
+    res.note.save();
+
+    return res;
+}
+
+function duplicateSubtreeWithoutRoot(origNoteId, newNoteId) {
+    if (origNoteId === 'root') {
+        throw new Error('Duplicating root is not possible');
+    }
+
+    const origNote = repository.getNote(origNoteId);
+    const noteIdMapping = getNoteIdMapping(origNote);
+
+    for (const childBranch of origNote.getChildBranches()) {
+        duplicateSubtreeInner(childBranch.getNote(), childBranch, newNoteId, noteIdMapping);
+    }
+}
+
+function duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapping) {
     if (origNote.isProtected && !protectedSessionService.isProtectedSessionAvailable()) {
         throw new Error(`Cannot duplicate note=${origNote.noteId} because it is protected and protected session is not available`);
     }
 
-    // might be null if orig note is not in the target parentNoteId
-    const origBranch = origNote.getBranches().find(branch => branch.parentNoteId === parentNoteId);
-
     const newNote = new Note(origNote);
-    newNote.noteId = undefined; // force creation of new note
+    newNote.noteId = noteIdMapping[origNote.noteId];
-    newNote.title += " (dup)";
+    newNote.dateCreated = dateUtils.localNowDateTime();
+    newNote.utcDateCreated = dateUtils.utcNowDateTime();
     newNote.save();
 
-    newNote.setContent(origNote.getContent());
+    let content = origNote.getContent();
+
+    if (['text', 'relation-map', 'search'].includes(origNote.type)) {
+        // fix links in the content
+        content = replaceByMap(content, noteIdMapping);
+    }
+
+    newNote.setContent(content);
 
     const newBranch = new Branch({
         noteId: newNote.noteId,
-        parentNoteId: parentNoteId,
+        parentNoteId: newParentNoteId,
         // here increasing just by 1 to make sure it's directly after original
         notePosition: origBranch ? origBranch.notePosition + 1 : null
     }).save();
@@ -746,22 +804,43 @@ function duplicateNote(noteId, parentNoteId) {
     for (const attribute of origNote.getOwnedAttributes()) {
         const attr = new Attribute(attribute);
         attr.attributeId = undefined; // force creation of new attribute
+        attr.utcDateCreated = dateUtils.utcNowDateTime();
         attr.noteId = newNote.noteId;
 
+        // if relation points to within the duplicated tree then replace the target to the duplicated note
+        // if it points outside of duplicated tree then keep the original target
+        if (attr.type === 'relation' && attr.value in noteIdMapping) {
+            attr.value = noteIdMapping[attr.value];
+        }
+
         attr.save();
     }
 
+    for (const childBranch of origNote.getChildBranches()) {
+        duplicateSubtreeInner(childBranch.getNote(), childBranch, newNote.noteId, noteIdMapping);
+    }
+
     return {
         note: newNote,
         branch: newBranch
     };
 }
 
+function getNoteIdMapping(origNote) {
+    const noteIdMapping = {};
+
+    // pregenerate new noteIds since we'll need to fix relation references even for not yet created notes
+    for (const origNoteId of origNote.getDescendantNoteIds()) {
+        noteIdMapping[origNoteId] = utils.newEntityId();
+    }
+    return noteIdMapping;
+}
+
 sqlInit.dbReady.then(() => {
     // first cleanup kickoff 5 minutes after startup
-    setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000);
+    setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000);
 
-    setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000);
+    setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000);
 });
 
 module.exports = {
@@ -772,7 +851,9 @@ module.exports = {
     undeleteNote,
     protectNoteRecursively,
     scanForLinks,
-    duplicateNote,
+    duplicateSubtree,
+    duplicateSubtreeWithoutRoot,
     getUndeletedParentBranches,
-    triggerNoteTitleChanged
+    triggerNoteTitleChanged,
+    eraseDeletedNotesNow
 };

src/services/options_init.js

@@ -31,10 +31,7 @@ function initNotSyncedOptions(initialized, startNotePath = 'root', opts = {}) {
     optionService.createOption('openTabs', JSON.stringify([
         {
             notePath: startNotePath,
-            active: true,
+            active: true
-            sidebar: {
-                widgets: []
-            }
         }
     ]), false);
 
@@ -103,6 +100,15 @@ function initStartupOptions() {
             log.info(`Created missing option "${name}" with default value "${value}"`);
         }
     }
+
+    if (process.env.TRILIUM_START_NOTE_ID) {
+        optionService.setOption('openTabs', JSON.stringify([
+            {
+                notePath: process.env.TRILIUM_START_NOTE_ID,
+                active: true
+            }
+        ]));
+    }
 }
 
 function getKeyboardDefaultOptions() {

src/services/protected_session.js

@@ -43,10 +43,18 @@ function decryptNotes(notes) {
 }
 
 function encrypt(plainText) {
+    if (plainText === null) {
+        return null;
+    }
+
     return dataEncryptionService.encrypt(getDataKey(), plainText);
 }
 
 function decrypt(cipherText) {
+    if (cipherText === null) {
+        return null;
+    }
+
     return dataEncryptionService.decrypt(getDataKey(), cipherText);
 }
 

src/services/search/expressions/note_content_protected_fulltext.js

@@ -32,26 +32,29 @@ class NoteContentProtectedFulltextExp extends Expression {
                 FROM notes JOIN note_contents USING (noteId) 
                 WHERE type IN ('text', 'code') AND isDeleted = 0 AND isProtected = 1`)) {
 
+            if (!inputNoteSet.hasNoteId(noteId) || !(noteId in noteCache.notes)) {
+                continue;
+            }
+
             try {
                 content = protectedSessionService.decryptString(content);
             }
             catch (e) {
-                log.info('Cannot decrypt content of note', noteId);
+                log.info(`Cannot decrypt content of note ${noteId}`);
                 continue;
             }
 
             content = content.toLowerCase();
 
             if (type === 'text' && mime === 'text/html') {
-                content = striptags(content);
+                if (content.length < 20000) { // striptags is slow for very large notes
+                    content = striptags(content);
+                }
+
                 content = content.replace(/&nbsp;/g, ' ');
             }
 
-            if (this.tokens.find(token => !content.includes(token))) {
+            if (!this.tokens.find(token => !content.includes(token))) {
-                continue;
-            }
-
-            if (inputNoteSet.hasNoteId(noteId) && noteId in noteCache.notes) {
                 resultNoteSet.add(noteCache.notes[noteId]);
             }
         }

src/services/search/expressions/note_content_unprotected_fulltext.js

@@ -26,18 +26,21 @@ class NoteContentUnprotectedFulltextExp extends Expression {
                 FROM notes JOIN note_contents USING (noteId) 
                 WHERE type IN ('text', 'code') AND isDeleted = 0 AND isProtected = 0`)) {
 
-            content = content.toString().toLowerCase();
+            if (!inputNoteSet.hasNoteId(noteId) || !(noteId in noteCache.notes)) {
-
-            if (type === 'text' && mime === 'text/html') {
-                content = striptags(content);
-                content = content.replace(/ /g, ' ');
-            }
-
-            if (this.tokens.find(token => !content.includes(token))) {
                 continue;
             }
 
-            if (inputNoteSet.hasNoteId(noteId) && noteId in noteCache.notes) {
+            content = content.toString().toLowerCase();
+
+            if (type === 'text' && mime === 'text/html') {
+                if (content.length < 20000) { // striptags is slow for very large notes
+                    content = striptags(content);
+                }
+
+                content = content.replace(/&nbsp;/g, ' ');
+            }
+
+            if (!this.tokens.find(token => !content.includes(token))) {
                 resultNoteSet.add(noteCache.notes[noteId]);
             }
         }

src/services/search/services/search.js

@@ -17,10 +17,14 @@ const utils = require('../../utils.js');
  */
 function findNotesWithExpression(expression) {
     const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()];
-    const allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
+    let allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
         ? hoistedNote.subtreeNotes
         : Object.values(noteCache.notes);
 
+    // in the process of loading data sometimes we create "skeleton" note instances which are expected to be filled later
+    // in case of inconsistent data this might not work and search will then crash on these
+    allNotes = allNotes.filter(note => note.type !== undefined);
+
     const allNoteSet = new NoteSet(allNotes);
 
     const searchContext = {

src/services/sync.js

@@ -20,10 +20,7 @@ const entityConstructor = require('../entities/entity_constructor');
 
 let proxyToggle = true;
 
-const stats = {
+let outstandingPullCount = 0;
-    outstandingPushes: 0,
-    outstandingPulls: 0
-};
 
 async function sync() {
     try {
@@ -135,11 +132,7 @@ async function pullChanges(syncContext) {
 
         const pulledDate = Date.now();
 
-        stats.outstandingPulls = resp.maxEntityChangeId - lastSyncedPull;
+        outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull);
-
-        if (stats.outstandingPulls < 0) {
-            stats.outstandingPulls = 0;
-        }
 
         const {entityChanges} = resp;
 
@@ -159,13 +152,13 @@ async function pullChanges(syncContext) {
                     syncUpdateService.updateEntity(entityChange, entity, syncContext.sourceId);
                 }
 
-                stats.outstandingPulls = resp.maxEntityChangeId - entityChange.id;
+                outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id);
             }
 
             setLastSyncedPull(entityChanges[entityChanges.length - 1].entityChange.id);
         });
 
-        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`);
+        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${outstandingPullCount} outstanding pulls`);
     }
 
     if (atLeastOnePullApplied) {
@@ -359,31 +352,25 @@ function setLastSyncedPush(entityChangeId) {
     optionService.setOption('lastSyncedPush', entityChangeId);
 }
 
-function updatePushStats() {
-    if (syncOptions.isSyncSetup()) {
-        const lastSyncedPush = optionService.getOption('lastSyncedPush');
-
-        stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE isSynced = 1 AND id > ?", [lastSyncedPush]);
-    }
-}
-
 function getMaxEntityChangeId() {
     return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes');
 }
 
+function getOutstandingPullCount() {
+    return outstandingPullCount;
+}
+
 sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(sync), 60000);
 
     // kickoff initial sync immediately
     setTimeout(cls.wrap(sync), 3000);
-
-    setInterval(cls.wrap(updatePushStats), 1000);
 });
 
 module.exports = {
     sync,
     login,
     getEntityChangesRecords,
-    stats,
+    getOutstandingPullCount,
     getMaxEntityChangeId
 };

src/services/ws.js

@@ -110,8 +110,7 @@ function sendPing(client, syncRows = []) {
 
     sendMessage(client, {
         type: 'sync',
-        data: syncRows,
+        data: syncRows
-        outstandingSyncs: stats.outstandingPushes + stats.outstandingPulls
     });
 }