release 0.45.6

libraries/ckeditor/ckeditor-content.css

@@ -1,12 +1,12 @@
 /* !!!!!! TRILIUM CUSTOM CHANGES !!!!!! */
 
-.ck-widget__type-around { /* gets rid of triangles: https://github.com/zadam/trilium/issues/1129 */
+.printed-content .ck-widget__selection-handle, .printed-content .ck-widget__type-around { /* gets rid of triangles: https://github.com/zadam/trilium/issues/1129 */
     display: none;
 }
 
 /*
- * CKEditor 5 (v22.0.0) content styles.
- * Generated on Thu, 27 Aug 2020 12:13:06 GMT.
+ * CKEditor 5 (v23.1.0) content styles.
+ * Generated on Thu, 29 Oct 2020 12:17:48 GMT.
  * For more information, check out https://ckeditor.com/docs/ckeditor5/latest/builds/guides/integration/content-styles.html
  */
 
@@ -23,32 +23,6 @@
     --ck-todo-list-checkmark-size: 16px;
 }
 
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-yellow {
-    background-color: var(--ck-highlight-marker-yellow);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-green {
-    background-color: var(--ck-highlight-marker-green);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-pink {
-    background-color: var(--ck-highlight-marker-pink);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-blue {
-    background-color: var(--ck-highlight-marker-blue);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .pen-red {
-    color: var(--ck-highlight-pen-red);
-    background-color: transparent;
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .pen-green {
-    color: var(--ck-highlight-pen-green);
-    background-color: transparent;
-}
 /* ckeditor5-image/theme/imagestyle.css */
 .ck-content .image-style-side {
     float: right;
@@ -84,6 +58,17 @@
     max-width: 100%;
     min-width: 50px;
 }
+/* ckeditor5-image/theme/imagecaption.css */
+.ck-content .image > figcaption {
+    display: table-caption;
+    caption-side: bottom;
+    word-break: break-word;
+    color: hsl(0, 0%, 20%);
+    background-color: hsl(0, 0%, 97%);
+    padding: .6em;
+    font-size: .75em;
+    outline-offset: -1px;
+}
 /* ckeditor5-image/theme/imageresize.css */
 .ck-content .image.image_resized {
     max-width: 100%;
@@ -98,22 +83,31 @@
 .ck-content .image.image_resized > figcaption {
     display: block;
 }
-/* ckeditor5-image/theme/imagecaption.css */
-.ck-content .image > figcaption {
-    display: table-caption;
-    caption-side: bottom;
-    word-break: break-word;
-    color: hsl(0, 0%, 20%);
-    background-color: hsl(0, 0%, 97%);
-    padding: .6em;
-    font-size: .75em;
-    outline-offset: -1px;
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-yellow {
+    background-color: var(--ck-highlight-marker-yellow);
 }
-/* ckeditor5-basic-styles/theme/code.css */
-.ck-content code {
-    background-color: hsla(0, 0%, 78%, 0.3);
-    padding: .15em;
-    border-radius: 2px;
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-green {
+    background-color: var(--ck-highlight-marker-green);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-pink {
+    background-color: var(--ck-highlight-marker-pink);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-blue {
+    background-color: var(--ck-highlight-marker-blue);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .pen-red {
+    color: var(--ck-highlight-pen-red);
+    background-color: transparent;
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .pen-green {
+    color: var(--ck-highlight-pen-green);
+    background-color: transparent;
 }
 /* ckeditor5-font/theme/fontsize.css */
 .ck-content .text-tiny {
@@ -146,6 +140,12 @@
     border-left: 0;
     border-right: solid 5px hsl(0, 0%, 80%);
 }
+/* ckeditor5-basic-styles/theme/code.css */
+.ck-content code {
+    background-color: hsla(0, 0%, 78%, 0.3);
+    padding: .15em;
+    border-radius: 2px;
+}
 /* ckeditor5-table/theme/table.css */
 .ck-content .table {
     margin: 1em auto;
@@ -215,13 +215,6 @@
     -ms-user-select: none;
     user-select: none;
 }
-/* ckeditor5-media-embed/theme/mediaembed.css */
-.ck-content .media {
-    clear: both;
-    margin: 1em 0;
-    display: block;
-    min-width: 15em;
-}
 /* ckeditor5-list/theme/todolist.css */
 .ck-content .todo-list {
     list-style: none;
@@ -289,6 +282,18 @@
 .ck-content .todo-list .todo-list__label .todo-list__label__description {
     vertical-align: middle;
 }
+/* ckeditor5-media-embed/theme/mediaembed.css */
+.ck-content .media {
+    clear: both;
+    margin: 1em 0;
+    display: block;
+    min-width: 15em;
+}
+/* ckeditor5-html-embed/theme/htmlembed.css */
+.ck-content .raw-html-embed {
+    margin: 1em auto;
+    min-width: 15em;
+}
 /* ckeditor5-horizontal-line/theme/horizontalline.css */
 .ck-content hr {
     margin: 15px 0;
@@ -330,4 +335,4 @@
     .ck-content .page-break::after {
         display: none;
     }
-}
+}

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.45.2",
+  "version": "0.45.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -2654,9 +2654,9 @@
       }
     },
     "electron": {
-      "version": "9.3.4",
-      "resolved": "https://registry.npmjs.org/electron/-/electron-9.3.4.tgz",
-      "integrity": "sha512-OHP8qMKgW8D8GtH+altB22WJw/lBOyyVdoz5e8D0/iPBmJU3Jm93vO4z4Eh/9DvdSXlH8bMHUCMLL9PVW6f+tw==",
+      "version": "9.3.5",
+      "resolved": "https://registry.npmjs.org/electron/-/electron-9.3.5.tgz",
+      "integrity": "sha512-EPmDsp7sO0UPtw7nLD1ufse/nBskP+ifXzBgUg9psCUlapkzuwYi6pmLAzKLW/bVjwgyUKwh1OKWILWfOeLGcQ==",
       "dev": true,
       "requires": {
         "@electron/get": "^1.0.1",
@@ -4838,6 +4838,11 @@
         "type-check": "~0.3.2"
       }
     },
+    "limiter": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz",
+      "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA=="
+    },
     "line-column": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz",
@@ -6913,6 +6918,22 @@
       "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
       "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
     },
+    "stream-throttle": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz",
+      "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=",
+      "requires": {
+        "commander": "^2.2.0",
+        "limiter": "^1.0.5"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "2.20.3",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+          "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        }
+      }
+    },
     "streamsearch": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",

package.json

@@ -2,7 +2,7 @@
   "name": "trilium",
   "productName": "Trilium Notes",
   "description": "Trilium Notes",
-  "version": "0.45.3",
+  "version": "0.45.6",
   "license": "AGPL-3.0-only",
   "main": "electron.js",
   "bin": {
@@ -65,6 +65,7 @@
     "semver": "7.3.2",
     "serve-favicon": "2.5.0",
     "session-file-store": "1.5.0",
+    "stream-throttle": "^0.1.3",
     "striptags": "3.1.1",
     "tmp": "^0.2.1",
     "turndown": "7.0.0",
@@ -76,7 +77,7 @@
   },
   "devDependencies": {
     "cross-env": "7.0.2",
-    "electron": "9.3.4",
+    "electron": "9.3.5",
     "electron-builder": "22.9.1",
     "electron-packager": "15.1.0",
     "electron-rebuild": "2.3.2",

src/entities/attribute.js

@@ -34,6 +34,10 @@ class Attribute extends Entity {
         this.isInheritable = !!this.isInheritable;
     }
 
+    isAutoLink() {
+        return this.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(this.name);
+    }
+
     /**
      * @returns {Note|null}
      */

src/entities/branch.js

@@ -38,7 +38,7 @@ class Branch extends Entity {
     }
 
     beforeSaving() {
-        if (this.notePosition === undefined) {
+        if (this.notePosition === undefined || this.notePosition === null) {
             const maxNotePos = sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [this.parentNoteId]);
             this.notePosition = maxNotePos === null ? 0 : maxNotePos + 10;
         }

src/public/app/services/frontend_script_api.js

@@ -274,7 +274,10 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
      *
      * @method
      * @param {string} notePath (or noteId)
-     * @param {string} [noteTitle] - if not present we'll use note title
+     * @param {object} [params]
+     * @param {boolean} [params.showTooltip=true] - enable/disable tooltip on the link
+     * @param {boolean} [params.showNotePath=false] - show also whole note's path as part of the link
+     * @param {string} [title=] - custom link tile with note's title as default
      */
     this.createNoteLink = linkService.createNoteLink;
 

src/public/app/services/link.js

@@ -130,7 +130,7 @@ function linkContextMenu(e) {
                 appContext.tabManager.openTabWithNote(notePath);
             }
             else if (command === 'openNoteInNewWindow') {
-                appContext.openInNewWindow(notePath);
+                appContext.triggerCommand('openInWindow', {notePath});
             }
         }
     });

src/public/app/services/note_create.js

@@ -88,7 +88,7 @@ function parseSelectedHtml(selectedHtml) {
     }
 }
 
-async function duplicateNote(noteId, parentNoteId) {
+async function duplicateSubtree(noteId, parentNoteId) {
     const {note} = await server.post(`notes/${noteId}/duplicate/${parentNoteId}`);
 
     await ws.waitForMaxKnownEntityChangeId();
@@ -102,5 +102,5 @@ async function duplicateNote(noteId, parentNoteId) {
 export default {
     createNote,
     createNewTopLevelNote,
-    duplicateNote
+    duplicateSubtree
 };

src/public/app/services/open.js

@@ -64,6 +64,7 @@ function getHost() {
 }
 
 export default {
+    download,
     downloadFileNote,
     openFileNote,
     downloadNoteRevision,

src/public/app/services/tree_context_menu.js

@@ -2,9 +2,9 @@ import treeService from './tree.js';
 import treeCache from "./tree_cache.js";
 import hoistedNoteService from './hoisted_note.js';
 import clipboard from './clipboard.js';
-import protectedSessionHolder from "./protected_session_holder.js";
 import noteCreateService from "./note_create.js";
 import contextMenu from "./context_menu.js";
+import appContext from "./app_context.js";
 
 class TreeContextMenu {
     /**
@@ -95,7 +95,7 @@ class TreeContextMenu {
                 enabled: !clipboard.isClipboardEmpty() && notSearch && noSelectedNotes },
             { title: 'Paste after', command: "pasteNotesAfterFromClipboard", uiIcon: "paste",
                 enabled: !clipboard.isClipboardEmpty() && isNotRoot && !isHoisted && parentNotSearch && noSelectedNotes },
-            { title: "Duplicate note(s) here", command: "duplicateNote", uiIcon: "empty",
+            { title: `Duplicate subtree <kbd data-command="duplicateSubtree">`, command: "duplicateSubtree", uiIcon: "empty",
                 enabled: parentNotSearch && isNotRoot && !isHoisted },
             { title: "----" },
             { title: "Export", command: "exportNote", uiIcon: "empty",
@@ -110,14 +110,7 @@ class TreeContextMenu {
         const notePath = treeService.getNotePath(this.node);
 
         if (command === 'openInTab') {
-
-            const start = Date.now();
-
-            await this.node.load(true);
-
-            console.log("Reload took", Date.now() - start, "ms");
-
-//            appContext.tabManager.openTabWithNote(notePath);
+            appContext.tabManager.openTabWithNote(notePath);
         }
         else if (command === "insertNoteAfter") {
             const parentNoteId = this.node.data.parentNoteId;

src/public/app/services/ws.js

@@ -8,8 +8,6 @@ import options from "./options.js";
 import treeCache from "./tree_cache.js";
 import noteAttributeCache from "./note_attribute_cache.js";
 
-const $outstandingSyncsCount = $("#outstanding-syncs-count");
-
 const messageHandlers = [];
 
 let ws;
@@ -64,8 +62,6 @@ async function handleMessage(event) {
         let syncRows = message.data;
         lastPingTs = Date.now();
 
-        $outstandingSyncsCount.html(message.outstandingSyncs);
-
         if (syncRows.length > 0) {
             logRows(syncRows);
 

src/public/app/setup.js

@@ -130,7 +130,7 @@ function SetupModel() {
 }
 
 async function checkOutstandingSyncs() {
-    const { stats, initialized } = await $.get('api/sync/stats');
+    const { outstandingPullCount, initialized } = await $.get('api/sync/stats');
 
     if (initialized) {
         if (utils.isElectron()) {
@@ -143,9 +143,7 @@ async function checkOutstandingSyncs() {
         }
     }
     else {
-        const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;
-
-        $("#outstanding-syncs").html(totalOutstandingSyncs);
+        $("#outstanding-syncs").html(outstandingPullCount);
     }
 }
 

src/public/app/widgets/attribute_list.js

@@ -214,7 +214,8 @@ export default class AttributeListWidget extends TabAwareWidget {
                         noteId: attribute.noteId,
                         type: attribute.type,
                         name: attribute.name,
-                        value: attribute.value
+                        value: attribute.value,
+                        isInheritable: attribute.isInheritable
                     },
                     isOwned: false,
                     x: e.pageX,

src/public/app/widgets/global_menu.js

@@ -41,7 +41,7 @@ const TPL = `
 
             <a class="dropdown-item sync-now-button" title="Trigger sync">
                 <span class="bx bx-refresh"></span>
-                Sync now (<span id="outstanding-syncs-count">0</span>)
+                Sync now
             </a>
 
             <a class="dropdown-item" data-trigger-command="openNewWindow">

src/public/app/widgets/note_detail.js

@@ -248,7 +248,7 @@ export default class NoteDetailWidget extends TabAwareWidget {
 
         this.$widget.find('.note-detail-printable:visible').printThis({
             header: $("<h2>").text(this.note && this.note.title).prop('outerHTML'),
-            footer: "<script>document.body.className += ' ck-content';</script>",
+            footer: "<script>document.body.className += ' ck-content printed-content';</script>",
             importCSS: false,
             loadCSS: [
                 "libraries/codemirror/codemirror.css",

src/public/app/widgets/note_tree.js

@@ -1341,7 +1341,7 @@ export default class NoteTreeWidget extends TabAwareWidget {
         protectedSessionService.protectNote(node.data.noteId, false, true);
     }
 
-    duplicateNoteCommand({node}) {
+    duplicateSubtreeCommand({node}) {
         const nodesToDuplicate = this.getSelectedOrActiveNodes(node);
 
         for (const nodeToDuplicate of nodesToDuplicate) {
@@ -1353,7 +1353,7 @@ export default class NoteTreeWidget extends TabAwareWidget {
 
             const branch = treeCache.getBranch(nodeToDuplicate.data.branchId);
 
-            noteCreateService.duplicateNote(nodeToDuplicate.data.noteId, branch.parentNoteId);
+            noteCreateService.duplicateSubtree(nodeToDuplicate.data.noteId, branch.parentNoteId);
         }
     }
 }

src/public/stylesheets/themes.css

@@ -143,6 +143,11 @@ body {
     --ck-color-dropdown-panel-background: var(--accented-background-color);
     --ck-color-dropdown-panel-border: var(--main-border-color);
 
+    /* -- Overrides the default .ck-splitbutton class colors. ----------------------------------- */
+
+    --ck-color-split-button-hover-background: var(--ck-color-button-default-hover-background);
+    --ck-color-split-button-hover-border: var(--main-border-color);
+
     /* -- Overrides the default .ck-input class colors. ----------------------------------------- */
 
     --ck-color-input-background: var(--accented-background-color);
@@ -199,6 +204,9 @@ body {
     --ck-color-engine-placeholder-text: var(--muted-text-color);
 
     --ck-z-modal: 10000;
+
+    --ck-color-widget-type-around-button: var(--main-border-color);
+    --ck-color-widget-type-around-button-hover: var(--main-border-color);
 }
 
 body {

src/routes/api/attributes.js

@@ -28,8 +28,10 @@ function updateNoteAttribute(req) {
             || body.name !== attribute.name
             || (body.type === 'relation' && body.value !== attribute.value)) {
 
+            let newAttribute;
+
             if (body.type !== 'relation' || !!body.value.trim()) {
-                const newAttribute = attribute.createClone(body.type, body.name, body.value);
+                newAttribute = attribute.createClone(body.type, body.name, body.value);
                 newAttribute.save();
             }
 
@@ -37,7 +39,7 @@ function updateNoteAttribute(req) {
             attribute.save();
 
             return {
-                attributeId: attribute.attributeId
+                attributeId: newAttribute ? newAttribute.attributeId : null
             };
         }
     }
@@ -52,8 +54,9 @@ function updateNoteAttribute(req) {
         attribute.type = body.type;
     }
 
-    if (body.value.trim()) {
+    if (body.type !== 'relation' || body.value.trim()) {
         attribute.value = body.value;
+        attribute.isDeleted = false;
     }
     else {
         // relations should never have empty target
@@ -144,8 +147,10 @@ function updateNoteAttributes(req) {
 
     // all the remaining existing attributes are not defined anymore and should be deleted
     for (const toDeleteAttr of existingAttrs) {
-        toDeleteAttr.isDeleted = true;
-        toDeleteAttr.save();
+        if (!toDeleteAttr.isAutoLink()) {
+            toDeleteAttr.isDeleted = true;
+            toDeleteAttr.save();
+        }
     }
 }
 

src/routes/api/link_map.js

@@ -3,15 +3,33 @@
 const sql = require('../../services/sql');
 
 function getRelations(noteIds) {
-    return (sql.getManyRows(`
-        SELECT noteId, name, value AS targetNoteId
-        FROM attributes
-        WHERE (noteId IN (???) OR value IN (???))
-          AND type = 'relation'
-          AND isDeleted = 0
-          AND noteId != ''
-          AND value != ''
-    `, Array.from(noteIds)));
+    noteIds = Array.from(noteIds);
+
+    return [
+        // first read all non-image relations
+        ...sql.getManyRows(`
+            SELECT noteId, name, value AS targetNoteId
+            FROM attributes
+            WHERE (noteId IN (???) OR value IN (???))
+              AND type = 'relation'
+              AND name != 'imageLink'
+              AND isDeleted = 0
+              AND noteId != ''
+              AND value != ''`, noteIds),
+        // ... then read only imageLink relations which are not connecting parent and child
+        // this is done to not show image links in the trivial case where they are direct children of the note to which they are included. Same heuristic as in note tree
+        ...sql.getManyRows(`
+            SELECT rel.noteId, rel.name, rel.value AS targetNoteId
+            FROM attributes AS rel
+            LEFT JOIN branches ON branches.parentNoteId = rel.noteId AND branches.noteId = rel.value AND branches.isDeleted = 0 
+            WHERE (rel.noteId IN (???) OR rel.value IN (???))
+              AND rel.type = 'relation'
+              AND rel.name = 'imageLink'
+              AND rel.isDeleted = 0
+              AND rel.noteId != ''
+              AND rel.value != ''
+              AND branches.branchId IS NULL`, noteIds)
+    ];
 }
 
 function getLinkMap(req) {

src/routes/api/notes.js

@@ -187,10 +187,10 @@ function changeTitle(req) {
     return note;
 }
 
-function duplicateNote(req) {
+function duplicateSubtree(req) {
     const {noteId, parentNoteId} = req.params;
 
-    return noteService.duplicateNote(noteId, parentNoteId);
+    return noteService.duplicateSubtree(noteId, parentNoteId);
 }
 
 module.exports = {
@@ -204,5 +204,5 @@ module.exports = {
     setNoteTypeMime,
     getRelationMap,
     changeTitle,
-    duplicateNote
+    duplicateSubtree
 };

src/routes/api/recent_changes.js

@@ -31,19 +31,36 @@ function getRecentChanges(req) {
         }
     }
 
+    // now we need to also collect date points not represented in note revisions:
+    // 1. creation for all notes (dateCreated)
+    // 2. deletion for deleted notes (dateModified)
     const notes = sql.getRows(`
-        SELECT
-            notes.noteId,
-            notes.isDeleted AS current_isDeleted,
-            notes.deleteId AS current_deleteId,
-            notes.isErased AS current_isErased,
-            notes.title AS current_title,
-            notes.isProtected AS current_isProtected,
-            notes.title,
-            notes.utcDateCreated AS utcDate,
-            notes.dateCreated AS date
-        FROM
-            notes`);
+            SELECT
+                notes.noteId,
+                notes.isDeleted AS current_isDeleted,
+                notes.deleteId AS current_deleteId,
+                notes.isErased AS current_isErased,
+                notes.title AS current_title,
+                notes.isProtected AS current_isProtected,
+                notes.title,
+                notes.utcDateCreated AS utcDate,
+                notes.dateCreated AS date
+            FROM
+                notes
+        UNION ALL
+            SELECT
+                notes.noteId,
+                notes.isDeleted AS current_isDeleted,
+                notes.deleteId AS current_deleteId,
+                notes.isErased AS current_isErased,
+                notes.title AS current_title,
+                notes.isProtected AS current_isProtected,
+                notes.title,
+                notes.utcDateModified AS utcDate,
+                notes.dateModified AS date
+            FROM
+                notes
+            WHERE notes.isDeleted = 1 AND notes.isErased = 0`);
 
     for (const note of notes) {
         if (noteCacheService.isInAncestor(note.noteId, ancestorNoteId)) {

src/routes/api/sender.js

@@ -4,6 +4,7 @@ const imageType = require('image-type');
 const imageService = require('../../services/image');
 const dateNoteService = require('../../services/date_notes');
 const noteService = require('../../services/notes');
+const attributeService = require('../../services/attributes');
 
 function uploadImage(req) {
     const file = req.file;
@@ -37,7 +38,7 @@ function saveNote(req) {
 
     if (req.body.labels) {
         for (const {name, value} of req.body.labels) {
-            note.setLabel(name, value);
+            note.setLabel(attributeService.sanitizeAttributeName(name), value);
         }
     }
 

src/routes/api/sync.js

@@ -13,13 +13,13 @@ const dateUtils = require('../../services/date_utils');
 const entityConstructor = require('../../entities/entity_constructor');
 const utils = require('../../services/utils');
 
-function testSync() {
+async function testSync() {
     try {
         if (!syncOptions.isSyncSetup()) {
             return { success: false, message: "Sync server host is not configured. Please configure sync first." };
         }
 
-        syncService.login();
+        await syncService.login();
 
         // login was successful so we'll kick off sync now
         // this is important in case when sync server has been just initialized
@@ -43,7 +43,7 @@ function getStats() {
 
     const stats = {
         initialized: optionService.getOption('initialized') === 'true',
-        stats: syncService.stats
+        outstandingPullCount: syncService.getOutstandingPullCount()
     };
 
     log.info(`Returning sync stats: ${JSON.stringify(stats)}`);

src/routes/routes.js

@@ -154,7 +154,7 @@ function register(app) {
     apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision);
     apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap);
     apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle);
-    apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateNote);
+    apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree);
 
     apiRoute(GET, '/api/edited-notes/:date', noteRevisionsApiRoute.getEditedNotesOnDate);
 

src/services/attributes.js

@@ -2,7 +2,6 @@
 
 const repository = require('./repository');
 const sql = require('./sql');
-const utils = require('./utils');
 const Attribute = require('../entities/attribute');
 
 const ATTRIBUTE_TYPES = [ 'label', 'relation' ];
@@ -146,6 +145,20 @@ function getBuiltinAttributeNames() {
         ]);
 }
 
+function sanitizeAttributeName(origName) {
+    let fixedName;
+
+    if (origName === '') {
+        fixedName = "unnamed";
+    }
+    else {
+        // any not allowed character should be replaced with underscore
+        fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
+    }
+
+    return fixedName;
+}
+
 module.exports = {
     getNotesWithLabel,
     getNotesWithLabels,
@@ -156,5 +169,6 @@ module.exports = {
     getAttributeNames,
     isAttributeType,
     isAttributeDangerous,
-    getBuiltinAttributeNames
+    getBuiltinAttributeNames,
+    sanitizeAttributeName
 };

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2020-11-10T22:54:39+01:00", buildRevision: "5157fc15e9f7fa960ee35685426868d5599933dc" };
+module.exports = { buildDate:"2020-12-04T22:08:24+01:00", buildRevision: "b7b1324dd0b2af6abc8cd5b98f620ca227582d4d" };

src/services/consistency_checks.js

@@ -11,6 +11,7 @@ const entityChangesService = require('./entity_changes.js');
 const optionsService = require('./options');
 const Branch = require('../entities/branch');
 const dateUtils = require('./date_utils');
+const attributeService = require('./attributes');
 
 class ConsistencyChecks {
     constructor(autoFix) {
@@ -607,20 +608,10 @@ class ConsistencyChecks {
     findWronglyNamedAttributes() {
         const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`);
 
-        const attrNameMatcher = new RegExp("^[\\p{L}\\p{N}_:]+$", "u");
-
         for (const origName of attrNames) {
-            if (!attrNameMatcher.test(origName)) {
-                let fixedName;
-
-                if (origName === '') {
-                    fixedName = "unnamed";
-                }
-                else {
-                    // any not allowed character should be replaced with underscore
-                    fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
-                }
+            const fixedName = attributeService.sanitizeAttributeName(origName);
 
+            if (fixedName !== origName) {
                 if (this.autoFix) {
                     // there isn't a good way to update this:
                     // - just SQL query will fix it in DB but not notify frontend (or other caches) that it has been fixed

src/services/handlers.js

@@ -1,7 +1,7 @@
 const eventService = require('./events');
 const scriptService = require('./script');
 const treeService = require('./tree');
-const log = require('./log');
+const noteService = require('./notes');
 const repository = require('./repository');
 const Attribute = require('../entities/attribute');
 
@@ -58,17 +58,21 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) =>
                 return;
             }
 
-            const targetNote = repository.getNote(entity.value);
+            const templateNote = repository.getNote(entity.value);
 
-            if (!targetNote || !targetNote.isStringNote()) {
+            if (!templateNote) {
                 return;
             }
 
-            const targetNoteContent = targetNote.getContent();
+            if (templateNote.isStringNote()) {
+                const templateNoteContent = templateNote.getContent();
 
-            if (targetNoteContent) {
-                note.setContent(targetNoteContent);
+                if (templateNoteContent) {
+                    note.setContent(templateNoteContent);
+                }
             }
+
+            noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId);
         }
         else if (entity.type === 'label' && entity.name === 'sorted') {
             treeService.sortNotesAlphabetically(entity.noteId);

src/services/image.js

@@ -37,7 +37,7 @@ function getImageType(buffer) {
         }
     }
     else {
-        return imageType(buffer);
+        return imageType(buffer) || "jpg"; // optimistic JPG default
     }
 }
 

src/services/import/enex.js

@@ -1,5 +1,6 @@
 const sax = require("sax");
 const stream = require('stream');
+const {Throttle} = require('stream-throttle');
 const log = require("../log");
 const utils = require("../utils");
 const sql = require("../sql");
@@ -7,6 +8,7 @@ const noteService = require("../notes");
 const imageService = require("../image");
 const protectedSessionService = require('../protected_session');
 const htmlSanitizer = require("../html_sanitizer");
+const attributeService = require("../attributes");
 
 // date format is e.g. 20181121T193703Z
 function parseDate(text) {
@@ -37,10 +39,6 @@ function importEnex(taskContext, file, parentNote) {
         isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
     })).note;
 
-    // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished
-    // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully.
-    const saveNotePromises = [];
-
     function extractContent(content) {
         const openingNoteIndex = content.indexOf('<en-note>');
 
@@ -105,9 +103,17 @@ function importEnex(taskContext, file, parentNote) {
         const previousTag = getPreviousTag();
 
         if (previousTag === 'note-attributes') {
+            let labelName = currentTag;
+
+            if (labelName === 'source-url') {
+                labelName = 'sourceUrl';
+            }
+
+            labelName = attributeService.sanitizeAttributeName(labelName);
+
             note.attributes.push({
                 type: 'label',
-                name: currentTag,
+                name: labelName,
                 value: text
             });
         }
@@ -149,7 +155,7 @@ function importEnex(taskContext, file, parentNote) {
             } else if (currentTag === 'tag') {
                 note.attributes.push({
                     type: 'label',
-                    name: text,
+                    name: attributeService.sanitizeAttributeName(text),
                     value: ''
                 })
             }
@@ -227,6 +233,10 @@ function importEnex(taskContext, file, parentNote) {
         taskContext.increaseProgressCount();
 
         for (const resource of resources) {
+            if (!resource.content) {
+                continue;
+            }
+
             const hash = utils.md5(resource.content);
 
             const mediaRegex = new RegExp(`<en-media hash="${hash}"[^>]*>`, 'g');
@@ -304,7 +314,7 @@ function importEnex(taskContext, file, parentNote) {
         path.pop();
 
         if (tag === 'note') {
-            saveNotePromises.push(saveNote());
+            saveNote();
         }
     });
 
@@ -323,12 +333,15 @@ function importEnex(taskContext, file, parentNote) {
     return new Promise((resolve, reject) =>
     {
         // resolve only when we parse the whole document AND saving of all notes have been finished
-        saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) });
+        saxStream.on("end", () => resolve(rootNote));
 
         const bufferStream = new stream.PassThrough();
         bufferStream.end(file.buffer);
 
-        bufferStream.pipe(saxStream);
+        bufferStream
+            // rate limiting to improve responsiveness during / after import
+            .pipe(new Throttle({rate: 500000}))
+            .pipe(saxStream);
     });
 }
 

src/services/import/zip.js

@@ -158,7 +158,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             }
 
             if (taskContext.data.safeImport && attributeService.isAttributeDangerous(attr.type, attr.name)) {
-                attr.name = 'disabled-' + attr.name;
+                attr.name = 'disabled:' + attr.name;
             }
 
             attributes.push(attr);

src/services/keyboard_actions.js

@@ -184,6 +184,12 @@ const DEFAULT_KEYBOARD_ACTIONS = [
         description: "Add note above to the selection",
         scope: "note-tree"
     },
+    {
+        actionName: "duplicateSubtree",
+        defaultShortcuts: [],
+        description: "Duplicate subtree",
+        scope: "note-tree"
+    },
 
 
     {
@@ -307,6 +313,12 @@ const DEFAULT_KEYBOARD_ACTIONS = [
         description: "Cuts the selection from the current note and creates subnote with the selected text",
         scope: "text-detail"
     },
+    {
+        actionName: "addIncludeNoteToText",
+        defaultShortcuts: [],
+        description: "Opens the dialog to include a note",
+        scope: "text-detail"
+    },
 
     {
         separator: "Attributes (labels & relations)"

src/services/notes.js

@@ -719,26 +719,71 @@ function eraseDeletedNotes() {
     log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
 }
 
-function duplicateNote(noteId, parentNoteId) {
-    const origNote = repository.getNote(noteId);
+// do a replace in str - all keys should be replaced by the corresponding values
+function replaceByMap(str, mapObj) {
+    const re = new RegExp(Object.keys(mapObj).join("|"),"g");
 
+    return str.replace(re, matched => mapObj[matched]);
+}
+
+function duplicateSubtree(origNoteId, newParentNoteId) {
+    if (origNoteId === 'root') {
+        throw new Error('Duplicating root is not possible');
+    }
+
+    const origNote = repository.getNote(origNoteId);
+    // might be null if orig note is not in the target newParentNoteId
+    const origBranch = origNote.getBranches().find(branch => branch.parentNoteId === newParentNoteId);
+
+    const noteIdMapping = getNoteIdMapping(origNote);
+
+    const res = duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapping);
+
+    if (!res.note.title.endsWith('(dup)')) {
+        res.note.title += " (dup)";
+    }
+
+    res.note.save();
+
+    return res;
+}
+
+function duplicateSubtreeWithoutRoot(origNoteId, newNoteId) {
+    if (origNoteId === 'root') {
+        throw new Error('Duplicating root is not possible');
+    }
+
+    const origNote = repository.getNote(origNoteId);
+    const noteIdMapping = getNoteIdMapping(origNote);
+
+    for (const childBranch of origNote.getChildBranches()) {
+        duplicateSubtreeInner(childBranch.getNote(), childBranch, newNoteId, noteIdMapping);
+    }
+}
+
+function duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapping) {
     if (origNote.isProtected && !protectedSessionService.isProtectedSessionAvailable()) {
         throw new Error(`Cannot duplicate note=${origNote.noteId} because it is protected and protected session is not available`);
     }
 
-    // might be null if orig note is not in the target parentNoteId
-    const origBranch = origNote.getBranches().find(branch => branch.parentNoteId === parentNoteId);
-
     const newNote = new Note(origNote);
-    newNote.noteId = undefined; // force creation of new note
-    newNote.title += " (dup)";
+    newNote.noteId = noteIdMapping[origNote.noteId];
+    newNote.dateCreated = dateUtils.localNowDateTime();
+    newNote.utcDateCreated = dateUtils.utcNowDateTime();
     newNote.save();
 
-    newNote.setContent(origNote.getContent());
+    let content = origNote.getContent();
+
+    if (['text', 'relation-map', 'search'].includes(origNote.type)) {
+        // fix links in the content
+        content = replaceByMap(content, noteIdMapping);
+    }
+
+    newNote.setContent(content);
 
     const newBranch = new Branch({
         noteId: newNote.noteId,
-        parentNoteId: parentNoteId,
+        parentNoteId: newParentNoteId,
         // here increasing just by 1 to make sure it's directly after original
         notePosition: origBranch ? origBranch.notePosition + 1 : null
     }).save();
@@ -746,17 +791,38 @@ function duplicateNote(noteId, parentNoteId) {
     for (const attribute of origNote.getOwnedAttributes()) {
         const attr = new Attribute(attribute);
         attr.attributeId = undefined; // force creation of new attribute
+        attr.utcDateCreated = dateUtils.utcNowDateTime();
         attr.noteId = newNote.noteId;
 
+        // if relation points to within the duplicated tree then replace the target to the duplicated note
+        // if it points outside of duplicated tree then keep the original target
+        if (attr.type === 'relation' && attr.value in noteIdMapping) {
+            attr.value = noteIdMapping[attr.value];
+        }
+
         attr.save();
     }
 
+    for (const childBranch of origNote.getChildBranches()) {
+        duplicateSubtreeInner(childBranch.getNote(), childBranch, newNote.noteId, noteIdMapping);
+    }
+
     return {
         note: newNote,
         branch: newBranch
     };
 }
 
+function getNoteIdMapping(origNote) {
+    const noteIdMapping = {};
+
+    // pregenerate new noteIds since we'll need to fix relation references even for not yet created notes
+    for (const origNoteId of origNote.getDescendantNoteIds()) {
+        noteIdMapping[origNoteId] = utils.newEntityId();
+    }
+    return noteIdMapping;
+}
+
 sqlInit.dbReady.then(() => {
     // first cleanup kickoff 5 minutes after startup
     setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000);
@@ -772,7 +838,8 @@ module.exports = {
     undeleteNote,
     protectNoteRecursively,
     scanForLinks,
-    duplicateNote,
+    duplicateSubtree,
+    duplicateSubtreeWithoutRoot,
     getUndeletedParentBranches,
     triggerNoteTitleChanged
 };

src/services/search/services/search.js

@@ -17,10 +17,14 @@ const utils = require('../../utils.js');
  */
 function findNotesWithExpression(expression) {
     const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()];
-    const allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
+    let allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
         ? hoistedNote.subtreeNotes
         : Object.values(noteCache.notes);
 
+    // in the process of loading data sometimes we create "skeleton" note instances which are expected to be filled later
+    // in case of inconsistent data this might not work and search will then crash on these
+    allNotes = allNotes.filter(note => note.type !== undefined);
+
     const allNoteSet = new NoteSet(allNotes);
 
     const searchContext = {

src/services/sync.js

@@ -20,10 +20,7 @@ const entityConstructor = require('../entities/entity_constructor');
 
 let proxyToggle = true;
 
-const stats = {
-    outstandingPushes: 0,
-    outstandingPulls: 0
-};
+let outstandingPullCount = 0;
 
 async function sync() {
     try {
@@ -135,11 +132,7 @@ async function pullChanges(syncContext) {
 
         const pulledDate = Date.now();
 
-        stats.outstandingPulls = resp.maxEntityChangeId - lastSyncedPull;
-
-        if (stats.outstandingPulls < 0) {
-            stats.outstandingPulls = 0;
-        }
+        outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull);
 
         const {entityChanges} = resp;
 
@@ -159,13 +152,13 @@ async function pullChanges(syncContext) {
                     syncUpdateService.updateEntity(entityChange, entity, syncContext.sourceId);
                 }
 
-                stats.outstandingPulls = resp.maxEntityChangeId - entityChange.id;
+                outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id);
             }
 
             setLastSyncedPull(entityChanges[entityChanges.length - 1].entityChange.id);
         });
 
-        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`);
+        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${outstandingPullCount} outstanding pulls`);
     }
 
     if (atLeastOnePullApplied) {
@@ -359,31 +352,25 @@ function setLastSyncedPush(entityChangeId) {
     optionService.setOption('lastSyncedPush', entityChangeId);
 }
 
-function updatePushStats() {
-    if (syncOptions.isSyncSetup()) {
-        const lastSyncedPush = optionService.getOption('lastSyncedPush');
-
-        stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE isSynced = 1 AND id > ?", [lastSyncedPush]);
-    }
-}
-
 function getMaxEntityChangeId() {
     return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes');
 }
 
+function getOutstandingPullCount() {
+    return outstandingPullCount;
+}
+
 sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(sync), 60000);
 
     // kickoff initial sync immediately
     setTimeout(cls.wrap(sync), 3000);
-
-    setInterval(cls.wrap(updatePushStats), 1000);
 });
 
 module.exports = {
     sync,
     login,
     getEntityChangesRecords,
-    stats,
+    getOutstandingPullCount,
     getMaxEntityChangeId
 };

src/services/ws.js

@@ -110,8 +110,7 @@ function sendPing(client, syncRows = []) {
 
     sendMessage(client, {
         type: 'sync',
-        data: syncRows,
-        outstandingSyncs: stats.outstandingPushes + stats.outstandingPulls
+        data: syncRows
     });
 }