release 0.45.5

libraries/ckeditor/ckeditor-content.css

@@ -1,12 +1,12 @@
 /* !!!!!! TRILIUM CUSTOM CHANGES !!!!!! */
 
-.ck-widget__type-around { /* gets rid of triangles: https://github.com/zadam/trilium/issues/1129 */
+.ck-widget__selection-handle, .ck-widget__type-around { /* gets rid of triangles: https://github.com/zadam/trilium/issues/1129 */
     display: none;
 }
 
 /*
- * CKEditor 5 (v22.0.0) content styles.
- * Generated on Thu, 27 Aug 2020 12:13:06 GMT.
+ * CKEditor 5 (v23.1.0) content styles.
+ * Generated on Thu, 29 Oct 2020 12:17:48 GMT.
  * For more information, check out https://ckeditor.com/docs/ckeditor5/latest/builds/guides/integration/content-styles.html
  */
 
@@ -23,32 +23,6 @@
     --ck-todo-list-checkmark-size: 16px;
 }
 
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-yellow {
-    background-color: var(--ck-highlight-marker-yellow);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-green {
-    background-color: var(--ck-highlight-marker-green);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-pink {
-    background-color: var(--ck-highlight-marker-pink);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .marker-blue {
-    background-color: var(--ck-highlight-marker-blue);
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .pen-red {
-    color: var(--ck-highlight-pen-red);
-    background-color: transparent;
-}
-/* ckeditor5-highlight/theme/highlight.css */
-.ck-content .pen-green {
-    color: var(--ck-highlight-pen-green);
-    background-color: transparent;
-}
 /* ckeditor5-image/theme/imagestyle.css */
 .ck-content .image-style-side {
     float: right;
@@ -84,6 +58,17 @@
     max-width: 100%;
     min-width: 50px;
 }
+/* ckeditor5-image/theme/imagecaption.css */
+.ck-content .image > figcaption {
+    display: table-caption;
+    caption-side: bottom;
+    word-break: break-word;
+    color: hsl(0, 0%, 20%);
+    background-color: hsl(0, 0%, 97%);
+    padding: .6em;
+    font-size: .75em;
+    outline-offset: -1px;
+}
 /* ckeditor5-image/theme/imageresize.css */
 .ck-content .image.image_resized {
     max-width: 100%;
@@ -98,22 +83,31 @@
 .ck-content .image.image_resized > figcaption {
     display: block;
 }
-/* ckeditor5-image/theme/imagecaption.css */
-.ck-content .image > figcaption {
-    display: table-caption;
-    caption-side: bottom;
-    word-break: break-word;
-    color: hsl(0, 0%, 20%);
-    background-color: hsl(0, 0%, 97%);
-    padding: .6em;
-    font-size: .75em;
-    outline-offset: -1px;
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-yellow {
+    background-color: var(--ck-highlight-marker-yellow);
 }
-/* ckeditor5-basic-styles/theme/code.css */
-.ck-content code {
-    background-color: hsla(0, 0%, 78%, 0.3);
-    padding: .15em;
-    border-radius: 2px;
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-green {
+    background-color: var(--ck-highlight-marker-green);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-pink {
+    background-color: var(--ck-highlight-marker-pink);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .marker-blue {
+    background-color: var(--ck-highlight-marker-blue);
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .pen-red {
+    color: var(--ck-highlight-pen-red);
+    background-color: transparent;
+}
+/* ckeditor5-highlight/theme/highlight.css */
+.ck-content .pen-green {
+    color: var(--ck-highlight-pen-green);
+    background-color: transparent;
 }
 /* ckeditor5-font/theme/fontsize.css */
 .ck-content .text-tiny {
@@ -146,6 +140,12 @@
     border-left: 0;
     border-right: solid 5px hsl(0, 0%, 80%);
 }
+/* ckeditor5-basic-styles/theme/code.css */
+.ck-content code {
+    background-color: hsla(0, 0%, 78%, 0.3);
+    padding: .15em;
+    border-radius: 2px;
+}
 /* ckeditor5-table/theme/table.css */
 .ck-content .table {
     margin: 1em auto;
@@ -215,13 +215,6 @@
     -ms-user-select: none;
     user-select: none;
 }
-/* ckeditor5-media-embed/theme/mediaembed.css */
-.ck-content .media {
-    clear: both;
-    margin: 1em 0;
-    display: block;
-    min-width: 15em;
-}
 /* ckeditor5-list/theme/todolist.css */
 .ck-content .todo-list {
     list-style: none;
@@ -289,6 +282,18 @@
 .ck-content .todo-list .todo-list__label .todo-list__label__description {
     vertical-align: middle;
 }
+/* ckeditor5-media-embed/theme/mediaembed.css */
+.ck-content .media {
+    clear: both;
+    margin: 1em 0;
+    display: block;
+    min-width: 15em;
+}
+/* ckeditor5-html-embed/theme/htmlembed.css */
+.ck-content .raw-html-embed {
+    margin: 1em auto;
+    min-width: 15em;
+}
 /* ckeditor5-horizontal-line/theme/horizontalline.css */
 .ck-content hr {
     margin: 15px 0;
@@ -330,4 +335,4 @@
     .ck-content .page-break::after {
         display: none;
     }
-}
+}

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.45.2",
+  "version": "0.45.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -4838,6 +4838,11 @@
         "type-check": "~0.3.2"
       }
     },
+    "limiter": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz",
+      "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA=="
+    },
     "line-column": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/line-column/-/line-column-1.0.2.tgz",
@@ -6913,6 +6918,22 @@
       "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
       "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
     },
+    "stream-throttle": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/stream-throttle/-/stream-throttle-0.1.3.tgz",
+      "integrity": "sha1-rdV8jXzHOoFjDTHNVdOWHPr7qcM=",
+      "requires": {
+        "commander": "^2.2.0",
+        "limiter": "^1.0.5"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "2.20.3",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+          "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        }
+      }
+    },
     "streamsearch": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",

package.json

@@ -2,7 +2,7 @@
   "name": "trilium",
   "productName": "Trilium Notes",
   "description": "Trilium Notes",
-  "version": "0.45.3",
+  "version": "0.45.5",
   "license": "AGPL-3.0-only",
   "main": "electron.js",
   "bin": {
@@ -65,6 +65,7 @@
     "semver": "7.3.2",
     "serve-favicon": "2.5.0",
     "session-file-store": "1.5.0",
+    "stream-throttle": "^0.1.3",
     "striptags": "3.1.1",
     "tmp": "^0.2.1",
     "turndown": "7.0.0",

src/entities/attribute.js

@@ -34,6 +34,10 @@ class Attribute extends Entity {
         this.isInheritable = !!this.isInheritable;
     }
 
+    isAutoLink() {
+        return this.type === 'relation' && ['internalLink', 'imageLink', 'relationMapLink', 'includeNoteLink'].includes(this.name);
+    }
+
     /**
      * @returns {Note|null}
      */

src/entities/branch.js

@@ -38,7 +38,7 @@ class Branch extends Entity {
     }
 
     beforeSaving() {
-        if (this.notePosition === undefined) {
+        if (this.notePosition === undefined || this.notePosition === null) {
             const maxNotePos = sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [this.parentNoteId]);
             this.notePosition = maxNotePos === null ? 0 : maxNotePos + 10;
         }

src/public/app/services/frontend_script_api.js

@@ -274,7 +274,10 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
      *
      * @method
      * @param {string} notePath (or noteId)
-     * @param {string} [noteTitle] - if not present we'll use note title
+     * @param {object} [params]
+     * @param {boolean} [params.showTooltip=true] - enable/disable tooltip on the link
+     * @param {boolean} [params.showNotePath=false] - show also whole note's path as part of the link
+     * @param {string} [title=] - custom link tile with note's title as default
      */
     this.createNoteLink = linkService.createNoteLink;
 

src/public/app/services/note_create.js

@@ -88,7 +88,7 @@ function parseSelectedHtml(selectedHtml) {
     }
 }
 
-async function duplicateNote(noteId, parentNoteId) {
+async function duplicateSubtree(noteId, parentNoteId) {
     const {note} = await server.post(`notes/${noteId}/duplicate/${parentNoteId}`);
 
     await ws.waitForMaxKnownEntityChangeId();
@@ -102,5 +102,5 @@ async function duplicateNote(noteId, parentNoteId) {
 export default {
     createNote,
     createNewTopLevelNote,
-    duplicateNote
+    duplicateSubtree
 };

src/public/app/services/open.js

@@ -64,6 +64,7 @@ function getHost() {
 }
 
 export default {
+    download,
     downloadFileNote,
     openFileNote,
     downloadNoteRevision,

src/public/app/services/tree_context_menu.js

@@ -95,7 +95,7 @@ class TreeContextMenu {
                 enabled: !clipboard.isClipboardEmpty() && notSearch && noSelectedNotes },
             { title: 'Paste after', command: "pasteNotesAfterFromClipboard", uiIcon: "paste",
                 enabled: !clipboard.isClipboardEmpty() && isNotRoot && !isHoisted && parentNotSearch && noSelectedNotes },
-            { title: "Duplicate note(s) here", command: "duplicateNote", uiIcon: "empty",
+            { title: "Duplicate subtree(s) here", command: "duplicateSubtree", uiIcon: "empty",
                 enabled: parentNotSearch && isNotRoot && !isHoisted },
             { title: "----" },
             { title: "Export", command: "exportNote", uiIcon: "empty",

src/public/app/services/ws.js

@@ -8,8 +8,6 @@ import options from "./options.js";
 import treeCache from "./tree_cache.js";
 import noteAttributeCache from "./note_attribute_cache.js";
 
-const $outstandingSyncsCount = $("#outstanding-syncs-count");
-
 const messageHandlers = [];
 
 let ws;
@@ -64,8 +62,6 @@ async function handleMessage(event) {
         let syncRows = message.data;
         lastPingTs = Date.now();
 
-        $outstandingSyncsCount.html(message.outstandingSyncs);
-
         if (syncRows.length > 0) {
             logRows(syncRows);
 

src/public/app/setup.js

@@ -130,7 +130,7 @@ function SetupModel() {
 }
 
 async function checkOutstandingSyncs() {
-    const { stats, initialized } = await $.get('api/sync/stats');
+    const { outstandingPullCount, initialized } = await $.get('api/sync/stats');
 
     if (initialized) {
         if (utils.isElectron()) {
@@ -143,9 +143,7 @@ async function checkOutstandingSyncs() {
         }
     }
     else {
-        const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;
-
-        $("#outstanding-syncs").html(totalOutstandingSyncs);
+        $("#outstanding-syncs").html(outstandingPullCount);
     }
 }
 

src/public/app/widgets/global_menu.js

@@ -41,7 +41,7 @@ const TPL = `
 
             <a class="dropdown-item sync-now-button" title="Trigger sync">
                 <span class="bx bx-refresh"></span>
-                Sync now (<span id="outstanding-syncs-count">0</span>)
+                Sync now
             </a>
 
             <a class="dropdown-item" data-trigger-command="openNewWindow">

src/public/app/widgets/note_tree.js

@@ -1341,7 +1341,7 @@ export default class NoteTreeWidget extends TabAwareWidget {
         protectedSessionService.protectNote(node.data.noteId, false, true);
     }
 
-    duplicateNoteCommand({node}) {
+    duplicateSubtreeCommand({node}) {
         const nodesToDuplicate = this.getSelectedOrActiveNodes(node);
 
         for (const nodeToDuplicate of nodesToDuplicate) {
@@ -1353,7 +1353,7 @@ export default class NoteTreeWidget extends TabAwareWidget {
 
             const branch = treeCache.getBranch(nodeToDuplicate.data.branchId);
 
-            noteCreateService.duplicateNote(nodeToDuplicate.data.noteId, branch.parentNoteId);
+            noteCreateService.duplicateSubtree(nodeToDuplicate.data.noteId, branch.parentNoteId);
         }
     }
 }

src/routes/api/attributes.js

@@ -28,8 +28,10 @@ function updateNoteAttribute(req) {
             || body.name !== attribute.name
             || (body.type === 'relation' && body.value !== attribute.value)) {
 
+            let newAttribute;
+
             if (body.type !== 'relation' || !!body.value.trim()) {
-                const newAttribute = attribute.createClone(body.type, body.name, body.value);
+                newAttribute = attribute.createClone(body.type, body.name, body.value);
                 newAttribute.save();
             }
 
@@ -37,7 +39,7 @@ function updateNoteAttribute(req) {
             attribute.save();
 
             return {
-                attributeId: attribute.attributeId
+                attributeId: newAttribute ? newAttribute.attributeId : null
             };
         }
     }
@@ -52,8 +54,9 @@ function updateNoteAttribute(req) {
         attribute.type = body.type;
     }
 
-    if (body.value.trim()) {
+    if (body.type !== 'relation' || body.value.trim()) {
         attribute.value = body.value;
+        attribute.isDeleted = false;
     }
     else {
         // relations should never have empty target
@@ -144,8 +147,10 @@ function updateNoteAttributes(req) {
 
     // all the remaining existing attributes are not defined anymore and should be deleted
     for (const toDeleteAttr of existingAttrs) {
-        toDeleteAttr.isDeleted = true;
-        toDeleteAttr.save();
+        if (!toDeleteAttr.isAutoLink()) {
+            toDeleteAttr.isDeleted = true;
+            toDeleteAttr.save();
+        }
     }
 }
 

src/routes/api/notes.js

@@ -187,10 +187,10 @@ function changeTitle(req) {
     return note;
 }
 
-function duplicateNote(req) {
+function duplicateSubtree(req) {
     const {noteId, parentNoteId} = req.params;
 
-    return noteService.duplicateNote(noteId, parentNoteId);
+    return noteService.duplicateSubtree(noteId, parentNoteId);
 }
 
 module.exports = {
@@ -204,5 +204,5 @@ module.exports = {
     setNoteTypeMime,
     getRelationMap,
     changeTitle,
-    duplicateNote
+    duplicateSubtree
 };

src/routes/api/recent_changes.js

@@ -31,19 +31,36 @@ function getRecentChanges(req) {
         }
     }
 
+    // now we need to also collect date points not represented in note revisions:
+    // 1. creation for all notes (dateCreated)
+    // 2. deletion for deleted notes (dateModified)
     const notes = sql.getRows(`
-        SELECT
-            notes.noteId,
-            notes.isDeleted AS current_isDeleted,
-            notes.deleteId AS current_deleteId,
-            notes.isErased AS current_isErased,
-            notes.title AS current_title,
-            notes.isProtected AS current_isProtected,
-            notes.title,
-            notes.utcDateCreated AS utcDate,
-            notes.dateCreated AS date
-        FROM
-            notes`);
+            SELECT
+                notes.noteId,
+                notes.isDeleted AS current_isDeleted,
+                notes.deleteId AS current_deleteId,
+                notes.isErased AS current_isErased,
+                notes.title AS current_title,
+                notes.isProtected AS current_isProtected,
+                notes.title,
+                notes.utcDateCreated AS utcDate,
+                notes.dateCreated AS date
+            FROM
+                notes
+        UNION ALL
+            SELECT
+                notes.noteId,
+                notes.isDeleted AS current_isDeleted,
+                notes.deleteId AS current_deleteId,
+                notes.isErased AS current_isErased,
+                notes.title AS current_title,
+                notes.isProtected AS current_isProtected,
+                notes.title,
+                notes.utcDateModified AS utcDate,
+                notes.dateModified AS date
+            FROM
+                notes
+            WHERE notes.isDeleted = 1 AND notes.isErased = 0`);
 
     for (const note of notes) {
         if (noteCacheService.isInAncestor(note.noteId, ancestorNoteId)) {

src/routes/api/sender.js

@@ -4,6 +4,7 @@ const imageType = require('image-type');
 const imageService = require('../../services/image');
 const dateNoteService = require('../../services/date_notes');
 const noteService = require('../../services/notes');
+const attributeService = require('../../services/attributes');
 
 function uploadImage(req) {
     const file = req.file;
@@ -37,7 +38,7 @@ function saveNote(req) {
 
     if (req.body.labels) {
         for (const {name, value} of req.body.labels) {
-            note.setLabel(name, value);
+            note.setLabel(attributeService.sanitizeAttributeName(name), value);
         }
     }
 

src/routes/api/sync.js

@@ -13,13 +13,13 @@ const dateUtils = require('../../services/date_utils');
 const entityConstructor = require('../../entities/entity_constructor');
 const utils = require('../../services/utils');
 
-function testSync() {
+async function testSync() {
     try {
         if (!syncOptions.isSyncSetup()) {
             return { success: false, message: "Sync server host is not configured. Please configure sync first." };
         }
 
-        syncService.login();
+        await syncService.login();
 
         // login was successful so we'll kick off sync now
         // this is important in case when sync server has been just initialized
@@ -43,7 +43,7 @@ function getStats() {
 
     const stats = {
         initialized: optionService.getOption('initialized') === 'true',
-        stats: syncService.stats
+        outstandingPullCount: syncService.getOutstandingPullCount()
     };
 
     log.info(`Returning sync stats: ${JSON.stringify(stats)}`);

src/routes/routes.js

@@ -154,7 +154,7 @@ function register(app) {
     apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision);
     apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap);
     apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle);
-    apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateNote);
+    apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree);
 
     apiRoute(GET, '/api/edited-notes/:date', noteRevisionsApiRoute.getEditedNotesOnDate);
 

src/services/attributes.js

@@ -2,7 +2,6 @@
 
 const repository = require('./repository');
 const sql = require('./sql');
-const utils = require('./utils');
 const Attribute = require('../entities/attribute');
 
 const ATTRIBUTE_TYPES = [ 'label', 'relation' ];
@@ -146,6 +145,20 @@ function getBuiltinAttributeNames() {
         ]);
 }
 
+function sanitizeAttributeName(origName) {
+    let fixedName;
+
+    if (origName === '') {
+        fixedName = "unnamed";
+    }
+    else {
+        // any not allowed character should be replaced with underscore
+        fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
+    }
+
+    return fixedName;
+}
+
 module.exports = {
     getNotesWithLabel,
     getNotesWithLabels,
@@ -156,5 +169,6 @@ module.exports = {
     getAttributeNames,
     isAttributeType,
     isAttributeDangerous,
-    getBuiltinAttributeNames
+    getBuiltinAttributeNames,
+    sanitizeAttributeName
 };

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2020-11-10T22:54:39+01:00", buildRevision: "5157fc15e9f7fa960ee35685426868d5599933dc" };
+module.exports = { buildDate:"2020-11-20T22:50:10+01:00", buildRevision: "e5fa1e0ed555c1c2cb4a14c426d7091d62b5beea" };

src/services/consistency_checks.js

@@ -11,6 +11,7 @@ const entityChangesService = require('./entity_changes.js');
 const optionsService = require('./options');
 const Branch = require('../entities/branch');
 const dateUtils = require('./date_utils');
+const attributeService = require('./attributes');
 
 class ConsistencyChecks {
     constructor(autoFix) {
@@ -607,20 +608,10 @@ class ConsistencyChecks {
     findWronglyNamedAttributes() {
         const attrNames = sql.getColumn(`SELECT DISTINCT name FROM attributes`);
 
-        const attrNameMatcher = new RegExp("^[\\p{L}\\p{N}_:]+$", "u");
-
         for (const origName of attrNames) {
-            if (!attrNameMatcher.test(origName)) {
-                let fixedName;
-
-                if (origName === '') {
-                    fixedName = "unnamed";
-                }
-                else {
-                    // any not allowed character should be replaced with underscore
-                    fixedName = origName.replace(/[^\p{L}\p{N}_:]/ug, "_");
-                }
+            const fixedName = attributeService.sanitizeAttributeName(origName);
 
+            if (fixedName !== origName) {
                 if (this.autoFix) {
                     // there isn't a good way to update this:
                     // - just SQL query will fix it in DB but not notify frontend (or other caches) that it has been fixed

src/services/handlers.js

@@ -1,7 +1,7 @@
 const eventService = require('./events');
 const scriptService = require('./script');
 const treeService = require('./tree');
-const log = require('./log');
+const noteService = require('./notes');
 const repository = require('./repository');
 const Attribute = require('../entities/attribute');
 
@@ -58,17 +58,21 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) =>
                 return;
             }
 
-            const targetNote = repository.getNote(entity.value);
+            const templateNote = repository.getNote(entity.value);
 
-            if (!targetNote || !targetNote.isStringNote()) {
+            if (!templateNote) {
                 return;
             }
 
-            const targetNoteContent = targetNote.getContent();
+            if (templateNote.isStringNote()) {
+                const templateNoteContent = templateNote.getContent();
 
-            if (targetNoteContent) {
-                note.setContent(targetNoteContent);
+                if (templateNoteContent) {
+                    note.setContent(templateNoteContent);
+                }
             }
+
+            noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId);
         }
         else if (entity.type === 'label' && entity.name === 'sorted') {
             treeService.sortNotesAlphabetically(entity.noteId);

src/services/image.js

@@ -37,7 +37,7 @@ function getImageType(buffer) {
         }
     }
     else {
-        return imageType(buffer);
+        return imageType(buffer) || "jpg"; // optimistic JPG default
     }
 }
 

src/services/import/enex.js

@@ -1,5 +1,6 @@
 const sax = require("sax");
 const stream = require('stream');
+const {Throttle} = require('stream-throttle');
 const log = require("../log");
 const utils = require("../utils");
 const sql = require("../sql");
@@ -7,6 +8,7 @@ const noteService = require("../notes");
 const imageService = require("../image");
 const protectedSessionService = require('../protected_session');
 const htmlSanitizer = require("../html_sanitizer");
+const attributeService = require("../attributes");
 
 // date format is e.g. 20181121T193703Z
 function parseDate(text) {
@@ -37,10 +39,6 @@ function importEnex(taskContext, file, parentNote) {
         isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(),
     })).note;
 
-    // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished
-    // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully.
-    const saveNotePromises = [];
-
     function extractContent(content) {
         const openingNoteIndex = content.indexOf('<en-note>');
 
@@ -105,9 +103,17 @@ function importEnex(taskContext, file, parentNote) {
         const previousTag = getPreviousTag();
 
         if (previousTag === 'note-attributes') {
+            let labelName = currentTag;
+
+            if (labelName === 'source-url') {
+                labelName = 'sourceUrl';
+            }
+
+            labelName = attributeService.sanitizeAttributeName(labelName);
+
             note.attributes.push({
                 type: 'label',
-                name: currentTag,
+                name: labelName,
                 value: text
             });
         }
@@ -149,7 +155,7 @@ function importEnex(taskContext, file, parentNote) {
             } else if (currentTag === 'tag') {
                 note.attributes.push({
                     type: 'label',
-                    name: text,
+                    name: attributeService.sanitizeAttributeName(text),
                     value: ''
                 })
             }
@@ -227,6 +233,10 @@ function importEnex(taskContext, file, parentNote) {
         taskContext.increaseProgressCount();
 
         for (const resource of resources) {
+            if (!resource.content) {
+                continue;
+            }
+
             const hash = utils.md5(resource.content);
 
             const mediaRegex = new RegExp(`<en-media hash="${hash}"[^>]*>`, 'g');
@@ -304,7 +314,7 @@ function importEnex(taskContext, file, parentNote) {
         path.pop();
 
         if (tag === 'note') {
-            saveNotePromises.push(saveNote());
+            saveNote();
         }
     });
 
@@ -323,12 +333,15 @@ function importEnex(taskContext, file, parentNote) {
     return new Promise((resolve, reject) =>
     {
         // resolve only when we parse the whole document AND saving of all notes have been finished
-        saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(rootNote)) });
+        saxStream.on("end", () => resolve(rootNote));
 
         const bufferStream = new stream.PassThrough();
         bufferStream.end(file.buffer);
 
-        bufferStream.pipe(saxStream);
+        bufferStream
+            // rate limiting to improve responsiveness during / after import
+            .pipe(new Throttle({rate: 500000}))
+            .pipe(saxStream);
     });
 }
 

src/services/notes.js

@@ -719,26 +719,68 @@ function eraseDeletedNotes() {
     log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
 }
 
-function duplicateNote(noteId, parentNoteId) {
-    const origNote = repository.getNote(noteId);
+// do a replace in str - all keys should be replaced by the corresponding values
+function replaceByMap(str, mapObj) {
+    const re = new RegExp(Object.keys(mapObj).join("|"),"g");
 
+    return str.replace(re, matched => mapObj[matched]);
+}
+
+function duplicateSubtree(origNoteId, newParentNoteId) {
+    if (origNoteId === 'root') {
+        throw new Error('Duplicating root is not possible');
+    }
+
+    const origNote = repository.getNote(origNoteId);
+    // might be null if orig note is not in the target newParentNoteId
+    const origBranch = origNote.getBranches().find(branch => branch.parentNoteId === newParentNoteId);
+
+    const noteIdMapping = getNoteIdMapping(origNote);
+
+    const res = duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapping);
+
+    res.note.title += " (dup)";
+    res.note.save();
+
+    return res;
+}
+
+function duplicateSubtreeWithoutRoot(origNoteId, newNoteId) {
+    if (origNoteId === 'root') {
+        throw new Error('Duplicating root is not possible');
+    }
+
+    const origNote = repository.getNote(origNoteId);
+    const noteIdMapping = getNoteIdMapping(origNote);
+
+    for (const childBranch of origNote.getChildBranches()) {
+        duplicateSubtreeInner(childBranch.getNote(), childBranch, newNoteId, noteIdMapping);
+    }
+}
+
+function duplicateSubtreeInner(origNote, origBranch, newParentNoteId, noteIdMapping) {
     if (origNote.isProtected && !protectedSessionService.isProtectedSessionAvailable()) {
         throw new Error(`Cannot duplicate note=${origNote.noteId} because it is protected and protected session is not available`);
     }
 
-    // might be null if orig note is not in the target parentNoteId
-    const origBranch = origNote.getBranches().find(branch => branch.parentNoteId === parentNoteId);
-
     const newNote = new Note(origNote);
-    newNote.noteId = undefined; // force creation of new note
-    newNote.title += " (dup)";
+    newNote.noteId = noteIdMapping[origNote.noteId];
+    newNote.dateCreated = dateUtils.localNowDateTime();
+    newNote.utcDateCreated = dateUtils.utcNowDateTime();
     newNote.save();
 
-    newNote.setContent(origNote.getContent());
+    let content = origNote.getContent();
+
+    if (['text', 'relation-map', 'search'].includes(origNote.type)) {
+        // fix links in the content
+        content = replaceByMap(content, noteIdMapping);
+    }
+
+    newNote.setContent(content);
 
     const newBranch = new Branch({
         noteId: newNote.noteId,
-        parentNoteId: parentNoteId,
+        parentNoteId: newParentNoteId,
         // here increasing just by 1 to make sure it's directly after original
         notePosition: origBranch ? origBranch.notePosition + 1 : null
     }).save();
@@ -746,17 +788,38 @@ function duplicateNote(noteId, parentNoteId) {
     for (const attribute of origNote.getOwnedAttributes()) {
         const attr = new Attribute(attribute);
         attr.attributeId = undefined; // force creation of new attribute
+        attr.utcDateCreated = dateUtils.utcNowDateTime();
         attr.noteId = newNote.noteId;
 
+        // if relation points to within the duplicated tree then replace the target to the duplicated note
+        // if it points outside of duplicated tree then keep the original target
+        if (attr.type === 'relation' && attr.value in noteIdMapping) {
+            attr.value = noteIdMapping[attr.value];
+        }
+
         attr.save();
     }
 
+    for (const childBranch of origNote.getChildBranches()) {
+        duplicateSubtreeInner(childBranch.getNote(), childBranch, newNote.noteId, noteIdMapping);
+    }
+
     return {
         note: newNote,
         branch: newBranch
     };
 }
 
+function getNoteIdMapping(origNote) {
+    const noteIdMapping = {};
+
+    // pregenerate new noteIds since we'll need to fix relation references even for not yet created notes
+    for (const origNoteId of origNote.getDescendantNoteIds()) {
+        noteIdMapping[origNoteId] = utils.newEntityId();
+    }
+    return noteIdMapping;
+}
+
 sqlInit.dbReady.then(() => {
     // first cleanup kickoff 5 minutes after startup
     setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000);
@@ -772,7 +835,8 @@ module.exports = {
     undeleteNote,
     protectNoteRecursively,
     scanForLinks,
-    duplicateNote,
+    duplicateSubtree,
+    duplicateSubtreeWithoutRoot,
     getUndeletedParentBranches,
     triggerNoteTitleChanged
 };

src/services/search/services/search.js

@@ -17,10 +17,14 @@ const utils = require('../../utils.js');
  */
 function findNotesWithExpression(expression) {
     const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()];
-    const allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
+    let allNotes = (hoistedNote && hoistedNote.noteId !== 'root')
         ? hoistedNote.subtreeNotes
         : Object.values(noteCache.notes);
 
+    // in the process of loading data sometimes we create "skeleton" note instances which are expected to be filled later
+    // in case of inconsistent data this might not work and search will then crash on these
+    allNotes = allNotes.filter(note => note.type !== undefined);
+
     const allNoteSet = new NoteSet(allNotes);
 
     const searchContext = {

src/services/sync.js

@@ -20,10 +20,7 @@ const entityConstructor = require('../entities/entity_constructor');
 
 let proxyToggle = true;
 
-const stats = {
-    outstandingPushes: 0,
-    outstandingPulls: 0
-};
+let outstandingPullCount = 0;
 
 async function sync() {
     try {
@@ -135,11 +132,7 @@ async function pullChanges(syncContext) {
 
         const pulledDate = Date.now();
 
-        stats.outstandingPulls = resp.maxEntityChangeId - lastSyncedPull;
-
-        if (stats.outstandingPulls < 0) {
-            stats.outstandingPulls = 0;
-        }
+        outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull);
 
         const {entityChanges} = resp;
 
@@ -159,13 +152,13 @@ async function pullChanges(syncContext) {
                     syncUpdateService.updateEntity(entityChange, entity, syncContext.sourceId);
                 }
 
-                stats.outstandingPulls = resp.maxEntityChangeId - entityChange.id;
+                outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id);
             }
 
             setLastSyncedPull(entityChanges[entityChanges.length - 1].entityChange.id);
         });
 
-        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`);
+        log.info(`Pulled ${entityChanges.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${outstandingPullCount} outstanding pulls`);
     }
 
     if (atLeastOnePullApplied) {
@@ -359,31 +352,25 @@ function setLastSyncedPush(entityChangeId) {
     optionService.setOption('lastSyncedPush', entityChangeId);
 }
 
-function updatePushStats() {
-    if (syncOptions.isSyncSetup()) {
-        const lastSyncedPush = optionService.getOption('lastSyncedPush');
-
-        stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE isSynced = 1 AND id > ?", [lastSyncedPush]);
-    }
-}
-
 function getMaxEntityChangeId() {
     return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes');
 }
 
+function getOutstandingPullCount() {
+    return outstandingPullCount;
+}
+
 sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(sync), 60000);
 
     // kickoff initial sync immediately
     setTimeout(cls.wrap(sync), 3000);
-
-    setInterval(cls.wrap(updatePushStats), 1000);
 });
 
 module.exports = {
     sync,
     login,
     getEntityChangesRecords,
-    stats,
+    getOutstandingPullCount,
     getMaxEntityChangeId
 };

src/services/ws.js

@@ -110,8 +110,7 @@ function sendPing(client, syncRows = []) {
 
     sendMessage(client, {
         type: 'sync',
-        data: syncRows,
-        outstandingSyncs: stats.outstandingPushes + stats.outstandingPulls
+        data: syncRows
     });
 }