Merge branch 'master' into next61

# Conflicts:
#	package-lock.json
#	src/public/app/services/note_content_renderer.js
#	src/public/app/widgets/note_tree.js
#	src/routes/routes.js
#	src/services/consistency_checks.js
#	src/services/notes.js
#	src/services/task_context.js

src/routes/api/clipper.js

@@ -1,6 +1,7 @@
 "use strict";
 
 const attributeService = require("../../services/attributes");
+const cloneService = require("../../services/cloning");
 const noteService = require('../../services/notes');
 const dateNoteService = require('../../services/date_notes');
 const dateUtils = require('../../services/date_utils');
@@ -13,46 +14,25 @@ const path = require('path');
 const BAttribute = require('../../becca/entities/battribute');
 const htmlSanitizer = require('../../services/html_sanitizer');
 const {formatAttrForSearch} = require("../../services/attribute_formatter");
-
-function findClippingNote(clipperInboxNote, pageUrl) {
-    const notes = clipperInboxNote.searchNotesInSubtree(
-        formatAttrForSearch({
-            type: 'label',
-            name: "pageUrl",
-            value: pageUrl
-        }, true)
-    );
-
-    for (const note of notes) {
-        if (note.getOwnedLabelValue('clipType') === 'clippings') {
-            return note;
-        }
-    }
-
-    return null;
-}
-
-function getClipperInboxNote() {
-    let clipperInbox = attributeService.getNoteWithLabel('clipperInbox');
-
-    if (!clipperInbox) {
-        clipperInbox = dateNoteService.getDayNote(dateUtils.localNowDate());
-    }
-
-    return clipperInbox;
-}
+const jsdom = require("jsdom");
+const { JSDOM } = jsdom;
 
 function addClipping(req) {
+    // if a note under the clipperInbox as the same 'pageUrl' attribute,
+    // add the content to that note and clone it under today's inbox
+    // otherwise just create a new note under today's inbox
     let {title, content, pageUrl, images} = req.body;
+    const clipType = 'clippings';
 
     const clipperInbox = getClipperInboxNote();
+    const dailyNote = dateNoteService.getDayNote(dateUtils.localNowDate());
 
     pageUrl = htmlSanitizer.sanitizeUrl(pageUrl);
-    let clippingNote = findClippingNote(clipperInbox, pageUrl);
+    let clippingNote = findClippingNote(clipperInbox, pageUrl, clipType);
 
     if (!clippingNote) {
         clippingNote = noteService.createNewNote({
-            parentNoteId: clipperInbox.noteId,
+            parentNoteId: dailyNote.noteId,
             title: title,
             content: '',
             type: 'text'
@@ -67,13 +47,45 @@ function addClipping(req) {
 
     const existingContent = clippingNote.getContent();
 
-    clippingNote.setContent(`${existingContent}${existingContent.trim() ? "<br/>" : ""}${rewrittenContent}`);
+    clippingNote.setContent(`${existingContent}${existingContent.trim() ? "<br>" : ""}${rewrittenContent}`);
+
+    if (clippingNote.parentNoteId !== dailyNote.noteId) {
+        cloneService.cloneNoteToParentNote(clippingNote.noteId, dailyNote.noteId);
+    }
 
     return {
         noteId: clippingNote.noteId
     };
 }
 
+function findClippingNote(clipperInboxNote, pageUrl, clipType) {
+    if (!pageUrl) {
+        return null;
+    }
+
+    const notes = clipperInboxNote.searchNotesInSubtree(
+        formatAttrForSearch({
+            type: 'label',
+            name: "pageUrl",
+            value: pageUrl
+        }, true)
+    );
+
+    return clipType
+        ? notes.find(note => note.getOwnedLabelValue('clipType') === clipType)
+        : notes[0];
+}
+
+function getClipperInboxNote() {
+    let clipperInbox = attributeService.getNoteWithLabel('clipperInbox');
+
+    if (!clipperInbox) {
+        clipperInbox = dateNoteService.getRootCalendarNote();
+    }
+
+    return clipperInbox;
+}
+
 function createNote(req) {
     let {title, content, pageUrl, images, clipType, labels} = req.body;
 
@@ -81,26 +93,31 @@ function createNote(req) {
         title = `Clipped note from ${pageUrl}`;
     }
 
-    const clipperInbox = getClipperInboxNote();
-
-    const {note} = noteService.createNewNote({
-        parentNoteId: clipperInbox.noteId,
-        title,
-        content,
-        type: 'text'
-    });
-
     clipType = htmlSanitizer.sanitize(clipType);
 
-    note.setLabel('clipType', clipType);
+    const clipperInbox = getClipperInboxNote();
+    const dailyNote = dateNoteService.getDayNote(dateUtils.localNowDate());
+    pageUrl = htmlSanitizer.sanitizeUrl(pageUrl);
+    let note = findClippingNote(clipperInbox, pageUrl, clipType);
 
-    if (pageUrl) {
-        pageUrl = htmlSanitizer.sanitizeUrl(pageUrl);
+    if (!note) {
+        note = noteService.createNewNote({
+            parentNoteId: dailyNote.noteId,
+            title,
+            content: '',
+            type: 'text'
+        }).note;
 
-        note.setLabel('pageUrl', pageUrl);
-        note.setLabel('iconClass', 'bx bx-globe');
+        note.setLabel('clipType', clipType);
+
+        if (pageUrl) {
+            pageUrl = htmlSanitizer.sanitizeUrl(pageUrl);
+
+            note.setLabel('pageUrl', pageUrl);
+            note.setLabel('iconClass', 'bx bx-globe');
+        }
     }
-    
+
     if (labels) {
         for (const labelName in labels) {
             const labelValue = htmlSanitizer.sanitize(labels[labelName]);
@@ -108,9 +125,9 @@ function createNote(req) {
         }
     }
 
+    const existingContent = note.getContent();
     const rewrittenContent = processContent(images, note, content);
-
-    note.setContent(rewrittenContent);
+    note.setContent(`${existingContent}${existingContent.trim() ? "<br/>" : ""}${rewrittenContent}`);
 
     return {
         noteId: note.noteId
@@ -158,6 +175,15 @@ function processContent(images, note, content) {
 
     // fallback if parsing/downloading images fails for some reason on the extension side (
     rewrittenContent = noteService.downloadImages(note.noteId, rewrittenContent);
+    // Check if rewrittenContent contains at least one HTML tag
+    if (!/<.+?>/.test(rewrittenContent)) {
+        rewrittenContent = `<p>${rewrittenContent}</p>`;
+    }
+    // Create a JSDOM object from the existing HTML content
+    const dom = new JSDOM(rewrittenContent);
+
+    // Get the content inside the body tag and serialize it
+    rewrittenContent = dom.window.document.body.innerHTML;
 
     return rewrittenContent;
 }
@@ -187,9 +213,19 @@ function handshake() {
     }
 }
 
+function findNotesByUrl(req){
+    let pageUrl = req.params.noteUrl;
+    const clipperInbox = getClipperInboxNote();
+    let foundPage = findClippingNote(clipperInbox, pageUrl, null);
+    return {
+        noteId: foundPage ? foundPage.noteId : null
+    }
+}
+
 module.exports = {
     createNote,
     addClipping,
     openNote,
-    handshake
+    handshake,
+    findNotesByUrl
 };

src/routes/api/options.js

@@ -49,7 +49,7 @@ const ALLOWED_OPTIONS = new Set([
     'compressImages',
     'downloadImagesAutomatically',
     'minTocHeadings',
-    'highlightedText',
+    'highlightsList',
     'checkForUpdates',
     'disableTray',
     'eraseUnusedAttachmentsAfterSeconds',

src/routes/api/recent_notes.js

@@ -11,7 +11,7 @@ function addRecentNote(req) {
     }).save();
 
     if (Math.random() < 0.05) {
-        // it's not necessary to run this everytime ...
+        // it's not necessary to run this every time ...
         const cutOffDate = dateUtils.utcDateTimeStr(new Date(Date.now() - 24 * 3600 * 1000));
 
         sql.execute(`DELETE FROM recent_notes WHERE utcDateCreated < ?`, [cutOffDate]);

src/routes/api/sql.js

@@ -28,6 +28,12 @@ function execute(req) {
         for (let query of queries) {
             query = query.trim();
 
+            while (query.startsWith('-- ')) {
+                // Query starts with one or more SQL comments, discard these before we execute.
+                const pivot = query.indexOf('\n');
+                query = pivot > 0 ? query.substr(pivot + 1).trim() : "";
+            }
+            
             if (!query) {
                 continue;
             }

src/routes/api/sync.js

@@ -62,7 +62,7 @@ function checkSync() {
 function syncNow() {
     log.info("Received request to trigger sync now.");
 
-    // when explicitly asked for set in progress status immediatelly for faster user feedback
+    // when explicitly asked for set in progress status immediately for faster user feedback
     ws.syncPullInProgress();
 
     return syncService.sync();