From 7ac2206e9b4a1d9cb3a35b39197197e534e2fd6b Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 3 May 2020 09:18:57 +0200 Subject: [PATCH 01/47] start of search overhaul --- src/services/note_cache.js | 254 +++++++++++++++++++++++++------------ 1 file changed, 176 insertions(+), 78 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 0012d2545..4285eccd9 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -7,6 +7,108 @@ const utils = require('./utils'); const hoistedNoteService = require('./hoisted_note'); const stringSimilarity = require('string-similarity'); +/** @var {Object.} */ +let notes; +/** @var {Object.} */ +let branches +/** @var {Object.} */ +let attributes; + +/** @var {Object.} */ +let noteAttributeCache = {}; + +let childParentToBranch = {}; + +class Note { + constructor(row) { + /** @param {string} */ + this.noteId = row.noteId; + /** @param {string} */ + this.title = row.title; + /** @param {boolean} */ + this.isProtected = !!row.isProtected; + /** @param {Note[]} */ + this.parents = []; + /** @param {Attribute[]} */ + this.ownedAttributes = []; + } + + /** @return {Attribute[]} */ + get attributes() { + if (this.noteId in noteAttributeCache) { + const attrArrs = [ + this.ownedAttributes + ]; + + for (const templateAttr of this.ownedAttributes.filter(oa => oa.type === 'relation' && oa.name === 'template')) { + const templateNote = notes[templateAttr.value]; + + if (templateNote) { + attrArrs.push(templateNote.attributes); + } + } + + if (this.noteId !== 'root') { + for (const parentNote of this.parents) { + attrArrs.push(parentNote.inheritableAttributes); + } + } + + noteAttributeCache[this.noteId] = attrArrs.flat(); + } + + return noteAttributeCache[this.noteId]; + } + + /** @return {Attribute[]} */ + get inheritableAttributes() { + return this.attributes.filter(attr => attr.isInheritable); + } + + hasAttribute(type, name) { + return this.attributes.find(attr => attr.type === type && attr.name === name); + } + + get isArchived() { + return this.hasAttribute('label', 'archived'); + } +} + +class Branch { + constructor(row) { + /** @param {string} */ + this.branchId = row.branchId; + /** @param {string} */ + this.noteId = row.noteId; + /** @param {string} */ + this.parentNoteId = row.parentNoteId; + /** @param {string} */ + this.prefix = row.prefix; + } + + /** @return {Note} */ + get parentNote() { + return notes[this.parentNoteId]; + } +} + +class Attribute { + constructor(row) { + /** @param {string} */ + this.attributeId = row.attributeId; + /** @param {string} */ + this.noteId = row.noteId; + /** @param {string} */ + this.type = row.type; + /** @param {string} */ + this.name = row.name; + /** @param {string} */ + this.value = row.value; + /** @param {boolean} */ + this.isInheritable = row.isInheritable; + } +} + let loaded = false; let loadedPromiseResolve; /** Is resolved after the initial load */ @@ -15,49 +117,60 @@ let loadedPromise = new Promise(res => loadedPromiseResolve = res); let noteTitles = {}; let protectedNoteTitles = {}; let noteIds; -let childParentToBranchId = {}; const childToParent = {}; let archived = {}; // key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here let prefixes = {}; -async function load() { - noteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 0`); - noteIds = Object.keys(noteTitles); +async function getMappedRows(query, cb) { + const map = {}; + const results = await sql.getRows(query, []); - prefixes = await sql.getMap(` - SELECT noteId || '-' || parentNoteId, prefix - FROM branches - WHERE isDeleted = 0 AND prefix IS NOT NULL AND prefix != ''`); + for (const row of results) { + const keys = Object.keys(row); - const branches = await sql.getRows(`SELECT branchId, noteId, parentNoteId FROM branches WHERE isDeleted = 0`); - - for (const rel of branches) { - childToParent[rel.noteId] = childToParent[rel.noteId] || []; - childToParent[rel.noteId].push(rel.parentNoteId); - childParentToBranchId[`${rel.noteId}-${rel.parentNoteId}`] = rel.branchId; + map[row[keys[0]]] = cb(row); } - archived = await sql.getMap(`SELECT noteId, isInheritable FROM attributes WHERE isDeleted = 0 AND type = 'label' AND name = 'archived'`); + return map; +} + +async function load() { + notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, + row => new Note(row)); + + branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, + row => new Branch(row)); + + attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, + row => new Attribute(row)); + + for (const branch of branches) { + const childNote = notes[branch.noteId]; + + if (!childNote) { + console.log(`Cannot find child note ${branch.noteId} of a branch ${branch.branchId}`); + continue; + } + + childNote.parents.push(branch.parentNote); + childParentToBranch[`${branch.noteId}-${branch.parentNoteId}`] = branch; + } if (protectedSessionService.isProtectedSessionAvailable()) { - await loadProtectedNotes(); - } - - for (const noteId in childToParent) { - resortChildToParent(noteId); + await decryptProtectedNotes(); } loaded = true; loadedPromiseResolve(); } -async function loadProtectedNotes() { - protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`); - - for (const noteId in protectedNoteTitles) { - protectedNoteTitles[noteId] = protectedSessionService.decryptString(protectedNoteTitles[noteId]); +async function decryptProtectedNotes() { + for (const note of notes) { + if (note.isProtected) { + note.title = protectedSessionService.decryptString(note.title); + } } } @@ -103,13 +216,9 @@ async function findNotes(query) { const tokens = allTokens.slice(); let results = []; - let noteIds = Object.keys(noteTitles); + for (const noteId in notes) { + const note = notes[noteId]; - if (protectedSessionService.isProtectedSessionAvailable()) { - noteIds = [...new Set(noteIds.concat(Object.keys(protectedNoteTitles)))]; - } - - for (const noteId of noteIds) { // autocomplete should be able to find notes by their noteIds as well (only leafs) if (noteId === query) { search(noteId, [], [], results); @@ -117,22 +226,12 @@ async function findNotes(query) { } // for leaf note it doesn't matter if "archived" label is inheritable or not - if (noteId in archived) { + if (note.isArchived) { continue; } - const parents = childToParent[noteId]; - if (!parents) { - continue; - } - - for (const parentNoteId of parents) { - // for parent note archived needs to be inheritable - if (archived[parentNoteId] === 1) { - continue; - } - - const title = getNoteTitle(noteId, parentNoteId).toLowerCase(); + for (const parentNote of note.parents) { + const title = getNoteTitle(note, parentNote).toLowerCase(); const foundTokens = []; for (const token of tokens) { @@ -144,7 +243,7 @@ async function findNotes(query) { if (foundTokens.length > 0) { const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - search(parentNoteId, remainingTokens, [noteId], results); + search(parentNote, remainingTokens, [noteId], results); } } } @@ -180,17 +279,21 @@ async function findNotes(query) { return apiResults; } -function search(noteId, tokens, path, results) { - if (tokens.length === 0) { - const retPath = getSomePath(noteId, path); +function getBranch(childNoteId, parentNoteId) { + return childParentToBranch[`${childNoteId}-${parentNoteId}`]; +} - if (retPath && !isNotePathArchived(retPath)) { +function search(note, tokens, path, results) { + if (tokens.length === 0) { + const retPath = getSomePath(note, path); + + if (retPath) { const thisNoteId = retPath[retPath.length - 1]; const thisParentNoteId = retPath[retPath.length - 2]; results.push({ noteId: thisNoteId, - branchId: childParentToBranchId[`${thisNoteId}-${thisParentNoteId}`], + branchId: getBranch(thisNoteId, thisParentNoteId), pathArray: retPath, titleArray: getNoteTitleArrayForPath(retPath) }); @@ -199,18 +302,12 @@ function search(noteId, tokens, path, results) { return; } - const parents = childToParent[noteId]; - if (!parents || noteId === 'root') { + if (!note.parents.length === 0 || noteId === 'root') { return; } - for (const parentNoteId of parents) { - // archived must be inheritable - if (archived[parentNoteId] === 1) { - continue; - } - - const title = getNoteTitle(noteId, parentNoteId).toLowerCase(); + for (const parentNote of note.parents) { + const title = getNoteTitle(note, parentNote).toLowerCase(); const foundTokens = []; for (const token of tokens) { @@ -222,17 +319,18 @@ function search(noteId, tokens, path, results) { if (foundTokens.length > 0) { const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - search(parentNoteId, remainingTokens, path.concat([noteId]), results); + search(parentNote, remainingTokens, path.concat([noteId]), results); } else { - search(parentNoteId, tokens, path.concat([noteId]), results); + search(parentNote, tokens, path.concat([noteId]), results); } } } function isNotePathArchived(notePath) { - // if the note is archived directly - if (archived[notePath[notePath.length - 1]] !== undefined) { + const noteId = notePath[notePath.length - 1]; + + if (archived[noteId] !== undefined) { return true; } @@ -268,8 +366,10 @@ function isInAncestor(noteId, ancestorNoteId) { return true; } - for (const parentNoteId of childToParent[noteId] || []) { - if (isInAncestor(parentNoteId, ancestorNoteId)) { + const note = notes[noteId]; + + for (const parentNote of notes.parents) { + if (isInAncestor(parentNote.noteId, ancestorNoteId)) { return true; } } @@ -288,21 +388,19 @@ function getNoteTitleFromPath(notePath) { } } -function getNoteTitle(noteId, parentNoteId) { - const prefix = prefixes[noteId + '-' + parentNoteId]; +function getNoteTitle(childNote, parentNote) { + let title; - let title = noteTitles[noteId]; - - if (!title) { - if (protectedSessionService.isProtectedSessionAvailable()) { - title = protectedNoteTitles[noteId]; - } - else { - title = '[protected]'; - } + if (childNote.isProtected) { + title = protectedSessionService.isProtectedSessionAvailable() ? childNote.title : '[protected]'; + } + else { + title = childNote.title; } - return (prefix ? (prefix + ' - ') : '') + title; + const branch = getBranch(childNote.noteId, parentNote.noteId); + + return (branch.prefix ? (branch.prefix + ' - ') : '') + title; } function getNoteTitleArrayForPath(path) { @@ -540,7 +638,7 @@ function isAvailable(noteId) { } eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { - loadedPromise.then(() => loadProtectedNotes()); + loadedPromise.then(() => decryptProtectedNotes()); }); sqlInit.dbReady.then(() => utils.stopWatch("Note cache load", load)); From 552fc5261aab954230ea73b9a217b613a9f1eabf Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 9 May 2020 23:42:26 +0200 Subject: [PATCH 02/47] new note cache WIP --- .idea/dataSources.xml | 2 +- src/services/note_cache.js | 78 +++++++++++++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index 70caa7f2a..fa5ae48bf 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -1,7 +1,7 @@ - + sqlite.xerial true org.sqlite.JDBC diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 4285eccd9..de4ee09f0 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -7,14 +7,14 @@ const utils = require('./utils'); const hoistedNoteService = require('./hoisted_note'); const stringSimilarity = require('string-similarity'); -/** @var {Object.} */ +/** @type {Object.} */ let notes; -/** @var {Object.} */ +/** @type {Object.} */ let branches -/** @var {Object.} */ +/** @type {Object.} */ let attributes; -/** @var {Object.} */ +/** @type {Object.} */ let noteAttributeCache = {}; let childParentToBranch = {}; @@ -109,6 +109,59 @@ class Attribute { } } +class FulltextReference { + /** + * @param type - attributeName, attributeValue, title + * @param id - attributeId, noteId + */ + constructor(type, id) { + this.type = type; + this.id = id; + } +} + +/** @type {Object.} */ +let fulltext = {}; + +/** @type {Object.} */ +let attributeMetas = {}; + +class AttributeMeta { + constructor(attribute) { + this.type = attribute.type; + this.name = attribute.name; + this.isInheritable = attribute.isInheritable; + this.attributeIds = new Set(attribute.attributeId); + } + + addAttribute(attribute) { + this.attributeIds.add(attribute.attributeId); + this.isInheritable = this.isInheritable || attribute.isInheritable; + } + + updateAttribute(attribute) { + if (attribute.isDeleted) { + this.attributeIds.delete(attribute.attributeId); + } + else { + this.attributeIds.add(attribute.attributeId); + } + + this.isInheritable = !!this.attributeIds.find(attributeId => attributes[attributeId].isInheritable); + } +} + +function addToAttributeMeta(attribute) { + const key = `${attribute.type}-${attribute.name}`; + + if (!(key in attributeMetas)) { + attributeMetas[key] = new AttributeMeta(attribute); + } + else { + attributeMetas[key].addAttribute(attribute); + } +} + let loaded = false; let loadedPromiseResolve; /** Is resolved after the initial load */ @@ -140,12 +193,27 @@ async function load() { notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, row => new Note(row)); + for (const note of notes) { + fulltext[note.title] = fulltext[note.title] || []; + fulltext[note.title].push(new FulltextReference('note', note.noteId)); + } + branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, row => new Branch(row)); attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, row => new Attribute(row)); + for (const attr of attributes) { + addToAttributeMeta(attributes); + + fulltext[attr.name] = fulltext[attr.name] || []; + fulltext[attr.name].push(new FulltextReference('aName', attr.attributeId)); + + fulltext[attr.value] = fulltext[attr.value] || []; + fulltext[attr.value].push(new FulltextReference('aVal', attr.attributeId)); + } + for (const branch of branches) { const childNote = notes[branch.noteId]; @@ -654,4 +722,4 @@ module.exports = { isInAncestor, load, findSimilarNotes -}; \ No newline at end of file +}; From 15bc9dce1c6a964edb86484af4f28de3d0eec0c5 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 10 May 2020 23:27:53 +0200 Subject: [PATCH 03/47] search overhaul WIP --- src/services/note_cache.js | 118 ++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 33 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index de4ee09f0..958d4265c 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -35,7 +35,7 @@ class Note { /** @return {Attribute[]} */ get attributes() { - if (this.noteId in noteAttributeCache) { + if (!(this.noteId in noteAttributeCache)) { const attrArrs = [ this.ownedAttributes ]; @@ -88,7 +88,13 @@ class Branch { /** @return {Note} */ get parentNote() { - return notes[this.parentNoteId]; + const note = notes[this.parentNoteId]; + + if (!note) { + console.log(`Cannot find note ${this.parentNoteId}`); + } + + return note; } } @@ -112,11 +118,11 @@ class Attribute { class FulltextReference { /** * @param type - attributeName, attributeValue, title - * @param id - attributeId, noteId + * @param noteId */ - constructor(type, id) { + constructor(type, noteId) { this.type = type; - this.id = id; + this.noteId = noteId; } } @@ -193,9 +199,11 @@ async function load() { notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, row => new Note(row)); - for (const note of notes) { - fulltext[note.title] = fulltext[note.title] || []; - fulltext[note.title].push(new FulltextReference('note', note.noteId)); + for (const note of Object.values(notes)) { + const title = note.title.toLowerCase(); + + fulltext[title] = fulltext[title] || []; + fulltext[title].push(new FulltextReference('note', note.noteId)); } branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, @@ -204,17 +212,25 @@ async function load() { attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, row => new Attribute(row)); - for (const attr of attributes) { + for (const attr of Object.values(attributes)) { + notes[attr.noteId].attributes.push(attr); + addToAttributeMeta(attributes); - fulltext[attr.name] = fulltext[attr.name] || []; - fulltext[attr.name].push(new FulltextReference('aName', attr.attributeId)); + const attrName = attr.name.toLowerCase(); + fulltext[attrName] = fulltext[attrName] || []; + fulltext[attrName].push(new FulltextReference('aName', attr.noteId)); - fulltext[attr.value] = fulltext[attr.value] || []; - fulltext[attr.value].push(new FulltextReference('aVal', attr.attributeId)); + const attrValue = attr.value.toLowerCase(); + fulltext[attrValue] = fulltext[attrValue] || []; + fulltext[attrValue].push(new FulltextReference('aVal', attr.noteId)); } - for (const branch of branches) { + for (const branch of Object.values(branches)) { + if (branch.branchId === 'root') { + continue; + } + const childNote = notes[branch.noteId]; if (!childNote) { @@ -252,6 +268,14 @@ function highlightResults(results, allTokens) { allTokens.sort((a, b) => a.length > b.length ? -1 : 1); for (const result of results) { + const note = notes[result.noteId]; + + for (const attr of note.attributes) { + if (allTokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { + result.pathTitle += ` @${attr.name}=${attr.value}`; + } + } + result.highlightedTitle = result.pathTitle; } @@ -282,14 +306,29 @@ async function findNotes(query) { .filter(token => token !== '/'); // '/' is used as separator const tokens = allTokens.slice(); + + const matchedNoteIds = new Set(); + + for (const token of tokens) { + for (const chunk in fulltext) { + if (chunk.includes(token)) { + for (const fulltextReference of fulltext[chunk]) { + matchedNoteIds.add(fulltextReference.noteId); + } + } + } + } + + // now we have set of noteIds which match at least one token + let results = []; - for (const noteId in notes) { + for (const noteId of matchedNoteIds) { const note = notes[noteId]; // autocomplete should be able to find notes by their noteIds as well (only leafs) if (noteId === query) { - search(noteId, [], [], results); + search(note, [], [], results); continue; } @@ -298,9 +337,19 @@ async function findNotes(query) { continue; } + const foundAttrTokens = []; + + for (const attribute of note.attributes) { + for (const token of tokens) { + if (attribute.name.includes(token) || attribute.value.includes(token)) { + foundAttrTokens.push(token); + } + } + } + for (const parentNote of note.parents) { - const title = getNoteTitle(note, parentNote).toLowerCase(); - const foundTokens = []; + const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const foundTokens = foundAttrTokens.slice(); for (const token of tokens) { if (title.includes(token)) { @@ -370,12 +419,12 @@ function search(note, tokens, path, results) { return; } - if (!note.parents.length === 0 || noteId === 'root') { + if (!note.parents.length === 0 || note.noteId === 'root') { return; } for (const parentNote of note.parents) { - const title = getNoteTitle(note, parentNote).toLowerCase(); + const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); const foundTokens = []; for (const token of tokens) { @@ -387,10 +436,10 @@ function search(note, tokens, path, results) { if (foundTokens.length > 0) { const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - search(parentNote, remainingTokens, path.concat([noteId]), results); + search(parentNote, remainingTokens, path.concat([note.noteId]), results); } else { - search(parentNote, tokens, path.concat([noteId]), results); + search(parentNote, tokens, path.concat([note.noteId]), results); } } } @@ -436,7 +485,7 @@ function isInAncestor(noteId, ancestorNoteId) { const note = notes[noteId]; - for (const parentNote of notes.parents) { + for (const parentNote of note.parents) { if (isInAncestor(parentNote.noteId, ancestorNoteId)) { return true; } @@ -456,7 +505,10 @@ function getNoteTitleFromPath(notePath) { } } -function getNoteTitle(childNote, parentNote) { +function getNoteTitle(childNoteId, parentNoteId) { + const childNote = notes[childNoteId]; + const parentNote = notes[parentNoteId]; + let title; if (childNote.isProtected) { @@ -466,9 +518,9 @@ function getNoteTitle(childNote, parentNote) { title = childNote.title; } - const branch = getBranch(childNote.noteId, parentNote.noteId); + const branch = parentNote ? getBranch(childNote.noteId, parentNote.noteId) : null; - return (branch.prefix ? (branch.prefix + ' - ') : '') + title; + return ((branch && branch.prefix) ? (branch.prefix + ' - ') : '') + title; } function getNoteTitleArrayForPath(path) { @@ -511,9 +563,9 @@ function getNoteTitleForPath(path) { * - this means that archived paths is returned only if there's no non-archived path * - you can check whether returned path is archived using isArchived() */ -function getSomePath(noteId, path = []) { - if (noteId === 'root') { - path.push(noteId); +function getSomePath(note, path = []) { + if (note.noteId === 'root') { + path.push(note.noteId); path.reverse(); if (!path.includes(hoistedNoteService.getHoistedNoteId())) { @@ -523,13 +575,13 @@ function getSomePath(noteId, path = []) { return path; } - const parents = childToParent[noteId]; - if (!parents || parents.length === 0) { + const parents = note.parents; + if (parents.length === 0) { return false; } - for (const parentNoteId of parents) { - const retPath = getSomePath(parentNoteId, path.concat([noteId])); + for (const parentNote of parents) { + const retPath = getSomePath(parentNote, path.concat([note.noteId])); if (retPath) { return retPath; From 8e8fa482417f2d6c22c88b9469a33337d59ba3da Mon Sep 17 00:00:00 2001 From: zadam Date: Mon, 11 May 2020 23:23:18 +0200 Subject: [PATCH 04/47] ws update --- package-lock.json | 86 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index ab50152f5..ddb15f349 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1263,7 +1263,7 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" } } @@ -1539,7 +1539,7 @@ }, "uuid": { "version": "2.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" } } @@ -1573,7 +1573,7 @@ "dependencies": { "semver": { "version": "4.3.6", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz", "integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=" } } @@ -1593,7 +1593,7 @@ }, "bl": { "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "resolved": "http://registry.npmjs.org/bl/-/bl-1.2.2.tgz", "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", "requires": { "readable-stream": "^2.3.5", @@ -1853,12 +1853,12 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" }, "uuid": { "version": "2.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" } } @@ -1973,7 +1973,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -2148,7 +2148,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -2465,7 +2465,7 @@ }, "commander": { "version": "2.8.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", + "resolved": "http://registry.npmjs.org/commander/-/commander-2.8.1.tgz", "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", "requires": { "graceful-readlink": ">= 1.0.0" @@ -3128,7 +3128,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -4957,7 +4957,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "getpass": { @@ -5221,7 +5221,7 @@ }, "got": { "version": "5.7.1", - "resolved": "https://registry.npmjs.org/got/-/got-5.7.1.tgz", + "resolved": "http://registry.npmjs.org/got/-/got-5.7.1.tgz", "integrity": "sha1-X4FjWmHkplifGAVp6k44FoClHzU=", "requires": { "create-error-class": "^3.0.1", @@ -5869,7 +5869,7 @@ }, "into-stream": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", + "resolved": "http://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", "integrity": "sha1-lvsKk2wSur1v8XUqF9BWFqvQlMY=", "requires": { "from2": "^2.1.1", @@ -6021,7 +6021,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, "is-object": { @@ -6621,7 +6621,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "requires": { "graceful-fs": "^4.1.2", @@ -7130,7 +7130,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, "minipass": { @@ -7230,7 +7230,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -7238,7 +7238,7 @@ "dependencies": { "minimist": { "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" } } @@ -7432,7 +7432,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "got": { @@ -7468,7 +7468,7 @@ }, "p-cancelable": { "version": "0.4.1", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", + "resolved": "http://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", "integrity": "sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ==" }, "p-event": { @@ -7592,7 +7592,7 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" } } @@ -7617,7 +7617,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "pify": { @@ -7674,7 +7674,7 @@ }, "get-stream": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", "integrity": "sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=", "requires": { "object-assign": "^4.0.1", @@ -7704,7 +7704,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -7744,7 +7744,7 @@ }, "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" }, "prepend-http": { @@ -7849,7 +7849,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -8227,7 +8227,7 @@ }, "onetime": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=" }, "open": { @@ -8379,7 +8379,7 @@ }, "p-is-promise": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=" }, "p-limit": { @@ -8860,7 +8860,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -9144,7 +9144,7 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" } } @@ -9169,7 +9169,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "pify": { @@ -9207,7 +9207,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -9259,7 +9259,7 @@ }, "get-stream": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", "integrity": "sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=", "requires": { "object-assign": "^4.0.1", @@ -9289,7 +9289,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -9477,7 +9477,7 @@ }, "query-string": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", + "resolved": "http://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", "integrity": "sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==", "requires": { "decode-uri-component": "^0.2.0", @@ -9616,7 +9616,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -10484,7 +10484,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -10509,7 +10509,7 @@ }, "strip-dirs": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/strip-dirs/-/strip-dirs-1.1.1.tgz", "integrity": "sha1-lgu9EoeETzl1pFWKoQOoJV4kVqA=", "requires": { "chalk": "^1.0.0", @@ -10767,7 +10767,7 @@ }, "through": { "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { @@ -10786,7 +10786,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "requires": { "core-util-is": "~1.0.0", @@ -11929,9 +11929,9 @@ } }, "ws": { - "version": "7.2.5", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.5.tgz", - "integrity": "sha512-C34cIU4+DB2vMyAbmEKossWq2ZQDr6QEyuuCzWrM9zfw1sGc0mYiJ0UnG9zzNykt49C2Fi34hvr2vssFQRS6EA==" + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.0.tgz", + "integrity": "sha512-iFtXzngZVXPGgpTlP1rBqsUK82p9tKqsWRPg5L56egiljujJT3vGAYnHANvFxBieXrTFavhzhxW52jnaWV+w2w==" }, "x-xss-protection": { "version": "1.3.0", diff --git a/package.json b/package.json index 7e36d93f4..5514c1495 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "turndown": "6.0.0", "turndown-plugin-gfm": "1.0.2", "unescape": "1.0.1", - "ws": "7.2.5", + "ws": "7.3.0", "yauzl": "^2.10.0", "yazl": "^2.5.1" }, From ccb5f3ee18c100086b2b33f7cb333f7099b77930 Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 13 May 2020 00:01:10 +0200 Subject: [PATCH 05/47] searching now works correctly in inherited attributes --- src/services/note_cache.js | 117 +++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 43 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 958d4265c..e4208d430 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -29,6 +29,8 @@ class Note { this.isProtected = !!row.isProtected; /** @param {Note[]} */ this.parents = []; + /** @param {Note[]} */ + this.children = []; /** @param {Attribute[]} */ this.ownedAttributes = []; } @@ -60,6 +62,14 @@ class Note { return noteAttributeCache[this.noteId]; } + addSubTreeNoteIdsTo(noteIdSet) { + noteIdSet.add(this.noteId); + + for (const child of this.children) { + child.addSubTreeNoteIdsTo(noteIdSet); + } + } + /** @return {Attribute[]} */ get inheritableAttributes() { return this.attributes.filter(attr => attr.isInheritable); @@ -111,22 +121,11 @@ class Attribute { /** @param {string} */ this.value = row.value; /** @param {boolean} */ - this.isInheritable = row.isInheritable; + this.isInheritable = !!row.isInheritable; } } -class FulltextReference { - /** - * @param type - attributeName, attributeValue, title - * @param noteId - */ - constructor(type, noteId) { - this.type = type; - this.noteId = noteId; - } -} - -/** @type {Object.} */ +/** @type {Object.} */ let fulltext = {}; /** @type {Object.} */ @@ -195,17 +194,21 @@ async function getMappedRows(query, cb) { return map; } +function updateFulltext(note) { + let ft = note.title.toLowerCase(); + + for (const attr of note.attributes) { + ft += '|' + attr.name.toLowerCase(); + ft += '|' + attr.value.toLowerCase(); + } + + fulltext[note.noteId] = ft; +} + async function load() { notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, row => new Note(row)); - for (const note of Object.values(notes)) { - const title = note.title.toLowerCase(); - - fulltext[title] = fulltext[title] || []; - fulltext[title].push(new FulltextReference('note', note.noteId)); - } - branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, row => new Branch(row)); @@ -213,17 +216,9 @@ async function load() { row => new Attribute(row)); for (const attr of Object.values(attributes)) { - notes[attr.noteId].attributes.push(attr); + notes[attr.noteId].ownedAttributes.push(attr); addToAttributeMeta(attributes); - - const attrName = attr.name.toLowerCase(); - fulltext[attrName] = fulltext[attrName] || []; - fulltext[attrName].push(new FulltextReference('aName', attr.noteId)); - - const attrValue = attr.value.toLowerCase(); - fulltext[attrValue] = fulltext[attrValue] || []; - fulltext[attrValue].push(new FulltextReference('aVal', attr.noteId)); } for (const branch of Object.values(branches)) { @@ -232,13 +227,16 @@ async function load() { } const childNote = notes[branch.noteId]; + const parentNote = branch.parentNote; if (!childNote) { console.log(`Cannot find child note ${branch.noteId} of a branch ${branch.branchId}`); continue; } - childNote.parents.push(branch.parentNote); + childNote.parents.push(parentNote); + parentNote.children.push(childNote); + childParentToBranch[`${branch.noteId}-${branch.parentNoteId}`] = branch; } @@ -246,6 +244,10 @@ async function load() { await decryptProtectedNotes(); } + for (const note of Object.values(notes)) { + updateFulltext(note); + } + loaded = true; loadedPromiseResolve(); } @@ -295,7 +297,7 @@ function highlightResults(results, allTokens) { } async function findNotes(query) { - if (!noteTitles || !query.length) { + if (!query.length) { return []; } @@ -310,15 +312,31 @@ async function findNotes(query) { const matchedNoteIds = new Set(); for (const token of tokens) { - for (const chunk in fulltext) { - if (chunk.includes(token)) { - for (const fulltextReference of fulltext[chunk]) { - matchedNoteIds.add(fulltextReference.noteId); - } + for (const noteId in fulltext) { + if (!fulltext[noteId].includes(token)) { + continue; } + + matchedNoteIds.add(noteId); + const note = notes[noteId]; + const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable); + + searchingAttrs: + for (const attr of inheritableAttrs) { + const lcName = attr.name.toLowerCase(); + const lcValue = attr.value.toLowerCase(); + + for (const token of tokens) { + if (lcName.includes(token) || lcValue.includes(token)) { + note.addSubTreeNoteIdsTo(matchedNoteIds); + + break searchingAttrs; + } + } + } } } - +//console.log(matchedNoteIds); // now we have set of noteIds which match at least one token let results = []; @@ -339,9 +357,10 @@ async function findNotes(query) { const foundAttrTokens = []; - for (const attribute of note.attributes) { + for (const attribute of note.ownedAttributes) { for (const token of tokens) { - if (attribute.name.includes(token) || attribute.value.includes(token)) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { foundAttrTokens.push(token); } } @@ -423,9 +442,20 @@ function search(note, tokens, path, results) { return; } + const foundAttrTokens = []; + + for (const attribute of note.ownedAttributes) { + for (const token of tokens) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { + foundAttrTokens.push(token); + } + } + } + for (const parentNote of note.parents) { const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); - const foundTokens = []; + const foundTokens = foundAttrTokens.slice(); for (const token of tokens) { if (title.includes(token)) { @@ -592,15 +622,16 @@ function getSomePath(note, path = []) { } function getNotePath(noteId) { - const retPath = getSomePath(noteId); + const note = notes[noteId]; + const retPath = getSomePath(note); if (retPath) { const noteTitle = getNoteTitleForPath(retPath); - const parentNoteId = childToParent[noteId][0]; + const parentNote = note.parents[0]; return { noteId: noteId, - branchId: childParentToBranchId[`${noteId}-${parentNoteId}`], + branchId: getBranch(noteId, parentNote.noteId).branchId, title: noteTitle, notePath: retPath, path: retPath.join('/') From b07accfd9d5085ee3a97ba8304b33bff83ff4f99 Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 13 May 2020 10:47:22 +0200 Subject: [PATCH 06/47] note cache refactoring + handling entity changes --- src/services/note_cache.js | 260 +++++++++++++++++++++---------------- 1 file changed, 149 insertions(+), 111 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index e4208d430..88f934c56 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -27,6 +27,8 @@ class Note { this.title = row.title; /** @param {boolean} */ this.isProtected = !!row.isProtected; + /** @param {boolean} */ + this.isDecrypted = false; /** @param {Note[]} */ this.parents = []; /** @param {Note[]} */ @@ -82,6 +84,16 @@ class Note { get isArchived() { return this.hasAttribute('label', 'archived'); } + + get hasInheritableOwnedArchivedLabel() { + return !!this.ownedAttributes.find(attr => attr.type === 'label' && attr.name === 'archived' && attr.isInheritable); + } + + // will sort the parents so that non-archived are first and archived at the end + // this is done so that non-archived paths are always explored as first when searching for note path + resortParents() { + this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); + } } class Branch { @@ -94,6 +106,8 @@ class Branch { this.parentNoteId = row.parentNoteId; /** @param {string} */ this.prefix = row.prefix; + + childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; } /** @return {Note} */ @@ -172,12 +186,6 @@ let loadedPromiseResolve; /** Is resolved after the initial load */ let loadedPromise = new Promise(res => loadedPromiseResolve = res); -let noteTitles = {}; -let protectedNoteTitles = {}; -let noteIds; -const childToParent = {}; -let archived = {}; - // key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here let prefixes = {}; @@ -236,8 +244,6 @@ async function load() { childNote.parents.push(parentNote); parentNote.children.push(childNote); - - childParentToBranch[`${branch.noteId}-${branch.parentNoteId}`] = branch; } if (protectedSessionService.isProtectedSessionAvailable()) { @@ -254,12 +260,31 @@ async function load() { async function decryptProtectedNotes() { for (const note of notes) { - if (note.isProtected) { + if (note.isProtected && !note.isDecrypted) { note.title = protectedSessionService.decryptString(note.title); + + note.isDecrypted = true; } } } +function formatAttribute(attr) { + if (attr.type === 'relation') { + return '@' + utils.escapeHtml(attr.name) + "=…"; + } + else if (attr.type === 'label') { + let label = '#' + utils.escapeHtml(attr.name); + + if (attr.value) { + const val = /[^\w_-]/.test(attr.value) ? '"' + attr.value + '"' : attr.value; + + label += '=' + utils.escapeHtml(val); + } + + return label; + } +} + function highlightResults(results, allTokens) { // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks // which would make the resulting HTML string invalid. @@ -274,7 +299,7 @@ function highlightResults(results, allTokens) { for (const attr of note.attributes) { if (allTokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { - result.pathTitle += ` @${attr.name}=${attr.value}`; + result.pathTitle += ` ${formatAttribute(attr)}`; } } @@ -296,6 +321,44 @@ function highlightResults(results, allTokens) { } } +/** + * Returns noteIds which have at least one matching tokens + * + * @param tokens + * @return {Set} + */ +function getCandidateNotes(tokens) { + const candidateNoteIds = new Set(); + + for (const token of tokens) { + for (const noteId in fulltext) { + if (!fulltext[noteId].includes(token)) { + continue; + } + + candidateNoteIds.add(noteId); + const note = notes[noteId]; + const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable); + + searchingAttrs: + // for matching inheritable attributes, include the whole note subtree to the candidates + for (const attr of inheritableAttrs) { + const lcName = attr.name.toLowerCase(); + const lcValue = attr.value.toLowerCase(); + + for (const token of tokens) { + if (lcName.includes(token) || lcValue.includes(token)) { + note.addSubTreeNoteIdsTo(candidateNoteIds); + + break searchingAttrs; + } + } + } + } + } + return candidateNoteIds; +} + async function findNotes(query) { if (!query.length) { return []; @@ -307,41 +370,14 @@ async function findNotes(query) { .split(/[ -]/) .filter(token => token !== '/'); // '/' is used as separator - const tokens = allTokens.slice(); + const candidateNoteIds = getCandidateNotes(allTokens); - const matchedNoteIds = new Set(); - - for (const token of tokens) { - for (const noteId in fulltext) { - if (!fulltext[noteId].includes(token)) { - continue; - } - - matchedNoteIds.add(noteId); - const note = notes[noteId]; - const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable); - - searchingAttrs: - for (const attr of inheritableAttrs) { - const lcName = attr.name.toLowerCase(); - const lcValue = attr.value.toLowerCase(); - - for (const token of tokens) { - if (lcName.includes(token) || lcValue.includes(token)) { - note.addSubTreeNoteIdsTo(matchedNoteIds); - - break searchingAttrs; - } - } - } - } - } -//console.log(matchedNoteIds); // now we have set of noteIds which match at least one token let results = []; + const tokens = allTokens.slice(); - for (const noteId of matchedNoteIds) { + for (const noteId of candidateNoteIds) { const note = notes[noteId]; // autocomplete should be able to find notes by their noteIds as well (only leafs) @@ -476,14 +512,17 @@ function search(note, tokens, path, results) { function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; + const note = notes[noteId]; - if (archived[noteId] !== undefined) { + if (note.isArchived) { return true; } for (let i = 0; i < notePath.length - 1; i++) { + const note = notes[notePath[i]]; + // this is going through parents so archived must be inheritable - if (archived[notePath[i]] === 1) { + if (note.hasInheritableOwnedArchivedLabel) { return true; } } @@ -550,7 +589,7 @@ function getNoteTitle(childNoteId, parentNoteId) { const branch = parentNote ? getBranch(childNote.noteId, parentNote.noteId) : null; - return ((branch && branch.prefix) ? (branch.prefix + ' - ') : '') + title; + return ((branch && branch.prefix) ? `${branch.prefix} - ` : '') + title; } function getNoteTitleArrayForPath(path) { @@ -639,11 +678,11 @@ function getNotePath(noteId) { } } -function evaluateSimilarity(text1, text2, noteId, results) { - let coeff = stringSimilarity.compareTwoStrings(text1, text2); +function evaluateSimilarity(text, note, results) { + let coeff = stringSimilarity.compareTwoStrings(text, note.title); if (coeff > 0.4) { - const notePath = getSomePath(noteId); + const notePath = getSomePath(note); // this takes care of note hoisting if (!notePath) { @@ -654,7 +693,7 @@ function evaluateSimilarity(text1, text2, noteId, results) { coeff -= 0.2; // archived penalization } - results.push({coeff, notePath, noteId}); + results.push({coeff, notePath, noteId: note.noteId}); } } @@ -668,11 +707,16 @@ function setImmediatePromise() { }); } -async function evaluateSimilarityDict(title, dict, results) { +async function findSimilarNotes(title) { + const results = []; let i = 0; - for (const noteId in dict) { - evaluateSimilarity(title, dict[noteId], noteId, results); + for (const note of Object.values(notes)) { + if (note.isProtected && !note.isDecrypted) { + continue; + } + + evaluateSimilarity(title, note, results); i++; @@ -680,16 +724,6 @@ async function evaluateSimilarityDict(title, dict, results) { await setImmediatePromise(); } } -} - -async function findSimilarNotes(title) { - const results = []; - - await evaluateSimilarityDict(title, noteTitles, results); - - if (protectedSessionService.isProtectedSessionAvailable()) { - await evaluateSimilarityDict(title, protectedNoteTitles, results); - } results.sort((a, b) => a.coeff > b.coeff ? -1 : 1); @@ -704,80 +738,84 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED } if (entityName === 'notes') { - const note = entity; + const {noteId} = entity; - if (note.isDeleted) { - delete noteTitles[note.noteId]; - delete childToParent[note.noteId]; + if (entity.isDeleted) { + delete notes[noteId]; + } + else if (noteId in notes) { + // we can assume we have protected session since we managed to update + notes[noteId].title = entity.title; + notes[noteId].isDecrypted = true; } else { - if (note.isProtected) { - // we can assume we have protected session since we managed to update - // removing from the maps is important when switching between protected & unprotected - protectedNoteTitles[note.noteId] = note.title; - delete noteTitles[note.noteId]; - } - else { - noteTitles[note.noteId] = note.title; - delete protectedNoteTitles[note.noteId]; - } + notes[noteId] = new Note(entity); } } else if (entityName === 'branches') { - const branch = entity; + const {branchId, noteId, parentNoteId} = entity; - if (branch.isDeleted) { - if (branch.noteId in childToParent) { - childToParent[branch.noteId] = childToParent[branch.noteId].filter(noteId => noteId !== branch.parentNoteId); + if (entity.isDeleted) { + const childNote = notes[noteId]; + + if (childNote) { + childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); } - delete prefixes[branch.noteId + '-' + branch.parentNoteId]; - delete childParentToBranchId[branch.noteId + '-' + branch.parentNoteId]; + const parentNote = notes[parentNoteId]; + + if (parentNote) { + childNote.children = childNote.children.filter(child => child.noteId !== noteId); + } + + delete childParentToBranch[`${noteId}-${parentNoteId}`]; + delete branches[branchId]; + } + else if (branchId in branches) { + // only relevant thing which can change in a branch is prefix + branches[branchId].prefix = entity.prefix; } else { - if (branch.prefix) { - prefixes[branch.noteId + '-' + branch.parentNoteId] = branch.prefix; + branches[branchId] = new Branch(entity); + + const note = notes[entity.noteId]; + + if (note) { + note.resortParents(); } - - childToParent[branch.noteId] = childToParent[branch.noteId] || []; - - if (!childToParent[branch.noteId].includes(branch.parentNoteId)) { - childToParent[branch.noteId].push(branch.parentNoteId); - } - - resortChildToParent(branch.noteId); - - childParentToBranchId[branch.noteId + '-' + branch.parentNoteId] = branch.branchId; } } else if (entityName === 'attributes') { - const attribute = entity; + const {attributeId, noteId} = entity; - if (attribute.type === 'label' && attribute.name === 'archived') { - // we're not using label object directly, since there might be other non-deleted archived label - const archivedLabel = await repository.getEntity(`SELECT * FROM attributes WHERE isDeleted = 0 AND type = 'label' - AND name = 'archived' AND noteId = ?`, [attribute.noteId]); + if (entity.isDeleted) { + const note = notes[noteId]; - if (archivedLabel) { - archived[attribute.noteId] = archivedLabel.isInheritable ? 1 : 0; + if (note) { + note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); } - else { - delete archived[attribute.noteId]; + + delete attributes[entity.attributeId]; + } + else if (attributeId in attributes) { + const attr = attributes[attributeId]; + + // attr name cannot change + attr.value = entity.value; + attr.isInheritable = entity.isInheritable; + } + else { + attributes[attributeId] = new Attribute(entity); + + const note = notes[noteId]; + + if (note) { + note.ownedAttributes.push(attributes[attributeId]); } } } }); -// will sort the childs so that non-archived are first and archived at the end -// this is done so that non-archived paths are always explored as first when searching for note path -function resortChildToParent(noteId) { - if (!(noteId in childToParent)) { - return; - } - - childToParent[noteId].sort((a, b) => archived[a] === 1 ? 1 : -1); -} - /** * @param noteId * @returns {boolean} - true if note exists (is not deleted) and is available in current note hoisting From 7992f32d34f432a1df84f2e7082fcbf43338d24f Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 13 May 2020 14:42:16 +0200 Subject: [PATCH 07/47] note cache refactoring --- package-lock.json | 34 ++-- package.json | 2 +- src/entities/attribute.js | 14 +- src/entities/branch.js | 8 +- src/routes/api/attributes.js | 5 +- src/routes/api/similar_notes.js | 4 +- src/services/note_cache.js | 307 ++++++++++++++++++-------------- 7 files changed, 210 insertions(+), 164 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f65b4c89..554169239 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "trilium", - "version": "0.42.1", + "version": "0.42.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2218,9 +2218,9 @@ } }, "cli-spinners": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz", - "integrity": "sha512-tgU3fKwzYjiLEQgPMD9Jt+JjHVL9kW93FiIMX/l7rivvOD4/LL0Mf7gda3+4U2KJBloybwgj5KEoQgGRioMiKQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.3.0.tgz", + "integrity": "sha512-Xs2Hf2nzrvJMFKimOR7YR0QwZ8fc0u98kdtwN1eNAZzNQgH3vK2pXzff6GJtKh7S5hoJ87ECiAiZFS2fb5Ii2w==", "dev": true }, "cli-table3": { @@ -3802,9 +3802,9 @@ } }, "electron-rebuild": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.10.1.tgz", - "integrity": "sha512-KSqp0Xiu7CCvKL2aEdPp/vNe2Rr11vaO8eM/wq9gQJTY02UjtAJ3l7WLV7Mf8oR+UJReJO8SWOWs/FozqK8ggA==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.11.0.tgz", + "integrity": "sha512-cn6AqZBQBVtaEyj5jZW1/LOezZZ22PA1HvhEP7asvYPJ8PDF4i4UFt9be4i9T7xJKiSiomXvY5Fd+dSq3FXZxA==", "dev": true, "requires": { "colors": "^1.3.3", @@ -3877,9 +3877,9 @@ } }, "yargs": { - "version": "14.2.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.2.tgz", - "integrity": "sha512-/4ld+4VV5RnrynMhPZJ/ZpOCGSCeghMykZ3BhdFBDa9Wy/RH6uEGNWDJog+aUlq+9OM1CFTgtYRW5Is1Po9NOA==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz", + "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==", "dev": true, "requires": { "cliui": "^5.0.0", @@ -3892,13 +3892,13 @@ "string-width": "^3.0.0", "which-module": "^2.0.0", "y18n": "^4.0.0", - "yargs-parser": "^15.0.0" + "yargs-parser": "^15.0.1" } }, "yargs-parser": { - "version": "15.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.0.tgz", - "integrity": "sha512-xLTUnCMc4JhxrPEPUYD5IBR1mWCK/aT6+RJ/K29JY2y1vD+FhtgKK0AXRWvI262q3QSffAQuTouFIKUuHX89wQ==", + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz", + "integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==", "dev": true, "requires": { "camelcase": "^5.0.0", @@ -9929,9 +9929,9 @@ } }, "rxjs": { - "version": "6.5.4", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", - "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", + "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", "dev": true, "requires": { "tslib": "^1.9.0" diff --git a/package.json b/package.json index eb32e74fe..65fd8a95c 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "electron": "9.0.0-beta.24", "electron-builder": "22.6.0", "electron-packager": "14.2.1", - "electron-rebuild": "1.10.1", + "electron-rebuild": "1.11.0", "jsdoc": "3.6.4", "lorem-ipsum": "2.0.3", "webpack": "5.0.0-beta.16", diff --git a/src/entities/attribute.js b/src/entities/attribute.js index 0800d6252..14b7c8a87 100644 --- a/src/entities/attribute.js +++ b/src/entities/attribute.js @@ -8,13 +8,13 @@ const sql = require('../services/sql'); /** * Attribute is key value pair owned by a note. * - * @property {string} attributeId - * @property {string} noteId - * @property {string} type - * @property {string} name + * @property {string} attributeId - immutable + * @property {string} noteId - immutable + * @property {string} type - immutable + * @property {string} name - immutable * @property {string} value * @property {int} position - * @property {boolean} isInheritable + * @property {boolean} isInheritable - immutable * @property {boolean} isDeleted * @property {string|null} deleteId - ID identifying delete transaction * @property {string} utcDateCreated @@ -108,14 +108,14 @@ class Attribute extends Entity { delete pojo.__note; } - createClone(type, name, value) { + createClone(type, name, value, isInheritable) { return new Attribute({ noteId: this.noteId, type: type, name: name, value: value, position: this.position, - isInheritable: this.isInheritable, + isInheritable: isInheritable, isDeleted: false, utcDateCreated: this.utcDateCreated, utcDateModified: this.utcDateModified diff --git a/src/entities/branch.js b/src/entities/branch.js index 7950526dc..060a275cd 100644 --- a/src/entities/branch.js +++ b/src/entities/branch.js @@ -9,9 +9,9 @@ const sql = require('../services/sql'); * Branch represents note's placement in the tree - it's essentially pair of noteId and parentNoteId. * Each note can have multiple (at least one) branches, meaning it can be placed into multiple places in the tree. * - * @property {string} branchId - primary key - * @property {string} noteId - * @property {string} parentNoteId + * @property {string} branchId - primary key, immutable + * @property {string} noteId - immutable + * @property {string} parentNoteId - immutable * @property {int} notePosition * @property {string} prefix * @property {boolean} isExpanded @@ -77,4 +77,4 @@ class Branch extends Entity { } } -module.exports = Branch; \ No newline at end of file +module.exports = Branch; diff --git a/src/routes/api/attributes.js b/src/routes/api/attributes.js index 131e7a77b..abab2ab9e 100644 --- a/src/routes/api/attributes.js +++ b/src/routes/api/attributes.js @@ -98,10 +98,11 @@ async function updateNoteAttributes(req) { if (attribute.type !== attributeEntity.type || attribute.name !== attributeEntity.name - || (attribute.type === 'relation' && attribute.value !== attributeEntity.value)) { + || (attribute.type === 'relation' && attribute.value !== attributeEntity.value) + || attribute.isInheritable !== attributeEntity.isInheritable) { if (attribute.type !== 'relation' || !!attribute.value.trim()) { - const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value); + const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value, attribute.isInheritable); await newAttribute.save(); } diff --git a/src/routes/api/similar_notes.js b/src/routes/api/similar_notes.js index 3403833f8..4d52e94fe 100644 --- a/src/routes/api/similar_notes.js +++ b/src/routes/api/similar_notes.js @@ -12,7 +12,7 @@ async function getSimilarNotes(req) { return [404, `Note ${noteId} not found.`]; } - const results = await noteCacheService.findSimilarNotes(note.title); + const results = await noteCacheService.findSimilarNotes(noteId); return results .filter(note => note.noteId !== noteId); @@ -20,4 +20,4 @@ async function getSimilarNotes(req) { module.exports = { getSimilarNotes -}; \ No newline at end of file +}; diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 88f934c56..e1b4efee2 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -35,46 +35,75 @@ class Note { this.children = []; /** @param {Attribute[]} */ this.ownedAttributes = []; + + /** @param {Attribute[]|null} */ + this.attributeCache = null; + /** @param {Attribute[]|null} */ + this.templateAttributeCache = null; + /** @param {Attribute[]|null} */ + this.inheritableAttributeCache = null; + + /** @param {string|null} */ + this.fulltextCache = null; } /** @return {Attribute[]} */ get attributes() { - if (!(this.noteId in noteAttributeCache)) { - const attrArrs = [ - this.ownedAttributes - ]; - - for (const templateAttr of this.ownedAttributes.filter(oa => oa.type === 'relation' && oa.name === 'template')) { - const templateNote = notes[templateAttr.value]; - - if (templateNote) { - attrArrs.push(templateNote.attributes); - } - } + if (!this.attributeCache) { + const parentAttributes = this.ownedAttributes.slice(); if (this.noteId !== 'root') { for (const parentNote of this.parents) { - attrArrs.push(parentNote.inheritableAttributes); + parentAttributes.push(...parentNote.inheritableAttributes); } } - noteAttributeCache[this.noteId] = attrArrs.flat(); + const templateAttributes = []; + + for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates + if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') { + const templateNote = notes[ownedAttr.value]; + + if (templateNote) { + templateAttributes.push(...templateNote.attributes); + } + } + } + + this.attributeCache = parentAttributes.concat(templateAttributes); + this.inheritableAttributeCache = []; + this.templateAttributeCache = []; + + for (const attr of this.attributeCache) { + if (attr.isInheritable) { + this.inheritableAttributeCache.push(attr); + } + + if (attr.type === 'relation' && attr.name === 'template') { + this.templateAttributeCache.push(attr); + } + } } - return noteAttributeCache[this.noteId]; - } - - addSubTreeNoteIdsTo(noteIdSet) { - noteIdSet.add(this.noteId); - - for (const child of this.children) { - child.addSubTreeNoteIdsTo(noteIdSet); - } + return this.attributeCache; } /** @return {Attribute[]} */ get inheritableAttributes() { - return this.attributes.filter(attr => attr.isInheritable); + if (!this.inheritableAttributeCache) { + this.attributes; // will refresh also this.inheritableAttributeCache + } + + return this.inheritableAttributeCache; + } + + /** @return {Attribute[]} */ + get templateAttributes() { + if (!this.templateAttributeCache) { + this.attributes; // will refresh also this.templateAttributeCache + } + + return this.templateAttributeCache; } hasAttribute(type, name) { @@ -94,6 +123,63 @@ class Note { resortParents() { this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); } + + get fulltext() { + if (!this.fulltextCache) { + this.fulltextCache = this.title.toLowerCase(); + + for (const attr of this.attributes) { + // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words + this.fulltextCache += ' ' + attr.name.toLowerCase(); + + if (attr.value) { + this.fulltextCache += ' ' + attr.value.toLowerCase(); + } + } + } + + return this.fulltextCache; + } + + invalidateThisCache() { + this.fulltextCache = null; + + this.attributeCache = null; + this.templateAttributeCache = null; + this.inheritableAttributeCache = null; + } + + invalidateSubtreeCaches() { + this.invalidateThisCache(); + + for (const childNote of this.children) { + childNote.invalidateSubtreeCaches(); + } + + for (const templateAttr of this.templateAttributes) { + const targetNote = templateAttr.targetNote; + + if (targetNote) { + targetNote.invalidateSubtreeCaches(); + } + } + } + + invalidateSubtreeFulltext() { + this.fulltextCache = null; + + for (const childNote of this.children) { + childNote.invalidateSubtreeFulltext(); + } + + for (const templateAttr of this.templateAttributes) { + const targetNote = templateAttr.targetNote; + + if (targetNote) { + targetNote.invalidateSubtreeFulltext(); + } + } + } } class Branch { @@ -137,47 +223,16 @@ class Attribute { /** @param {boolean} */ this.isInheritable = !!row.isInheritable; } -} -/** @type {Object.} */ -let fulltext = {}; - -/** @type {Object.} */ -let attributeMetas = {}; - -class AttributeMeta { - constructor(attribute) { - this.type = attribute.type; - this.name = attribute.name; - this.isInheritable = attribute.isInheritable; - this.attributeIds = new Set(attribute.attributeId); + get isAffectingSubtree() { + return this.isInheritable + || (this.type === 'relation' && this.name === 'template'); } - addAttribute(attribute) { - this.attributeIds.add(attribute.attributeId); - this.isInheritable = this.isInheritable || attribute.isInheritable; - } - - updateAttribute(attribute) { - if (attribute.isDeleted) { - this.attributeIds.delete(attribute.attributeId); + get targetNote() { + if (this.type === 'relation') { + return notes[this.value]; } - else { - this.attributeIds.add(attribute.attributeId); - } - - this.isInheritable = !!this.attributeIds.find(attributeId => attributes[attributeId].isInheritable); - } -} - -function addToAttributeMeta(attribute) { - const key = `${attribute.type}-${attribute.name}`; - - if (!(key in attributeMetas)) { - attributeMetas[key] = new AttributeMeta(attribute); - } - else { - attributeMetas[key].addAttribute(attribute); } } @@ -186,9 +241,6 @@ let loadedPromiseResolve; /** Is resolved after the initial load */ let loadedPromise = new Promise(res => loadedPromiseResolve = res); -// key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here -let prefixes = {}; - async function getMappedRows(query, cb) { const map = {}; const results = await sql.getRows(query, []); @@ -202,17 +254,6 @@ async function getMappedRows(query, cb) { return map; } -function updateFulltext(note) { - let ft = note.title.toLowerCase(); - - for (const attr of note.attributes) { - ft += '|' + attr.name.toLowerCase(); - ft += '|' + attr.value.toLowerCase(); - } - - fulltext[note.noteId] = ft; -} - async function load() { notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, row => new Note(row)); @@ -225,8 +266,6 @@ async function load() { for (const attr of Object.values(attributes)) { notes[attr.noteId].ownedAttributes.push(attr); - - addToAttributeMeta(attributes); } for (const branch of Object.values(branches)) { @@ -250,10 +289,6 @@ async function load() { await decryptProtectedNotes(); } - for (const note of Object.values(notes)) { - updateFulltext(note); - } - loaded = true; loadedPromiseResolve(); } @@ -325,38 +360,21 @@ function highlightResults(results, allTokens) { * Returns noteIds which have at least one matching tokens * * @param tokens - * @return {Set} + * @return {String[]} */ function getCandidateNotes(tokens) { - const candidateNoteIds = new Set(); + const candidateNotes = []; - for (const token of tokens) { - for (const noteId in fulltext) { - if (!fulltext[noteId].includes(token)) { - continue; + for (const note of Object.values(notes)) { + for (const token of tokens) { + if (note.fulltext.includes(token)) { + candidateNotes.push(note); + break; } - - candidateNoteIds.add(noteId); - const note = notes[noteId]; - const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable); - - searchingAttrs: - // for matching inheritable attributes, include the whole note subtree to the candidates - for (const attr of inheritableAttrs) { - const lcName = attr.name.toLowerCase(); - const lcValue = attr.value.toLowerCase(); - - for (const token of tokens) { - if (lcName.includes(token) || lcValue.includes(token)) { - note.addSubTreeNoteIdsTo(candidateNoteIds); - - break searchingAttrs; - } - } - } } } - return candidateNoteIds; + + return candidateNotes; } async function findNotes(query) { @@ -370,18 +388,16 @@ async function findNotes(query) { .split(/[ -]/) .filter(token => token !== '/'); // '/' is used as separator - const candidateNoteIds = getCandidateNotes(allTokens); + const candidateNotes = getCandidateNotes(allTokens); // now we have set of noteIds which match at least one token let results = []; const tokens = allTokens.slice(); - for (const noteId of candidateNoteIds) { - const note = notes[noteId]; - + for (const note of candidateNotes) { // autocomplete should be able to find notes by their noteIds as well (only leafs) - if (noteId === query) { + if (note.noteId === query) { search(note, [], [], results); continue; } @@ -415,7 +431,7 @@ async function findNotes(query) { if (foundTokens.length > 0) { const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - search(parentNote, remainingTokens, [noteId], results); + search(parentNote, remainingTokens, [note.noteId], results); } } } @@ -678,11 +694,11 @@ function getNotePath(noteId) { } } -function evaluateSimilarity(text, note, results) { - let coeff = stringSimilarity.compareTwoStrings(text, note.title); +function evaluateSimilarity(sourceNote, candidateNote, results) { + let coeff = stringSimilarity.compareTwoStrings(sourceNote.fulltext, candidateNote.fulltext); if (coeff > 0.4) { - const notePath = getSomePath(note); + const notePath = getSomePath(candidateNote); // this takes care of note hoisting if (!notePath) { @@ -693,7 +709,7 @@ function evaluateSimilarity(text, note, results) { coeff -= 0.2; // archived penalization } - results.push({coeff, notePath, noteId: note.noteId}); + results.push({coeff, notePath, noteId: candidateNote.noteId}); } } @@ -707,16 +723,22 @@ function setImmediatePromise() { }); } -async function findSimilarNotes(title) { +async function findSimilarNotes(noteId) { const results = []; let i = 0; + const origNote = notes[noteId]; + + if (!origNote) { + return []; + } + for (const note of Object.values(notes)) { if (note.isProtected && !note.isDecrypted) { continue; } - evaluateSimilarity(title, note, results); + evaluateSimilarity(origNote, note, results); i++; @@ -744,9 +766,12 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED delete notes[noteId]; } else if (noteId in notes) { + const note = notes[noteId]; + // we can assume we have protected session since we managed to update - notes[noteId].title = entity.title; - notes[noteId].isDecrypted = true; + note.title = entity.title; + note.isDecrypted = true; + note.fulltextCache = null; } else { notes[noteId] = new Note(entity); @@ -760,6 +785,10 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED if (childNote) { childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); + + if (childNote.parents.length > 0) { + childNote.invalidateSubtreeCaches(); + } } const parentNote = notes[parentNoteId]; @@ -787,30 +816,46 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED } else if (entityName === 'attributes') { const {attributeId, noteId} = entity; + const note = notes[noteId]; + const attr = attributes[attributeId]; if (entity.isDeleted) { - const note = notes[noteId]; - - if (note) { + if (note && attr) { note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); + + if (attr.isAffectingSubtree) { + note.invalidateSubtreeCaches(); + } } - delete attributes[entity.attributeId]; + delete attributes[attributeId]; } else if (attributeId in attributes) { const attr = attributes[attributeId]; - // attr name cannot change + // attr name and isInheritable are immutable attr.value = entity.value; - attr.isInheritable = entity.isInheritable; + + if (attr.isAffectingSubtree) { + note.invalidateSubtreeFulltext(); + } + else { + note.fulltextCache = null; + } } else { - attributes[attributeId] = new Attribute(entity); - - const note = notes[noteId]; + const attr = new Attribute(entity); + attributes[attributeId] = attr; if (note) { - note.ownedAttributes.push(attributes[attributeId]); + note.ownedAttributes.push(attr); + + if (attr.isAffectingSubtree) { + note.invalidateSubtreeCaches(); + } + else { + this.invalidateThisCache(); + } } } } From a287bb59eabaf06866fccdb261f6da8b5f592b8c Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 13 May 2020 23:06:13 +0200 Subject: [PATCH 08/47] note cache fixes for created notes --- src/public/app/services/app_context.js | 3 + .../app/services/protected_session_holder.js | 11 ++- src/routes/api/clipper.js | 12 ++- src/services/note_cache.js | 91 +++++++++++-------- 4 files changed, 71 insertions(+), 46 deletions(-) diff --git a/src/public/app/services/app_context.js b/src/public/app/services/app_context.js index 238db11d3..862861f6a 100644 --- a/src/public/app/services/app_context.js +++ b/src/public/app/services/app_context.js @@ -11,6 +11,7 @@ import Component from "../widgets/component.js"; import keyboardActionsService from "./keyboard_actions.js"; import MobileScreenSwitcherExecutor from "../widgets/mobile_widgets/mobile_screen_switcher.js"; import MainTreeExecutors from "./main_tree_executors.js"; +import protectedSessionHolder from "./protected_session_holder.js"; class AppContext extends Component { constructor(isMainWindow) { @@ -110,6 +111,8 @@ const appContext = new AppContext(window.glob.isMainWindow); // we should save all outstanding changes before the page/app is closed $(window).on('beforeunload', () => { + protectedSessionHolder.resetSessionCookie(); + appContext.triggerEvent('beforeUnload'); }); diff --git a/src/public/app/services/protected_session_holder.js b/src/public/app/services/protected_session_holder.js index 7f0dc2a7d..fc041b51e 100644 --- a/src/public/app/services/protected_session_holder.js +++ b/src/public/app/services/protected_session_holder.js @@ -12,15 +12,19 @@ setInterval(() => { resetProtectedSession(); } -}, 5000); +}, 10000); function setProtectedSessionId(id) { // using session cookie so that it disappears after browser/tab is closed utils.setSessionCookie(PROTECTED_SESSION_ID_KEY, id); } -function resetProtectedSession() { +function resetSessionCookie() { utils.setSessionCookie(PROTECTED_SESSION_ID_KEY, null); +} + +function resetProtectedSession() { + resetSessionCookie(); // most secure solution - guarantees nothing remained in memory // since this expires because user doesn't use the app, it shouldn't be disruptive @@ -47,8 +51,9 @@ function touchProtectedSessionIfNecessary(note) { export default { setProtectedSessionId, + resetSessionCookie, resetProtectedSession, isProtectedSessionAvailable, touchProtectedSession, touchProtectedSessionIfNecessary -}; \ No newline at end of file +}; diff --git a/src/routes/api/clipper.js b/src/routes/api/clipper.js index 6ad2efea4..38473fe06 100644 --- a/src/routes/api/clipper.js +++ b/src/routes/api/clipper.js @@ -109,11 +109,17 @@ async function addImagesToNote(images, note, content) { const {note: imageNote, url} = await imageService.saveImage(note.noteId, buffer, filename, true); + await new Attribute({ + noteId: imageNote.noteId, + type: 'label', + name: 'hideInAutocomplete' + }).save(); + await new Attribute({ noteId: note.noteId, type: 'relation', - value: imageNote.noteId, - name: 'imageLink' + name: 'imageLink', + value: imageNote.noteId }).save(); console.log(`Replacing ${imageId} with ${url}`); @@ -155,4 +161,4 @@ module.exports = { addClipping, openNote, handshake -}; \ No newline at end of file +}; diff --git a/src/services/note_cache.js b/src/services/note_cache.js index e1b4efee2..d1f6134af 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -1,7 +1,6 @@ const sql = require('./sql'); const sqlInit = require('./sql_init'); const eventService = require('./events'); -const repository = require('./repository'); const protectedSessionService = require('./protected_session'); const utils = require('./utils'); const hoistedNoteService = require('./hoisted_note'); @@ -14,9 +13,6 @@ let branches /** @type {Object.} */ let attributes; -/** @type {Object.} */ -let noteAttributeCache = {}; - let childParentToBranch = {}; class Note { @@ -28,7 +24,7 @@ class Note { /** @param {boolean} */ this.isProtected = !!row.isProtected; /** @param {boolean} */ - this.isDecrypted = false; + this.isDecrypted = !row.isProtected || !!row.isContentAvailable; /** @param {Note[]} */ this.parents = []; /** @param {Note[]} */ @@ -45,6 +41,10 @@ class Note { /** @param {string|null} */ this.fulltextCache = null; + + if (protectedSessionService.isProtectedSessionAvailable()) { + decryptProtectedNote(this); + } } /** @return {Attribute[]} */ @@ -114,6 +114,12 @@ class Note { return this.hasAttribute('label', 'archived'); } + get isHideInAutocompleteOrArchived() { + return this.attributes.find(attr => + attr.type === 'label' + && ["archived", "hideInAutocomplete"].includes(attr.name)); + } + get hasInheritableOwnedArchivedLabel() { return !!this.ownedAttributes.find(attr => attr.type === 'label' && attr.name === 'archived' && attr.isInheritable); } @@ -126,6 +132,11 @@ class Note { get fulltext() { if (!this.fulltextCache) { + if (this.isHideInAutocompleteOrArchived) { + this.fulltextCache = " "; // can't be empty + return this.fulltextCache; + } + this.fulltextCache = this.title.toLowerCase(); for (const attr of this.attributes) { @@ -193,6 +204,21 @@ class Branch { /** @param {string} */ this.prefix = row.prefix; + if (this.branchId === 'root') { + return; + } + + const childNote = notes[this.noteId]; + const parentNote = this.parentNote; + + if (!childNote) { + console.log(`Cannot find child note ${this.noteId} of a branch ${this.branchId}`); + return; + } + + childNote.parents.push(parentNote); + parentNote.children.push(childNote); + childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; } @@ -222,6 +248,8 @@ class Attribute { this.value = row.value; /** @param {boolean} */ this.isInheritable = !!row.isInheritable; + + notes[this.noteId].ownedAttributes.push(this); } get isAffectingSubtree() { @@ -264,42 +292,21 @@ async function load() { attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, row => new Attribute(row)); - for (const attr of Object.values(attributes)) { - notes[attr.noteId].ownedAttributes.push(attr); - } - - for (const branch of Object.values(branches)) { - if (branch.branchId === 'root') { - continue; - } - - const childNote = notes[branch.noteId]; - const parentNote = branch.parentNote; - - if (!childNote) { - console.log(`Cannot find child note ${branch.noteId} of a branch ${branch.branchId}`); - continue; - } - - childNote.parents.push(parentNote); - parentNote.children.push(childNote); - } - - if (protectedSessionService.isProtectedSessionAvailable()) { - await decryptProtectedNotes(); - } - loaded = true; loadedPromiseResolve(); } -async function decryptProtectedNotes() { - for (const note of notes) { - if (note.isProtected && !note.isDecrypted) { - note.title = protectedSessionService.decryptString(note.title); +function decryptProtectedNote(note) { + if (note.isProtected && !note.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { + note.title = protectedSessionService.decryptString(note.title); - note.isDecrypted = true; - } + note.isDecrypted = true; + } +} + +async function decryptProtectedNotes() { + for (const note of Object.values(notes)) { + decryptProtectedNote(note); } } @@ -770,11 +777,17 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED // we can assume we have protected session since we managed to update note.title = entity.title; - note.isDecrypted = true; + note.isProtected = entity.isProtected; + note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; note.fulltextCache = null; + + decryptProtectedNote(note); } else { - notes[noteId] = new Note(entity); + const note = new Note(entity); + notes[noteId] = note; + + decryptProtectedNote(note); } } else if (entityName === 'branches') { @@ -848,8 +861,6 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED attributes[attributeId] = attr; if (note) { - note.ownedAttributes.push(attr); - if (attr.isAffectingSubtree) { note.invalidateSubtreeCaches(); } From 1ec446137d4e01317cd4949d7757d0ef78bbd3f0 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 14 May 2020 21:30:36 +0200 Subject: [PATCH 09/47] fulltext also searches for branch prefixes --- src/services/note_cache.js | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index d1f6134af..8a9e9d7be 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -25,6 +25,8 @@ class Note { this.isProtected = !!row.isProtected; /** @param {boolean} */ this.isDecrypted = !row.isProtected || !!row.isContentAvailable; + /** @param {Branch[]} */ + this.parentBranches = []; /** @param {Note[]} */ this.parents = []; /** @param {Note[]} */ @@ -139,6 +141,12 @@ class Note { this.fulltextCache = this.title.toLowerCase(); + for (const branch of this.parentBranches) { + if (branch.prefix) { + this.fulltextCache += ' ' + branch.prefix; + } + } + for (const attr of this.attributes) { // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words this.fulltextCache += ' ' + attr.name.toLowerCase(); @@ -217,6 +225,8 @@ class Branch { } childNote.parents.push(parentNote); + childNote.parentBranches.push(this); + parentNote.children.push(childNote); childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; @@ -792,12 +802,12 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED } else if (entityName === 'branches') { const {branchId, noteId, parentNoteId} = entity; + const childNote = notes[noteId]; if (entity.isDeleted) { - const childNote = notes[noteId]; - if (childNote) { childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); + childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); if (childNote.parents.length > 0) { childNote.invalidateSubtreeCaches(); @@ -807,7 +817,7 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED const parentNote = notes[parentNoteId]; if (parentNote) { - childNote.children = childNote.children.filter(child => child.noteId !== noteId); + parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); } delete childParentToBranch[`${noteId}-${parentNoteId}`]; @@ -816,14 +826,16 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED else if (branchId in branches) { // only relevant thing which can change in a branch is prefix branches[branchId].prefix = entity.prefix; + + if (childNote) { + childNote.fulltextCache = null; + } } else { branches[branchId] = new Branch(entity); - const note = notes[entity.noteId]; - - if (note) { - note.resortParents(); + if (childNote) { + childNote.resortParents(); } } } From a3e2369599ddad91540d3eed6d0fe429e312a2a6 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 14 May 2020 23:11:59 +0200 Subject: [PATCH 10/47] add also content fulltext --- src/services/note_cache.js | 122 +++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 40 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 8a9e9d7be..7d3dee44a 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -42,7 +42,7 @@ class Note { this.inheritableAttributeCache = null; /** @param {string|null} */ - this.fulltextCache = null; + this.flatTextCache = null; if (protectedSessionService.isProtectedSessionAvailable()) { decryptProtectedNote(this); @@ -132,36 +132,39 @@ class Note { this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); } - get fulltext() { - if (!this.fulltextCache) { + /** + * @return {string} - returns flattened textual representation of note, prefixes and attributes usable for searching + */ + get flatText() { + if (!this.flatTextCache) { if (this.isHideInAutocompleteOrArchived) { - this.fulltextCache = " "; // can't be empty - return this.fulltextCache; + this.flatTextCache = " "; // can't be empty + return this.flatTextCache; } - this.fulltextCache = this.title.toLowerCase(); + this.flatTextCache = this.title.toLowerCase(); for (const branch of this.parentBranches) { if (branch.prefix) { - this.fulltextCache += ' ' + branch.prefix; + this.flatTextCache += ' ' + branch.prefix; } } for (const attr of this.attributes) { // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words - this.fulltextCache += ' ' + attr.name.toLowerCase(); + this.flatTextCache += ' ' + attr.name.toLowerCase(); if (attr.value) { - this.fulltextCache += ' ' + attr.value.toLowerCase(); + this.flatTextCache += ' ' + attr.value.toLowerCase(); } } } - return this.fulltextCache; + return this.flatTextCache; } invalidateThisCache() { - this.fulltextCache = null; + this.flatTextCache = null; this.attributeCache = null; this.templateAttributeCache = null; @@ -184,18 +187,18 @@ class Note { } } - invalidateSubtreeFulltext() { - this.fulltextCache = null; + invalidateSubtreeFlatText() { + this.flatTextCache = null; for (const childNote of this.children) { - childNote.invalidateSubtreeFulltext(); + childNote.invalidateSubtreeFlatText(); } for (const templateAttr of this.templateAttributes) { const targetNote = templateAttr.targetNote; if (targetNote) { - targetNote.invalidateSubtreeFulltext(); + targetNote.invalidateSubtreeFlatText(); } } } @@ -384,7 +387,7 @@ function getCandidateNotes(tokens) { for (const note of Object.values(notes)) { for (const token of tokens) { - if (note.fulltext.includes(token)) { + if (note.flatText.includes(token)) { candidateNotes.push(note); break; } @@ -394,27 +397,14 @@ function getCandidateNotes(tokens) { return candidateNotes; } -async function findNotes(query) { - if (!query.length) { - return []; - } - - const allTokens = query - .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc - .toLowerCase() - .split(/[ -]/) - .filter(token => token !== '/'); // '/' is used as separator - - const candidateNotes = getCandidateNotes(allTokens); - - // now we have set of noteIds which match at least one token - +function findInNoteCache(tokens) { let results = []; - const tokens = allTokens.slice(); + + const candidateNotes = getCandidateNotes(tokens); for (const note of candidateNotes) { // autocomplete should be able to find notes by their noteIds as well (only leafs) - if (note.noteId === query) { + if (tokens.length === 1 && note.noteId === tokens[0]) { search(note, [], [], results); continue; } @@ -453,6 +443,58 @@ async function findNotes(query) { } } + return results; +} + +async function findInNoteContent(tokens) { + const wheres = tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); + + const noteIds = await sql.getColumn(` + SELECT notes.noteId + FROM notes + JOIN note_contents ON notes.noteId = note_contents.noteId + WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); + + const results = []; + + for (const noteId of noteIds) { + const note = notes[noteId]; + + if (!note) { + continue; + } + + const notePath = getSomePath(note); + const parentNoteId = notePath.length > 1 ? notePath[notePath.length - 2] : null; + + results.push({ + noteId: noteId, + branchId: getBranch(noteId, parentNoteId), + pathArray: notePath, + titleArray: getNoteTitleArrayForPath(notePath) + }); + } + + return results; +} + +async function findNotes(query, searchInContent = false) { + if (!query.trim().length) { + return []; + } + + const tokens = query + .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc + .toLowerCase() + .split(/[ -]/) + .filter(token => token !== '/'); // '/' is used as separator + + const cacheResults = findInNoteCache(tokens); + + const contentResults = searchInContent ? await findInNoteContent(tokens) : []; + + let results = cacheResults.concat(contentResults); + if (hoistedNoteService.getHoistedNoteId() !== 'root') { results = results.filter(res => res.pathArray.includes(hoistedNoteService.getHoistedNoteId())); } @@ -479,7 +521,7 @@ async function findNotes(query) { }; }); - highlightResults(apiResults, allTokens); + highlightResults(apiResults, tokens); return apiResults; } @@ -494,7 +536,7 @@ function search(note, tokens, path, results) { if (retPath) { const thisNoteId = retPath[retPath.length - 1]; - const thisParentNoteId = retPath[retPath.length - 2]; + const thisParentNoteId = retPath.length > 1 ? retPath[retPath.length - 2] : null; results.push({ noteId: thisNoteId, @@ -712,7 +754,7 @@ function getNotePath(noteId) { } function evaluateSimilarity(sourceNote, candidateNote, results) { - let coeff = stringSimilarity.compareTwoStrings(sourceNote.fulltext, candidateNote.fulltext); + let coeff = stringSimilarity.compareTwoStrings(sourceNote.flatText, candidateNote.flatText); if (coeff > 0.4) { const notePath = getSomePath(candidateNote); @@ -789,7 +831,7 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED note.title = entity.title; note.isProtected = entity.isProtected; note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; - note.fulltextCache = null; + note.flatTextCache = null; decryptProtectedNote(note); } @@ -828,7 +870,7 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED branches[branchId].prefix = entity.prefix; if (childNote) { - childNote.fulltextCache = null; + childNote.flatTextCache = null; } } else { @@ -862,10 +904,10 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED attr.value = entity.value; if (attr.isAffectingSubtree) { - note.invalidateSubtreeFulltext(); + note.invalidateSubtreeFlatText(); } else { - note.fulltextCache = null; + note.flatTextCache = null; } } else { From 5f1f65a3c2c1818d98856649f9602a4164d441cb Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 14 May 2020 23:21:48 +0200 Subject: [PATCH 11/47] fuction reorganization --- src/services/note_cache.js | 214 ++++++++++++++++++------------------- 1 file changed, 107 insertions(+), 107 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 7d3dee44a..9d37ba5bd 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -309,71 +309,52 @@ async function load() { loadedPromiseResolve(); } -function decryptProtectedNote(note) { - if (note.isProtected && !note.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { - note.title = protectedSessionService.decryptString(note.title); - - note.isDecrypted = true; +async function findNotes(query, searchInContent) { + if (!query.trim().length) { + return []; } -} -async function decryptProtectedNotes() { - for (const note of Object.values(notes)) { - decryptProtectedNote(note); + const tokens = query + .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc + .toLowerCase() + .split(/[ -]/) + .filter(token => token !== '/'); // '/' is used as separator + + const cacheResults = findInNoteCache(tokens); + + const contentResults = searchInContent ? await findInNoteContent(tokens) : []; + + let results = cacheResults.concat(contentResults); + + if (hoistedNoteService.getHoistedNoteId() !== 'root') { + results = results.filter(res => res.pathArray.includes(hoistedNoteService.getHoistedNoteId())); } -} -function formatAttribute(attr) { - if (attr.type === 'relation') { - return '@' + utils.escapeHtml(attr.name) + "=…"; - } - else if (attr.type === 'label') { - let label = '#' + utils.escapeHtml(attr.name); - - if (attr.value) { - const val = /[^\w_-]/.test(attr.value) ? '"' + attr.value + '"' : attr.value; - - label += '=' + utils.escapeHtml(val); + // sort results by depth of the note. This is based on the assumption that more important results + // are closer to the note root. + results.sort((a, b) => { + if (a.pathArray.length === b.pathArray.length) { + return a.title < b.title ? -1 : 1; } - return label; - } -} + return a.pathArray.length < b.pathArray.length ? -1 : 1; + }); -function highlightResults(results, allTokens) { - // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks - // which would make the resulting HTML string invalid. - // { and } are used for marking and tag (to avoid matches on single 'b' character) - allTokens = allTokens.map(token => token.replace('/[<\{\}]/g', '')); + const apiResults = results.slice(0, 200).map(res => { + const notePath = res.pathArray.join('/'); - // sort by the longest so we first highlight longest matches - allTokens.sort((a, b) => a.length > b.length ? -1 : 1); + return { + noteId: res.noteId, + branchId: res.branchId, + path: notePath, + pathTitle: res.titleArray.join(' / '), + noteTitle: getNoteTitleFromPath(notePath) + }; + }); - for (const result of results) { - const note = notes[result.noteId]; + highlightResults(apiResults, tokens); - for (const attr of note.attributes) { - if (allTokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { - result.pathTitle += ` ${formatAttribute(attr)}`; - } - } - - result.highlightedTitle = result.pathTitle; - } - - for (const token of allTokens) { - const tokenRegex = new RegExp("(" + utils.escapeRegExp(token) + ")", "gi"); - - for (const result of results) { - result.highlightedTitle = result.highlightedTitle.replace(tokenRegex, "{$1}"); - } - } - - for (const result of results) { - result.highlightedTitle = result.highlightedTitle - .replace(/{/g, "") - .replace(/}/g, ""); - } + return apiResults; } /** @@ -478,58 +459,6 @@ async function findInNoteContent(tokens) { return results; } -async function findNotes(query, searchInContent = false) { - if (!query.trim().length) { - return []; - } - - const tokens = query - .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc - .toLowerCase() - .split(/[ -]/) - .filter(token => token !== '/'); // '/' is used as separator - - const cacheResults = findInNoteCache(tokens); - - const contentResults = searchInContent ? await findInNoteContent(tokens) : []; - - let results = cacheResults.concat(contentResults); - - if (hoistedNoteService.getHoistedNoteId() !== 'root') { - results = results.filter(res => res.pathArray.includes(hoistedNoteService.getHoistedNoteId())); - } - - // sort results by depth of the note. This is based on the assumption that more important results - // are closer to the note root. - results.sort((a, b) => { - if (a.pathArray.length === b.pathArray.length) { - return a.title < b.title ? -1 : 1; - } - - return a.pathArray.length < b.pathArray.length ? -1 : 1; - }); - - const apiResults = results.slice(0, 200).map(res => { - const notePath = res.pathArray.join('/'); - - return { - noteId: res.noteId, - branchId: res.branchId, - path: notePath, - pathTitle: res.titleArray.join(' / '), - noteTitle: getNoteTitleFromPath(notePath) - }; - }); - - highlightResults(apiResults, tokens); - - return apiResults; -} - -function getBranch(childNoteId, parentNoteId) { - return childParentToBranch[`${childNoteId}-${parentNoteId}`]; -} - function search(note, tokens, path, results) { if (tokens.length === 0) { const retPath = getSomePath(note, path); @@ -585,6 +514,77 @@ function search(note, tokens, path, results) { } } +function highlightResults(results, allTokens) { + // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks + // which would make the resulting HTML string invalid. + // { and } are used for marking and tag (to avoid matches on single 'b' character) + allTokens = allTokens.map(token => token.replace('/[<\{\}]/g', '')); + + // sort by the longest so we first highlight longest matches + allTokens.sort((a, b) => a.length > b.length ? -1 : 1); + + for (const result of results) { + const note = notes[result.noteId]; + + for (const attr of note.attributes) { + if (allTokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { + result.pathTitle += ` ${formatAttribute(attr)}`; + } + } + + result.highlightedTitle = result.pathTitle; + } + + for (const token of allTokens) { + const tokenRegex = new RegExp("(" + utils.escapeRegExp(token) + ")", "gi"); + + for (const result of results) { + result.highlightedTitle = result.highlightedTitle.replace(tokenRegex, "{$1}"); + } + } + + for (const result of results) { + result.highlightedTitle = result.highlightedTitle + .replace(/{/g, "") + .replace(/}/g, ""); + } +} + +function decryptProtectedNote(note) { + if (note.isProtected && !note.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { + note.title = protectedSessionService.decryptString(note.title); + + note.isDecrypted = true; + } +} + +async function decryptProtectedNotes() { + for (const note of Object.values(notes)) { + decryptProtectedNote(note); + } +} + +function formatAttribute(attr) { + if (attr.type === 'relation') { + return '@' + utils.escapeHtml(attr.name) + "=…"; + } + else if (attr.type === 'label') { + let label = '#' + utils.escapeHtml(attr.name); + + if (attr.value) { + const val = /[^\w_-]/.test(attr.value) ? '"' + attr.value + '"' : attr.value; + + label += '=' + utils.escapeHtml(val); + } + + return label; + } +} + +function getBranch(childNoteId, parentNoteId) { + return childParentToBranch[`${childNoteId}-${parentNoteId}`]; +} + function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; const note = notes[noteId]; From f07025f7415c537f23e572cbc93c6d63212b7561 Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 15 May 2020 11:04:55 +0200 Subject: [PATCH 12/47] start of note cache expression implementation --- src/routes/api/autocomplete.js | 4 +- src/services/note_cache.js | 250 +++++++++++++++++++++++++++++---- 2 files changed, 221 insertions(+), 33 deletions(-) diff --git a/src/routes/api/autocomplete.js b/src/routes/api/autocomplete.js index 307746b6c..a8255b3eb 100644 --- a/src/routes/api/autocomplete.js +++ b/src/routes/api/autocomplete.js @@ -18,7 +18,7 @@ async function getAutocomplete(req) { results = await getRecentNotes(activeNoteId); } else { - results = await noteCacheService.findNotes(query); + results = await noteCacheService.findNotesWithFulltext(query); } const msTaken = Date.now() - timestampStarted; @@ -67,4 +67,4 @@ async function getRecentNotes(activeNoteId) { module.exports = { getAutocomplete -}; \ No newline at end of file +}; diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 9d37ba5bd..4f4700d19 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -12,6 +12,13 @@ let notes; let branches /** @type {Object.} */ let attributes; +/** @type {Object.} Points from attribute type-name to list of attributes them */ +let attributeIndex; + +/** @return {Attribute[]} */ +function findAttributes(type, name) { + return attributeIndex[`${type}-${name}`] || []; +} let childParentToBranch = {}; @@ -37,10 +44,11 @@ class Note { /** @param {Attribute[]|null} */ this.attributeCache = null; /** @param {Attribute[]|null} */ - this.templateAttributeCache = null; - /** @param {Attribute[]|null} */ this.inheritableAttributeCache = null; + /** @param {Attribute[]} */ + this.targetRelations = []; + /** @param {string|null} */ this.flatTextCache = null; @@ -74,16 +82,11 @@ class Note { this.attributeCache = parentAttributes.concat(templateAttributes); this.inheritableAttributeCache = []; - this.templateAttributeCache = []; for (const attr of this.attributeCache) { if (attr.isInheritable) { this.inheritableAttributeCache.push(attr); } - - if (attr.type === 'relation' && attr.name === 'template') { - this.templateAttributeCache.push(attr); - } } } @@ -99,15 +102,6 @@ class Note { return this.inheritableAttributeCache; } - /** @return {Attribute[]} */ - get templateAttributes() { - if (!this.templateAttributeCache) { - this.attributes; // will refresh also this.templateAttributeCache - } - - return this.templateAttributeCache; - } - hasAttribute(type, name) { return this.attributes.find(attr => attr.type === type && attr.name === name); } @@ -167,7 +161,6 @@ class Note { this.flatTextCache = null; this.attributeCache = null; - this.templateAttributeCache = null; this.inheritableAttributeCache = null; } @@ -178,11 +171,13 @@ class Note { childNote.invalidateSubtreeCaches(); } - for (const templateAttr of this.templateAttributes) { - const targetNote = templateAttr.targetNote; + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; - if (targetNote) { - targetNote.invalidateSubtreeCaches(); + if (note) { + note.invalidateSubtreeCaches(); + } } } } @@ -194,14 +189,59 @@ class Note { childNote.invalidateSubtreeFlatText(); } - for (const templateAttr of this.templateAttributes) { - const targetNote = templateAttr.targetNote; + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; - if (targetNote) { - targetNote.invalidateSubtreeFlatText(); + if (note) { + note.invalidateSubtreeFlatText(); + } } } } + + get isTemplate() { + return !!this.targetRelations.find(rel => rel.name === 'template'); + } + + /** @return {Note[]} */ + get subtreeNotes() { + const arr = [[this]]; + + for (const childNote of this.children) { + arr.push(childNote.subtreeNotes); + } + + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; + + if (note) { + arr.push(note.subtreeNotes); + } + } + } + + return arr.flat(); + } + + /** @return {Note[]} - returns only notes which are templated, does not include their subtrees + * in effect returns notes which are influenced by note's non-inheritable attributes */ + get templatedNotes() { + const arr = [this]; + + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; + + if (note) { + arr.push(note); + } + } + } + + return arr; + } } class Branch { @@ -263,6 +303,16 @@ class Attribute { this.isInheritable = !!row.isInheritable; notes[this.noteId].ownedAttributes.push(this); + + const key = `${this.type-this.name}`; + attributeIndex[key] = attributeIndex[key] || []; + attributeIndex[key].push(this); + + const targetNote = this.targetNote; + + if (targetNote) { + targetNote.targetRelations.push(this); + } } get isAffectingSubtree() { @@ -270,6 +320,10 @@ class Attribute { || (this.type === 'relation' && this.name === 'template'); } + get note() { + return notes[this.noteId]; + } + get targetNote() { if (this.type === 'relation') { return notes[this.value]; @@ -309,7 +363,133 @@ async function load() { loadedPromiseResolve(); } -async function findNotes(query, searchInContent) { +const expression = { + operator: 'and', + operands: [ + { + operator: 'exists', + fieldName: 'hokus' + } + ] +}; + +class AndOp { + constructor(subExpressions) { + this.subExpressions = subExpressions; + } + + execute(noteSet) { + for (const subExpression of this.subExpressions) { + noteSet = subExpression.execute(noteSet); + } + + return noteSet; + } +} + +class OrOp { + constructor(subExpressions) { + this.subExpressions = subExpressions; + } + + execute(noteSet) { + const resultNoteSet = new NoteSet(); + + for (const subExpression of this.subExpressions) { + resultNoteSet.mergeIn(subExpression.execute(noteSet)); + } + + return resultNoteSet; + } +} + +class NoteSet { + constructor(arr = []) { + this.arr = arr; + } + + add(note) { + this.arr.push(note); + } + + addAll(notes) { + this.arr.push(...notes); + } + + hasNoteId(noteId) { + // TODO: optimize + return !!this.arr.find(note => note.noteId === noteId); + } + + mergeIn(anotherNoteSet) { + this.arr = this.arr.concat(anotherNoteSet.arr); + } +} + +class ExistsOp { + constructor(attributeType, attributeName) { + this.attributeType = attributeType; + this.attributeName = attributeName; + } + + execute(noteSet) { + const attrs = findAttributes(this.attributeType, this.attributeName); + const resultNoteSet = new NoteSet(); + + for (const attr of attrs) { + const note = attr.note; + + if (noteSet.hasNoteId(note.noteId)) { + if (attr.isInheritable) { + resultNoteSet.addAll(note.subtreeNotes); + } + else if (note.isTemplate) { + resultNoteSet.addAll(note.templatedNotes); + } + else { + resultNoteSet.add(note); + } + } + } + } +} + +class EqualsOp { + constructor(attributeType, attributeName, attributeValue) { + this.attributeType = attributeType; + this.attributeName = attributeName; + this.attributeValue = attributeValue; + } + + execute(noteSet) { + const attrs = findAttributes(this.attributeType, this.attributeName); + const resultNoteSet = new NoteSet(); + + for (const attr of attrs) { + const note = attr.note; + + if (noteSet.hasNoteId(note.noteId) && attr.value === this.attributeValue) { + if (attr.isInheritable) { + resultNoteSet.addAll(note.subtreeNotes); + } + else if (note.isTemplate) { + resultNoteSet.addAll(note.templatedNotes); + } + else { + resultNoteSet.add(note); + } + } + } + } +} + +async function findNotesWithExpression(expression) { + const allNoteSet = new NoteSet(Object.values(notes)); + + expression.execute(allNoteSet); +} + +async function findNotesWithFulltext(query, searchInContent) { if (!query.trim().length) { return []; } @@ -888,14 +1068,22 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED if (entity.isDeleted) { if (note && attr) { + // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); - if (attr.isAffectingSubtree) { - note.invalidateSubtreeCaches(); + const targetNote = attr.targetNote; + + if (targetNote) { + targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); } } delete attributes[attributeId]; + delete attributeIndex[`${attr.type}-${attr.name}`]; } else if (attributeId in attributes) { const attr = attributes[attributeId]; @@ -903,7 +1091,7 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED // attr name and isInheritable are immutable attr.value = entity.value; - if (attr.isAffectingSubtree) { + if (attr.isAffectingSubtree || note.isTemplate) { note.invalidateSubtreeFlatText(); } else { @@ -915,7 +1103,7 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED attributes[attributeId] = attr; if (note) { - if (attr.isAffectingSubtree) { + if (attr.isAffectingSubtree || note.isTemplate) { note.invalidateSubtreeCaches(); } else { @@ -944,7 +1132,7 @@ sqlInit.dbReady.then(() => utils.stopWatch("Note cache load", load)); module.exports = { loadedPromise, - findNotes, + findNotesWithFulltext, getNotePath, getNoteTitleForPath, getNoteTitleFromPath, From 78ea0b4ba9fc1993e9c7c7de868002fb4a0189f9 Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 15 May 2020 23:50:36 +0200 Subject: [PATCH 13/47] refactoring ... --- src/services/note_cache.js | 379 +++++++++++++++++++------------------ 1 file changed, 200 insertions(+), 179 deletions(-) diff --git a/src/services/note_cache.js b/src/services/note_cache.js index 4f4700d19..e52619f3d 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -136,22 +136,26 @@ class Note { return this.flatTextCache; } - this.flatTextCache = this.title.toLowerCase(); + this.flatTextCache = ''; for (const branch of this.parentBranches) { if (branch.prefix) { - this.flatTextCache += ' ' + branch.prefix; + this.flatTextCache += branch.prefix + ' - '; } } + this.flatTextCache += this.title; + for (const attr of this.attributes) { // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words - this.flatTextCache += ' ' + attr.name.toLowerCase(); + this.flatTextCache += (attr.type === 'label' ? '#' : '@') + attr.name; if (attr.value) { - this.flatTextCache += ' ' + attr.value.toLowerCase(); + this.flatTextCache += '=' + attr.value; } } + + this.flatTextCache = this.flatTextCache.toLowerCase(); } return this.flatTextCache; @@ -205,11 +209,11 @@ class Note { } /** @return {Note[]} */ - get subtreeNotes() { + get subtreeNotesIncludingTemplated() { const arr = [[this]]; for (const childNote of this.children) { - arr.push(childNote.subtreeNotes); + arr.push(childNote.subtreeNotesIncludingTemplated); } for (const targetRelation of this.targetRelations) { @@ -217,7 +221,7 @@ class Note { const note = targetRelation.note; if (note) { - arr.push(note.subtreeNotes); + arr.push(note.subtreeNotesIncludingTemplated); } } } @@ -225,6 +229,17 @@ class Note { return arr.flat(); } + /** @return {Note[]} */ + get subtreeNotes() { + const arr = [[this]]; + + for (const childNote of this.children) { + arr.push(childNote.subtreeNotes); + } + + return arr.flat(); + } + /** @return {Note[]} - returns only notes which are templated, does not include their subtrees * in effect returns notes which are influenced by note's non-inheritable attributes */ get templatedNotes() { @@ -378,9 +393,9 @@ class AndOp { this.subExpressions = subExpressions; } - execute(noteSet) { + execute(noteSet, searchContext) { for (const subExpression of this.subExpressions) { - noteSet = subExpression.execute(noteSet); + noteSet = subExpression.execute(noteSet, searchContext); } return noteSet; @@ -392,11 +407,11 @@ class OrOp { this.subExpressions = subExpressions; } - execute(noteSet) { + execute(noteSet, searchContext) { const resultNoteSet = new NoteSet(); for (const subExpression of this.subExpressions) { - resultNoteSet.mergeIn(subExpression.execute(noteSet)); + resultNoteSet.mergeIn(subExpression.execute(noteSet, searchContext)); } return resultNoteSet; @@ -404,25 +419,25 @@ class OrOp { } class NoteSet { - constructor(arr = []) { - this.arr = arr; + constructor(notes = []) { + this.notes = notes; } add(note) { - this.arr.push(note); + this.notes.push(note); } addAll(notes) { - this.arr.push(...notes); + this.notes.push(...notes); } hasNoteId(noteId) { // TODO: optimize - return !!this.arr.find(note => note.noteId === noteId); + return !!this.notes.find(note => note.noteId === noteId); } mergeIn(anotherNoteSet) { - this.arr = this.arr.concat(anotherNoteSet.arr); + this.notes = this.notes.concat(anotherNoteSet.arr); } } @@ -441,7 +456,7 @@ class ExistsOp { if (noteSet.hasNoteId(note.noteId)) { if (attr.isInheritable) { - resultNoteSet.addAll(note.subtreeNotes); + resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); } else if (note.isTemplate) { resultNoteSet.addAll(note.templatedNotes); @@ -470,7 +485,7 @@ class EqualsOp { if (noteSet.hasNoteId(note.noteId) && attr.value === this.attributeValue) { if (attr.isInheritable) { - resultNoteSet.addAll(note.subtreeNotes); + resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); } else if (note.isTemplate) { resultNoteSet.addAll(note.templatedNotes); @@ -483,10 +498,173 @@ class EqualsOp { } } -async function findNotesWithExpression(expression) { - const allNoteSet = new NoteSet(Object.values(notes)); +class NoteContentFulltextOp { + constructor(tokens) { + this.tokens = tokens; + } - expression.execute(allNoteSet); + async execute(noteSet) { + const resultNoteSet = new NoteSet(); + const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); + + const noteIds = await sql.getColumn(` + SELECT notes.noteId + FROM notes + JOIN note_contents ON notes.noteId = note_contents.noteId + WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); + + const results = []; + + for (const noteId of noteIds) { + if (noteSet.hasNoteId(noteId) && noteId in notes) { + resultNoteSet.add(notes[noteId]); + } + } + + return results; + } +} + +class NoteCacheFulltextOp { + constructor(tokens) { + this.tokens = tokens; + } + + execute(noteSet, searchContext) { + const resultNoteSet = new NoteSet(); + + const candidateNotes = this.getCandidateNotes(noteSet); + + for (const note of candidateNotes) { + // autocomplete should be able to find notes by their noteIds as well (only leafs) + if (this.tokens.length === 1 && note.noteId === this.tokens[0]) { + this.searchDownThePath(note, [], [], resultNoteSet, searchContext); + continue; + } + + // for leaf note it doesn't matter if "archived" label is inheritable or not + if (note.isArchived) { + continue; + } + + const foundAttrTokens = []; + + for (const attribute of note.ownedAttributes) { + for (const token of this.tokens) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { + foundAttrTokens.push(token); + } + } + } + + for (const parentNote of note.parents) { + const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const foundTokens = foundAttrTokens.slice(); + + for (const token of this.tokens) { + if (title.includes(token)) { + foundTokens.push(token); + } + } + + if (foundTokens.length > 0) { + const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); + + this.searchDownThePath(parentNote, remainingTokens, [note.noteId], resultNoteSet, searchContext); + } + } + } + + return resultNoteSet; + } + + /** + * Returns noteIds which have at least one matching tokens + * + * @param {NoteSet} noteSet + * @return {String[]} + */ + getCandidateNotes(noteSet) { + const candidateNotes = []; + + for (const note of noteSet.notes) { + for (const token of this.tokens) { + if (note.flatText.includes(token)) { + candidateNotes.push(note); + break; + } + } + } + + return candidateNotes; + } + + searchDownThePath(note, tokens, path, resultNoteSet, searchContext) { + if (tokens.length === 0) { + const retPath = getSomePath(note, path); + + if (retPath) { + const noteId = retPath[retPath.length - 1]; + searchContext.noteIdToNotePath[noteId] = retPath; + + resultNoteSet.add(notes[noteId]); + } + + return; + } + + if (!note.parents.length === 0 || note.noteId === 'root') { + return; + } + + const foundAttrTokens = []; + + for (const attribute of note.ownedAttributes) { + for (const token of tokens) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { + foundAttrTokens.push(token); + } + } + } + + for (const parentNote of note.parents) { + const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const foundTokens = foundAttrTokens.slice(); + + for (const token of tokens) { + if (title.includes(token)) { + foundTokens.push(token); + } + } + + if (foundTokens.length > 0) { + const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); + + this.searchDownThePath(parentNote, remainingTokens, path.concat([note.noteId]), resultNoteSet, searchContext); + } + else { + this.searchDownThePath(parentNote, tokens, path.concat([note.noteId]), resultNoteSet, searchContext); + } + } + } +} + +async function findNotesWithExpression(expression) { + + const hoistedNote = notes[hoistedNoteService.getHoistedNoteId()]; + const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') + ? hoistedNote.subtreeNotes + : Object.values(notes); + + const allNoteSet = new NoteSet(allNotes); + + const searchContext = { + noteIdToNotePath: {} + }; + + expression.execute(allNoteSet, searchContext); } async function findNotesWithFulltext(query, searchInContent) { @@ -537,163 +715,6 @@ async function findNotesWithFulltext(query, searchInContent) { return apiResults; } -/** - * Returns noteIds which have at least one matching tokens - * - * @param tokens - * @return {String[]} - */ -function getCandidateNotes(tokens) { - const candidateNotes = []; - - for (const note of Object.values(notes)) { - for (const token of tokens) { - if (note.flatText.includes(token)) { - candidateNotes.push(note); - break; - } - } - } - - return candidateNotes; -} - -function findInNoteCache(tokens) { - let results = []; - - const candidateNotes = getCandidateNotes(tokens); - - for (const note of candidateNotes) { - // autocomplete should be able to find notes by their noteIds as well (only leafs) - if (tokens.length === 1 && note.noteId === tokens[0]) { - search(note, [], [], results); - continue; - } - - // for leaf note it doesn't matter if "archived" label is inheritable or not - if (note.isArchived) { - continue; - } - - const foundAttrTokens = []; - - for (const attribute of note.ownedAttributes) { - for (const token of tokens) { - if (attribute.name.toLowerCase().includes(token) - || attribute.value.toLowerCase().includes(token)) { - foundAttrTokens.push(token); - } - } - } - - for (const parentNote of note.parents) { - const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); - const foundTokens = foundAttrTokens.slice(); - - for (const token of tokens) { - if (title.includes(token)) { - foundTokens.push(token); - } - } - - if (foundTokens.length > 0) { - const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - - search(parentNote, remainingTokens, [note.noteId], results); - } - } - } - - return results; -} - -async function findInNoteContent(tokens) { - const wheres = tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); - - const noteIds = await sql.getColumn(` - SELECT notes.noteId - FROM notes - JOIN note_contents ON notes.noteId = note_contents.noteId - WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); - - const results = []; - - for (const noteId of noteIds) { - const note = notes[noteId]; - - if (!note) { - continue; - } - - const notePath = getSomePath(note); - const parentNoteId = notePath.length > 1 ? notePath[notePath.length - 2] : null; - - results.push({ - noteId: noteId, - branchId: getBranch(noteId, parentNoteId), - pathArray: notePath, - titleArray: getNoteTitleArrayForPath(notePath) - }); - } - - return results; -} - -function search(note, tokens, path, results) { - if (tokens.length === 0) { - const retPath = getSomePath(note, path); - - if (retPath) { - const thisNoteId = retPath[retPath.length - 1]; - const thisParentNoteId = retPath.length > 1 ? retPath[retPath.length - 2] : null; - - results.push({ - noteId: thisNoteId, - branchId: getBranch(thisNoteId, thisParentNoteId), - pathArray: retPath, - titleArray: getNoteTitleArrayForPath(retPath) - }); - } - - return; - } - - if (!note.parents.length === 0 || note.noteId === 'root') { - return; - } - - const foundAttrTokens = []; - - for (const attribute of note.ownedAttributes) { - for (const token of tokens) { - if (attribute.name.toLowerCase().includes(token) - || attribute.value.toLowerCase().includes(token)) { - foundAttrTokens.push(token); - } - } - } - - for (const parentNote of note.parents) { - const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); - const foundTokens = foundAttrTokens.slice(); - - for (const token of tokens) { - if (title.includes(token)) { - foundTokens.push(token); - } - } - - if (foundTokens.length > 0) { - const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - - search(parentNote, remainingTokens, path.concat([note.noteId]), results); - } - else { - search(parentNote, tokens, path.concat([note.noteId]), results); - } - } -} - function highlightResults(results, allTokens) { // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks // which would make the resulting HTML string invalid. From e3071e630a33688d8002273db6ae486d84984897 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 16 May 2020 22:11:09 +0200 Subject: [PATCH 14/47] note cache refactoring WIP --- src/public/app/dialogs/add_link.js | 4 +- src/public/app/dialogs/attributes.js | 2 +- src/public/app/dialogs/clone_to.js | 4 +- src/public/app/dialogs/include_note.js | 4 +- src/public/app/dialogs/move_to.js | 4 +- src/public/app/services/note_autocomplete.js | 32 ++-- src/public/app/widgets/promoted_attributes.js | 4 +- src/routes/api/autocomplete.js | 9 +- src/services/note_cache.js | 150 +++++++++--------- 9 files changed, 102 insertions(+), 111 deletions(-) diff --git a/src/public/app/dialogs/add_link.js b/src/public/app/dialogs/add_link.js index 43f67df85..47270c164 100644 --- a/src/public/app/dialogs/add_link.js +++ b/src/public/app/dialogs/add_link.js @@ -75,7 +75,7 @@ function updateTitleFormGroupVisibility() { } $form.on('submit', () => { - const notePath = $autoComplete.getSelectedPath(); + const notePath = $autoComplete.getSelectedNotePath(); if (notePath) { $dialog.modal('hide'); @@ -89,4 +89,4 @@ $form.on('submit', () => { } return false; -}); \ No newline at end of file +}); diff --git a/src/public/app/dialogs/attributes.js b/src/public/app/dialogs/attributes.js index 2cf844420..e4264114c 100644 --- a/src/public/app/dialogs/attributes.js +++ b/src/public/app/dialogs/attributes.js @@ -269,7 +269,7 @@ function initKoPlugins() { init: function (element, valueAccessor, allBindings, viewModel, bindingContext) { noteAutocompleteService.initNoteAutocomplete($(element)); - $(element).setSelectedPath(bindingContext.$data.selectedPath); + $(element).setSelectedNotePath(bindingContext.$data.selectedPath); $(element).on('autocomplete:selected', function (event, suggestion, dataset) { bindingContext.$data.selectedPath = $(element).val().trim() ? suggestion.path : ''; diff --git a/src/public/app/dialogs/clone_to.js b/src/public/app/dialogs/clone_to.js index eab144aac..14e231aae 100644 --- a/src/public/app/dialogs/clone_to.js +++ b/src/public/app/dialogs/clone_to.js @@ -52,7 +52,7 @@ async function cloneNotesTo(notePath) { } $form.on('submit', () => { - const notePath = $noteAutoComplete.getSelectedPath(); + const notePath = $noteAutoComplete.getSelectedNotePath(); if (notePath) { $dialog.modal('hide'); @@ -64,4 +64,4 @@ $form.on('submit', () => { } return false; -}); \ No newline at end of file +}); diff --git a/src/public/app/dialogs/include_note.js b/src/public/app/dialogs/include_note.js index 71d3cb220..1aa2b149e 100644 --- a/src/public/app/dialogs/include_note.js +++ b/src/public/app/dialogs/include_note.js @@ -38,7 +38,7 @@ async function includeNote(notePath) { } $form.on('submit', () => { - const notePath = $autoComplete.getSelectedPath(); + const notePath = $autoComplete.getSelectedNotePath(); if (notePath) { $dialog.modal('hide'); @@ -50,4 +50,4 @@ $form.on('submit', () => { } return false; -}); \ No newline at end of file +}); diff --git a/src/public/app/dialogs/move_to.js b/src/public/app/dialogs/move_to.js index 4dbc6bcca..afd9ceb5f 100644 --- a/src/public/app/dialogs/move_to.js +++ b/src/public/app/dialogs/move_to.js @@ -41,7 +41,7 @@ async function moveNotesTo(parentNoteId) { } $form.on('submit', () => { - const notePath = $noteAutoComplete.getSelectedPath(); + const notePath = $noteAutoComplete.getSelectedNotePath(); if (notePath) { $dialog.modal('hide'); @@ -55,4 +55,4 @@ $form.on('submit', () => { } return false; -}); \ No newline at end of file +}); diff --git a/src/public/app/services/note_autocomplete.js b/src/public/app/services/note_autocomplete.js index 4f6b6a575..c9ae9be8d 100644 --- a/src/public/app/services/note_autocomplete.js +++ b/src/public/app/services/note_autocomplete.js @@ -3,7 +3,7 @@ import appContext from "./app_context.js"; import utils from './utils.js'; // this key needs to have this value so it's hit by the tooltip -const SELECTED_PATH_KEY = "data-note-path"; +const SELECTED_NOTE_PATH_KEY = "data-note-path"; async function autocompleteSource(term, cb) { const result = await server.get('autocomplete' @@ -12,8 +12,8 @@ async function autocompleteSource(term, cb) { if (result.length === 0) { result.push({ - pathTitle: "No results", - path: "" + notePathTitle: "No results", + notePath: "" }); } @@ -25,7 +25,7 @@ function clearText($el) { return; } - $el.setSelectedPath(""); + $el.setSelectedNotePath(""); $el.autocomplete("val", "").trigger('change'); } @@ -34,7 +34,7 @@ function showRecentNotes($el) { return; } - $el.setSelectedPath(""); + $el.setSelectedNotePath(""); $el.autocomplete("val", ""); $el.trigger('focus'); } @@ -91,10 +91,10 @@ function initNoteAutocomplete($el, options) { }, [ { source: autocompleteSource, - displayKey: 'pathTitle', + displayKey: 'notePathTitle', templates: { suggestion: function(suggestion) { - return suggestion.highlightedTitle; + return suggestion.highlightedNotePathTitle; } }, // we can't cache identical searches because notes can be created / renamed, new recent notes can be added @@ -102,7 +102,7 @@ function initNoteAutocomplete($el, options) { } ]); - $el.on('autocomplete:selected', (event, suggestion) => $el.setSelectedPath(suggestion.path)); + $el.on('autocomplete:selected', (event, suggestion) => $el.setSelectedNotePath(suggestion.notePath)); $el.on('autocomplete:closed', () => { if (!$el.val().trim()) { clearText($el); @@ -113,24 +113,24 @@ function initNoteAutocomplete($el, options) { } function init() { - $.fn.getSelectedPath = function () { + $.fn.getSelectedNotePath = function () { if (!$(this).val().trim()) { return ""; } else { - return $(this).attr(SELECTED_PATH_KEY); + return $(this).attr(SELECTED_NOTE_PATH_KEY); } }; - $.fn.setSelectedPath = function (path) { - path = path || ""; + $.fn.setSelectedNotePath = function (notePath) { + notePath = notePath || ""; - $(this).attr(SELECTED_PATH_KEY, path); + $(this).attr(SELECTED_NOTE_PATH_KEY, notePath); $(this) .closest(".input-group") .find(".go-to-selected-note-button") - .toggleClass("disabled", !path.trim()) - .attr(SELECTED_PATH_KEY, path); // we also set attr here so tooltip can be displayed + .toggleClass("disabled", !notePath.trim()) + .attr(SELECTED_NOTE_PATH_KEY, notePath); // we also set attr here so tooltip can be displayed }; } @@ -139,4 +139,4 @@ export default { initNoteAutocomplete, showRecentNotes, init -} \ No newline at end of file +} diff --git a/src/public/app/widgets/promoted_attributes.js b/src/public/app/widgets/promoted_attributes.js index 58e025d07..859ca2291 100644 --- a/src/public/app/widgets/promoted_attributes.js +++ b/src/public/app/widgets/promoted_attributes.js @@ -200,7 +200,7 @@ export default class PromotedAttributesWidget extends TabAwareWidget { this.promotedAttributeChanged(event); }); - $input.setSelectedPath(valueAttr.value); + $input.setSelectedNotePath(valueAttr.value); } else { ws.logError("Unknown attribute type=" + valueAttr.type); @@ -250,7 +250,7 @@ export default class PromotedAttributesWidget extends TabAwareWidget { value = $attr.is(':checked') ? "true" : "false"; } else if ($attr.prop("attribute-type") === "relation") { - const selectedPath = $attr.getSelectedPath(); + const selectedPath = $attr.getSelectedNotePath(); value = selectedPath ? treeService.getNoteIdFromNotePath(selectedPath) : ""; } diff --git a/src/routes/api/autocomplete.js b/src/routes/api/autocomplete.js index a8255b3eb..f5c416eb0 100644 --- a/src/routes/api/autocomplete.js +++ b/src/routes/api/autocomplete.js @@ -18,7 +18,7 @@ async function getAutocomplete(req) { results = await getRecentNotes(activeNoteId); } else { - results = await noteCacheService.findNotesWithFulltext(query); + results = await noteCacheService.findNotesForAutocomplete(query); } const msTaken = Date.now() - timestampStarted; @@ -57,10 +57,9 @@ async function getRecentNotes(activeNoteId) { const title = noteCacheService.getNoteTitleForPath(rn.notePath.split('/')); return { - path: rn.notePath, - pathTitle: title, - highlightedTitle: title, - noteTitle: noteCacheService.getNoteTitleFromPath(rn.notePath) + notePath: rn.notePath, + notePathTitle: title, + highlightedNotePathTitle: utils.escapeHtml(title) }; }); } diff --git a/src/services/note_cache.js b/src/services/note_cache.js index e52619f3d..b35c449e8 100644 --- a/src/services/note_cache.js +++ b/src/services/note_cache.js @@ -371,6 +371,8 @@ async function load() { branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, row => new Branch(row)); + attributeIndex = []; + attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, row => new Attribute(row)); @@ -378,17 +380,7 @@ async function load() { loadedPromiseResolve(); } -const expression = { - operator: 'and', - operands: [ - { - operator: 'exists', - fieldName: 'hokus' - } - ] -}; - -class AndOp { +class AndExp { constructor(subExpressions) { this.subExpressions = subExpressions; } @@ -402,7 +394,7 @@ class AndOp { } } -class OrOp { +class OrExp { constructor(subExpressions) { this.subExpressions = subExpressions; } @@ -441,7 +433,7 @@ class NoteSet { } } -class ExistsOp { +class ExistsExp { constructor(attributeType, attributeName) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -469,7 +461,7 @@ class ExistsOp { } } -class EqualsOp { +class EqualsExp { constructor(attributeType, attributeName, attributeValue) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -498,7 +490,7 @@ class EqualsOp { } } -class NoteContentFulltextOp { +class NoteContentFulltextExp { constructor(tokens) { this.tokens = tokens; } @@ -525,7 +517,7 @@ class NoteContentFulltextOp { } } -class NoteCacheFulltextOp { +class NoteCacheFulltextExp { constructor(tokens) { this.tokens = tokens; } @@ -569,7 +561,7 @@ class NoteCacheFulltextOp { } if (foundTokens.length > 0) { - const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); + const remainingTokens = this.tokens.filter(token => !foundTokens.includes(token)); this.searchDownThePath(parentNote, remainingTokens, [note.noteId], resultNoteSet, searchContext); } @@ -651,6 +643,21 @@ class NoteCacheFulltextOp { } } +class SearchResult { + constructor(notePathArray) { + this.notePathArray = notePathArray; + this.notePathTitle = getNoteTitleForPath(notePathArray); + } + + get notePath() { + return this.notePathArray.join('/'); + } + + get noteId() { + return this.notePathArray[this.notePathArray.length - 1]; + } +} + async function findNotesWithExpression(expression) { const hoistedNote = notes[hoistedNoteService.getHoistedNoteId()]; @@ -664,10 +671,27 @@ async function findNotesWithExpression(expression) { noteIdToNotePath: {} }; - expression.execute(allNoteSet, searchContext); + const noteSet = await expression.execute(allNoteSet, searchContext); + + let searchResults = noteSet.notes + .map(note => searchContext.noteIdToNotePath[note.noteId] || getSomePath(note)) + .filter(notePathArray => notePathArray.includes(hoistedNoteService.getHoistedNoteId())) + .map(notePathArray => new SearchResult(notePathArray)); + + // sort results by depth of the note. This is based on the assumption that more important results + // are closer to the note root. + searchResults.sort((a, b) => { + if (a.notePathArray.length === b.notePathArray.length) { + return a.notePathTitle < b.notePathTitle ? -1 : 1; + } + + return a.notePathArray.length < b.notePathArray.length ? -1 : 1; + }); + + return searchResults; } -async function findNotesWithFulltext(query, searchInContent) { +async function findNotesForAutocomplete(query) { if (!query.trim().length) { return []; } @@ -678,74 +702,54 @@ async function findNotesWithFulltext(query, searchInContent) { .split(/[ -]/) .filter(token => token !== '/'); // '/' is used as separator - const cacheResults = findInNoteCache(tokens); + const expression = new NoteCacheFulltextExp(tokens); - const contentResults = searchInContent ? await findInNoteContent(tokens) : []; + let searchResults = await findNotesWithExpression(expression); - let results = cacheResults.concat(contentResults); + searchResults = searchResults.slice(0, 200); - if (hoistedNoteService.getHoistedNoteId() !== 'root') { - results = results.filter(res => res.pathArray.includes(hoistedNoteService.getHoistedNoteId())); - } - - // sort results by depth of the note. This is based on the assumption that more important results - // are closer to the note root. - results.sort((a, b) => { - if (a.pathArray.length === b.pathArray.length) { - return a.title < b.title ? -1 : 1; - } - - return a.pathArray.length < b.pathArray.length ? -1 : 1; - }); - - const apiResults = results.slice(0, 200).map(res => { - const notePath = res.pathArray.join('/'); + highlightSearchResults(searchResults, tokens); + return searchResults.map(result => { return { - noteId: res.noteId, - branchId: res.branchId, - path: notePath, - pathTitle: res.titleArray.join(' / '), - noteTitle: getNoteTitleFromPath(notePath) - }; + notePath: result.notePath, + notePathTitle: result.notePathTitle, + highlightedNotePathTitle: result.highlightedNotePathTitle + } }); - - highlightResults(apiResults, tokens); - - return apiResults; } -function highlightResults(results, allTokens) { +function highlightSearchResults(searchResults, tokens) { // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks // which would make the resulting HTML string invalid. // { and } are used for marking and tag (to avoid matches on single 'b' character) - allTokens = allTokens.map(token => token.replace('/[<\{\}]/g', '')); + tokens = tokens.map(token => token.replace('/[<\{\}]/g', '')); // sort by the longest so we first highlight longest matches - allTokens.sort((a, b) => a.length > b.length ? -1 : 1); + tokens.sort((a, b) => a.length > b.length ? -1 : 1); - for (const result of results) { + for (const result of searchResults) { const note = notes[result.noteId]; + result.highlightedNotePathTitle = result.notePathTitle; + for (const attr of note.attributes) { - if (allTokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { - result.pathTitle += ` ${formatAttribute(attr)}`; + if (tokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { + result.highlightedNotePathTitle += ` ${formatAttribute(attr)}`; } } - - result.highlightedTitle = result.pathTitle; } - for (const token of allTokens) { + for (const token of tokens) { const tokenRegex = new RegExp("(" + utils.escapeRegExp(token) + ")", "gi"); - for (const result of results) { - result.highlightedTitle = result.highlightedTitle.replace(tokenRegex, "{$1}"); + for (const result of searchResults) { + result.highlightedNotePathTitle = result.highlightedNotePathTitle.replace(tokenRegex, "{$1}"); } } - for (const result of results) { - result.highlightedTitle = result.highlightedTitle + for (const result of searchResults) { + result.highlightedNotePathTitle = result.highlightedNotePathTitle .replace(/{/g, "") .replace(/}/g, ""); } @@ -839,17 +843,6 @@ function isInAncestor(noteId, ancestorNoteId) { return false; } -function getNoteTitleFromPath(notePath) { - const pathArr = notePath.split("/"); - - if (pathArr.length === 1) { - return getNoteTitle(pathArr[0], 'root'); - } - else { - return getNoteTitle(pathArr[pathArr.length - 1], pathArr[pathArr.length - 2]); - } -} - function getNoteTitle(childNoteId, parentNoteId) { const childNote = notes[childNoteId]; const parentNote = notes[parentNoteId]; @@ -868,17 +861,17 @@ function getNoteTitle(childNoteId, parentNoteId) { return ((branch && branch.prefix) ? `${branch.prefix} - ` : '') + title; } -function getNoteTitleArrayForPath(path) { +function getNoteTitleArrayForPath(notePathArray) { const titles = []; - if (path[0] === hoistedNoteService.getHoistedNoteId() && path.length === 1) { + if (notePathArray[0] === hoistedNoteService.getHoistedNoteId() && notePathArray.length === 1) { return [ getNoteTitle(hoistedNoteService.getHoistedNoteId()) ]; } let parentNoteId = 'root'; let hoistedNotePassed = false; - for (const noteId of path) { + for (const noteId of notePathArray) { // start collecting path segment titles only after hoisted note if (hoistedNotePassed) { const title = getNoteTitle(noteId, parentNoteId); @@ -896,8 +889,8 @@ function getNoteTitleArrayForPath(path) { return titles; } -function getNoteTitleForPath(path) { - const titles = getNoteTitleArrayForPath(path); +function getNoteTitleForPath(notePathArray) { + const titles = getNoteTitleArrayForPath(notePathArray); return titles.join(' / '); } @@ -1153,10 +1146,9 @@ sqlInit.dbReady.then(() => utils.stopWatch("Note cache load", load)); module.exports = { loadedPromise, - findNotesWithFulltext, + findNotesForAutocomplete, getNotePath, getNoteTitleForPath, - getNoteTitleFromPath, isAvailable, isArchived, isInAncestor, From dcd371b5b15c17c2472967207a034f9aed74d187 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 16 May 2020 23:12:29 +0200 Subject: [PATCH 15/47] note cache breakup into classes, WIP --- src/routes/api/autocomplete.js | 2 +- src/routes/api/import.js | 4 +- src/routes/api/note_revisions.js | 2 +- src/routes/api/recent_changes.js | 4 +- src/routes/api/search.js | 4 +- src/routes/api/similar_notes.js | 2 +- src/services/note_cache.js | 1157 ----------------- src/services/note_cache/entities/attribute.js | 43 + src/services/note_cache/entities/branch.js | 42 + src/services/note_cache/entities/note.js | 236 ++++ src/services/note_cache/expressions/and.js | 13 + src/services/note_cache/expressions/equals.js | 28 + src/services/note_cache/expressions/exists.js | 27 + .../expressions/note_cache_fulltext.js | 125 ++ .../expressions/note_content_fulltext.js | 26 + src/services/note_cache/expressions/or.js | 15 + src/services/note_cache/note_cache.js | 224 ++++ src/services/note_cache/note_cache_service.js | 233 ++++ src/services/note_cache/note_set.js | 22 + src/services/note_cache/search.js | 113 ++ src/services/note_cache/search_result.js | 14 + src/services/search.js | 4 +- src/services/tree.js | 4 +- 23 files changed, 1174 insertions(+), 1170 deletions(-) delete mode 100644 src/services/note_cache.js create mode 100644 src/services/note_cache/entities/attribute.js create mode 100644 src/services/note_cache/entities/branch.js create mode 100644 src/services/note_cache/entities/note.js create mode 100644 src/services/note_cache/expressions/and.js create mode 100644 src/services/note_cache/expressions/equals.js create mode 100644 src/services/note_cache/expressions/exists.js create mode 100644 src/services/note_cache/expressions/note_cache_fulltext.js create mode 100644 src/services/note_cache/expressions/note_content_fulltext.js create mode 100644 src/services/note_cache/expressions/or.js create mode 100644 src/services/note_cache/note_cache.js create mode 100644 src/services/note_cache/note_cache_service.js create mode 100644 src/services/note_cache/note_set.js create mode 100644 src/services/note_cache/search.js create mode 100644 src/services/note_cache/search_result.js diff --git a/src/routes/api/autocomplete.js b/src/routes/api/autocomplete.js index f5c416eb0..ef0b13724 100644 --- a/src/routes/api/autocomplete.js +++ b/src/routes/api/autocomplete.js @@ -1,6 +1,6 @@ "use strict"; -const noteCacheService = require('../../services/note_cache'); +const noteCacheService = require('../../services/note_cache/note_cache.js'); const repository = require('../../services/repository'); const log = require('../../services/log'); const utils = require('../../services/utils'); diff --git a/src/routes/api/import.js b/src/routes/api/import.js index 7237450d3..f0529e888 100644 --- a/src/routes/api/import.js +++ b/src/routes/api/import.js @@ -8,7 +8,7 @@ const zipImportService = require('../../services/import/zip'); const singleImportService = require('../../services/import/single'); const cls = require('../../services/cls'); const path = require('path'); -const noteCacheService = require('../../services/note_cache'); +const noteCacheService = require('../../services/note_cache/note_cache.js'); const log = require('../../services/log'); const TaskContext = require('../../services/task_context.js'); @@ -85,4 +85,4 @@ async function importToBranch(req) { module.exports = { importToBranch -}; \ No newline at end of file +}; diff --git a/src/routes/api/note_revisions.js b/src/routes/api/note_revisions.js index 7f325f725..dff91b4e6 100644 --- a/src/routes/api/note_revisions.js +++ b/src/routes/api/note_revisions.js @@ -1,7 +1,7 @@ "use strict"; const repository = require('../../services/repository'); -const noteCacheService = require('../../services/note_cache'); +const noteCacheService = require('../../services/note_cache/note_cache.js'); const protectedSessionService = require('../../services/protected_session'); const noteRevisionService = require('../../services/note_revisions'); const utils = require('../../services/utils'); diff --git a/src/routes/api/recent_changes.js b/src/routes/api/recent_changes.js index a4049072a..536e8a861 100644 --- a/src/routes/api/recent_changes.js +++ b/src/routes/api/recent_changes.js @@ -3,7 +3,7 @@ const sql = require('../../services/sql'); const protectedSessionService = require('../../services/protected_session'); const noteService = require('../../services/notes'); -const noteCacheService = require('../../services/note_cache'); +const noteCacheService = require('../../services/note_cache/note_cache.js'); async function getRecentChanges(req) { const {ancestorNoteId} = req.params; @@ -102,4 +102,4 @@ async function getRecentChanges(req) { module.exports = { getRecentChanges -}; \ No newline at end of file +}; diff --git a/src/routes/api/search.js b/src/routes/api/search.js index b88989fb2..a5596830d 100644 --- a/src/routes/api/search.js +++ b/src/routes/api/search.js @@ -1,7 +1,7 @@ "use strict"; const repository = require('../../services/repository'); -const noteCacheService = require('../../services/note_cache'); +const noteCacheService = require('../../services/note_cache/note_cache.js'); const log = require('../../services/log'); const scriptService = require('../../services/script'); const searchService = require('../../services/search'); @@ -110,4 +110,4 @@ async function searchFromRelation(note, relationName) { module.exports = { searchNotes, searchFromNote -}; \ No newline at end of file +}; diff --git a/src/routes/api/similar_notes.js b/src/routes/api/similar_notes.js index 4d52e94fe..22ddf427a 100644 --- a/src/routes/api/similar_notes.js +++ b/src/routes/api/similar_notes.js @@ -1,6 +1,6 @@ "use strict"; -const noteCacheService = require('../../services/note_cache'); +const noteCacheService = require('../../services/note_cache/note_cache.js'); const repository = require('../../services/repository'); async function getSimilarNotes(req) { diff --git a/src/services/note_cache.js b/src/services/note_cache.js deleted file mode 100644 index b35c449e8..000000000 --- a/src/services/note_cache.js +++ /dev/null @@ -1,1157 +0,0 @@ -const sql = require('./sql'); -const sqlInit = require('./sql_init'); -const eventService = require('./events'); -const protectedSessionService = require('./protected_session'); -const utils = require('./utils'); -const hoistedNoteService = require('./hoisted_note'); -const stringSimilarity = require('string-similarity'); - -/** @type {Object.} */ -let notes; -/** @type {Object.} */ -let branches -/** @type {Object.} */ -let attributes; -/** @type {Object.} Points from attribute type-name to list of attributes them */ -let attributeIndex; - -/** @return {Attribute[]} */ -function findAttributes(type, name) { - return attributeIndex[`${type}-${name}`] || []; -} - -let childParentToBranch = {}; - -class Note { - constructor(row) { - /** @param {string} */ - this.noteId = row.noteId; - /** @param {string} */ - this.title = row.title; - /** @param {boolean} */ - this.isProtected = !!row.isProtected; - /** @param {boolean} */ - this.isDecrypted = !row.isProtected || !!row.isContentAvailable; - /** @param {Branch[]} */ - this.parentBranches = []; - /** @param {Note[]} */ - this.parents = []; - /** @param {Note[]} */ - this.children = []; - /** @param {Attribute[]} */ - this.ownedAttributes = []; - - /** @param {Attribute[]|null} */ - this.attributeCache = null; - /** @param {Attribute[]|null} */ - this.inheritableAttributeCache = null; - - /** @param {Attribute[]} */ - this.targetRelations = []; - - /** @param {string|null} */ - this.flatTextCache = null; - - if (protectedSessionService.isProtectedSessionAvailable()) { - decryptProtectedNote(this); - } - } - - /** @return {Attribute[]} */ - get attributes() { - if (!this.attributeCache) { - const parentAttributes = this.ownedAttributes.slice(); - - if (this.noteId !== 'root') { - for (const parentNote of this.parents) { - parentAttributes.push(...parentNote.inheritableAttributes); - } - } - - const templateAttributes = []; - - for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates - if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') { - const templateNote = notes[ownedAttr.value]; - - if (templateNote) { - templateAttributes.push(...templateNote.attributes); - } - } - } - - this.attributeCache = parentAttributes.concat(templateAttributes); - this.inheritableAttributeCache = []; - - for (const attr of this.attributeCache) { - if (attr.isInheritable) { - this.inheritableAttributeCache.push(attr); - } - } - } - - return this.attributeCache; - } - - /** @return {Attribute[]} */ - get inheritableAttributes() { - if (!this.inheritableAttributeCache) { - this.attributes; // will refresh also this.inheritableAttributeCache - } - - return this.inheritableAttributeCache; - } - - hasAttribute(type, name) { - return this.attributes.find(attr => attr.type === type && attr.name === name); - } - - get isArchived() { - return this.hasAttribute('label', 'archived'); - } - - get isHideInAutocompleteOrArchived() { - return this.attributes.find(attr => - attr.type === 'label' - && ["archived", "hideInAutocomplete"].includes(attr.name)); - } - - get hasInheritableOwnedArchivedLabel() { - return !!this.ownedAttributes.find(attr => attr.type === 'label' && attr.name === 'archived' && attr.isInheritable); - } - - // will sort the parents so that non-archived are first and archived at the end - // this is done so that non-archived paths are always explored as first when searching for note path - resortParents() { - this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); - } - - /** - * @return {string} - returns flattened textual representation of note, prefixes and attributes usable for searching - */ - get flatText() { - if (!this.flatTextCache) { - if (this.isHideInAutocompleteOrArchived) { - this.flatTextCache = " "; // can't be empty - return this.flatTextCache; - } - - this.flatTextCache = ''; - - for (const branch of this.parentBranches) { - if (branch.prefix) { - this.flatTextCache += branch.prefix + ' - '; - } - } - - this.flatTextCache += this.title; - - for (const attr of this.attributes) { - // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words - this.flatTextCache += (attr.type === 'label' ? '#' : '@') + attr.name; - - if (attr.value) { - this.flatTextCache += '=' + attr.value; - } - } - - this.flatTextCache = this.flatTextCache.toLowerCase(); - } - - return this.flatTextCache; - } - - invalidateThisCache() { - this.flatTextCache = null; - - this.attributeCache = null; - this.inheritableAttributeCache = null; - } - - invalidateSubtreeCaches() { - this.invalidateThisCache(); - - for (const childNote of this.children) { - childNote.invalidateSubtreeCaches(); - } - - for (const targetRelation of this.targetRelations) { - if (targetRelation.name === 'template') { - const note = targetRelation.note; - - if (note) { - note.invalidateSubtreeCaches(); - } - } - } - } - - invalidateSubtreeFlatText() { - this.flatTextCache = null; - - for (const childNote of this.children) { - childNote.invalidateSubtreeFlatText(); - } - - for (const targetRelation of this.targetRelations) { - if (targetRelation.name === 'template') { - const note = targetRelation.note; - - if (note) { - note.invalidateSubtreeFlatText(); - } - } - } - } - - get isTemplate() { - return !!this.targetRelations.find(rel => rel.name === 'template'); - } - - /** @return {Note[]} */ - get subtreeNotesIncludingTemplated() { - const arr = [[this]]; - - for (const childNote of this.children) { - arr.push(childNote.subtreeNotesIncludingTemplated); - } - - for (const targetRelation of this.targetRelations) { - if (targetRelation.name === 'template') { - const note = targetRelation.note; - - if (note) { - arr.push(note.subtreeNotesIncludingTemplated); - } - } - } - - return arr.flat(); - } - - /** @return {Note[]} */ - get subtreeNotes() { - const arr = [[this]]; - - for (const childNote of this.children) { - arr.push(childNote.subtreeNotes); - } - - return arr.flat(); - } - - /** @return {Note[]} - returns only notes which are templated, does not include their subtrees - * in effect returns notes which are influenced by note's non-inheritable attributes */ - get templatedNotes() { - const arr = [this]; - - for (const targetRelation of this.targetRelations) { - if (targetRelation.name === 'template') { - const note = targetRelation.note; - - if (note) { - arr.push(note); - } - } - } - - return arr; - } -} - -class Branch { - constructor(row) { - /** @param {string} */ - this.branchId = row.branchId; - /** @param {string} */ - this.noteId = row.noteId; - /** @param {string} */ - this.parentNoteId = row.parentNoteId; - /** @param {string} */ - this.prefix = row.prefix; - - if (this.branchId === 'root') { - return; - } - - const childNote = notes[this.noteId]; - const parentNote = this.parentNote; - - if (!childNote) { - console.log(`Cannot find child note ${this.noteId} of a branch ${this.branchId}`); - return; - } - - childNote.parents.push(parentNote); - childNote.parentBranches.push(this); - - parentNote.children.push(childNote); - - childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; - } - - /** @return {Note} */ - get parentNote() { - const note = notes[this.parentNoteId]; - - if (!note) { - console.log(`Cannot find note ${this.parentNoteId}`); - } - - return note; - } -} - -class Attribute { - constructor(row) { - /** @param {string} */ - this.attributeId = row.attributeId; - /** @param {string} */ - this.noteId = row.noteId; - /** @param {string} */ - this.type = row.type; - /** @param {string} */ - this.name = row.name; - /** @param {string} */ - this.value = row.value; - /** @param {boolean} */ - this.isInheritable = !!row.isInheritable; - - notes[this.noteId].ownedAttributes.push(this); - - const key = `${this.type-this.name}`; - attributeIndex[key] = attributeIndex[key] || []; - attributeIndex[key].push(this); - - const targetNote = this.targetNote; - - if (targetNote) { - targetNote.targetRelations.push(this); - } - } - - get isAffectingSubtree() { - return this.isInheritable - || (this.type === 'relation' && this.name === 'template'); - } - - get note() { - return notes[this.noteId]; - } - - get targetNote() { - if (this.type === 'relation') { - return notes[this.value]; - } - } -} - -let loaded = false; -let loadedPromiseResolve; -/** Is resolved after the initial load */ -let loadedPromise = new Promise(res => loadedPromiseResolve = res); - -async function getMappedRows(query, cb) { - const map = {}; - const results = await sql.getRows(query, []); - - for (const row of results) { - const keys = Object.keys(row); - - map[row[keys[0]]] = cb(row); - } - - return map; -} - -async function load() { - notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, - row => new Note(row)); - - branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, - row => new Branch(row)); - - attributeIndex = []; - - attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, - row => new Attribute(row)); - - loaded = true; - loadedPromiseResolve(); -} - -class AndExp { - constructor(subExpressions) { - this.subExpressions = subExpressions; - } - - execute(noteSet, searchContext) { - for (const subExpression of this.subExpressions) { - noteSet = subExpression.execute(noteSet, searchContext); - } - - return noteSet; - } -} - -class OrExp { - constructor(subExpressions) { - this.subExpressions = subExpressions; - } - - execute(noteSet, searchContext) { - const resultNoteSet = new NoteSet(); - - for (const subExpression of this.subExpressions) { - resultNoteSet.mergeIn(subExpression.execute(noteSet, searchContext)); - } - - return resultNoteSet; - } -} - -class NoteSet { - constructor(notes = []) { - this.notes = notes; - } - - add(note) { - this.notes.push(note); - } - - addAll(notes) { - this.notes.push(...notes); - } - - hasNoteId(noteId) { - // TODO: optimize - return !!this.notes.find(note => note.noteId === noteId); - } - - mergeIn(anotherNoteSet) { - this.notes = this.notes.concat(anotherNoteSet.arr); - } -} - -class ExistsExp { - constructor(attributeType, attributeName) { - this.attributeType = attributeType; - this.attributeName = attributeName; - } - - execute(noteSet) { - const attrs = findAttributes(this.attributeType, this.attributeName); - const resultNoteSet = new NoteSet(); - - for (const attr of attrs) { - const note = attr.note; - - if (noteSet.hasNoteId(note.noteId)) { - if (attr.isInheritable) { - resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); - } - else if (note.isTemplate) { - resultNoteSet.addAll(note.templatedNotes); - } - else { - resultNoteSet.add(note); - } - } - } - } -} - -class EqualsExp { - constructor(attributeType, attributeName, attributeValue) { - this.attributeType = attributeType; - this.attributeName = attributeName; - this.attributeValue = attributeValue; - } - - execute(noteSet) { - const attrs = findAttributes(this.attributeType, this.attributeName); - const resultNoteSet = new NoteSet(); - - for (const attr of attrs) { - const note = attr.note; - - if (noteSet.hasNoteId(note.noteId) && attr.value === this.attributeValue) { - if (attr.isInheritable) { - resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); - } - else if (note.isTemplate) { - resultNoteSet.addAll(note.templatedNotes); - } - else { - resultNoteSet.add(note); - } - } - } - } -} - -class NoteContentFulltextExp { - constructor(tokens) { - this.tokens = tokens; - } - - async execute(noteSet) { - const resultNoteSet = new NoteSet(); - const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); - - const noteIds = await sql.getColumn(` - SELECT notes.noteId - FROM notes - JOIN note_contents ON notes.noteId = note_contents.noteId - WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); - - const results = []; - - for (const noteId of noteIds) { - if (noteSet.hasNoteId(noteId) && noteId in notes) { - resultNoteSet.add(notes[noteId]); - } - } - - return results; - } -} - -class NoteCacheFulltextExp { - constructor(tokens) { - this.tokens = tokens; - } - - execute(noteSet, searchContext) { - const resultNoteSet = new NoteSet(); - - const candidateNotes = this.getCandidateNotes(noteSet); - - for (const note of candidateNotes) { - // autocomplete should be able to find notes by their noteIds as well (only leafs) - if (this.tokens.length === 1 && note.noteId === this.tokens[0]) { - this.searchDownThePath(note, [], [], resultNoteSet, searchContext); - continue; - } - - // for leaf note it doesn't matter if "archived" label is inheritable or not - if (note.isArchived) { - continue; - } - - const foundAttrTokens = []; - - for (const attribute of note.ownedAttributes) { - for (const token of this.tokens) { - if (attribute.name.toLowerCase().includes(token) - || attribute.value.toLowerCase().includes(token)) { - foundAttrTokens.push(token); - } - } - } - - for (const parentNote of note.parents) { - const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); - const foundTokens = foundAttrTokens.slice(); - - for (const token of this.tokens) { - if (title.includes(token)) { - foundTokens.push(token); - } - } - - if (foundTokens.length > 0) { - const remainingTokens = this.tokens.filter(token => !foundTokens.includes(token)); - - this.searchDownThePath(parentNote, remainingTokens, [note.noteId], resultNoteSet, searchContext); - } - } - } - - return resultNoteSet; - } - - /** - * Returns noteIds which have at least one matching tokens - * - * @param {NoteSet} noteSet - * @return {String[]} - */ - getCandidateNotes(noteSet) { - const candidateNotes = []; - - for (const note of noteSet.notes) { - for (const token of this.tokens) { - if (note.flatText.includes(token)) { - candidateNotes.push(note); - break; - } - } - } - - return candidateNotes; - } - - searchDownThePath(note, tokens, path, resultNoteSet, searchContext) { - if (tokens.length === 0) { - const retPath = getSomePath(note, path); - - if (retPath) { - const noteId = retPath[retPath.length - 1]; - searchContext.noteIdToNotePath[noteId] = retPath; - - resultNoteSet.add(notes[noteId]); - } - - return; - } - - if (!note.parents.length === 0 || note.noteId === 'root') { - return; - } - - const foundAttrTokens = []; - - for (const attribute of note.ownedAttributes) { - for (const token of tokens) { - if (attribute.name.toLowerCase().includes(token) - || attribute.value.toLowerCase().includes(token)) { - foundAttrTokens.push(token); - } - } - } - - for (const parentNote of note.parents) { - const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); - const foundTokens = foundAttrTokens.slice(); - - for (const token of tokens) { - if (title.includes(token)) { - foundTokens.push(token); - } - } - - if (foundTokens.length > 0) { - const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - - this.searchDownThePath(parentNote, remainingTokens, path.concat([note.noteId]), resultNoteSet, searchContext); - } - else { - this.searchDownThePath(parentNote, tokens, path.concat([note.noteId]), resultNoteSet, searchContext); - } - } - } -} - -class SearchResult { - constructor(notePathArray) { - this.notePathArray = notePathArray; - this.notePathTitle = getNoteTitleForPath(notePathArray); - } - - get notePath() { - return this.notePathArray.join('/'); - } - - get noteId() { - return this.notePathArray[this.notePathArray.length - 1]; - } -} - -async function findNotesWithExpression(expression) { - - const hoistedNote = notes[hoistedNoteService.getHoistedNoteId()]; - const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') - ? hoistedNote.subtreeNotes - : Object.values(notes); - - const allNoteSet = new NoteSet(allNotes); - - const searchContext = { - noteIdToNotePath: {} - }; - - const noteSet = await expression.execute(allNoteSet, searchContext); - - let searchResults = noteSet.notes - .map(note => searchContext.noteIdToNotePath[note.noteId] || getSomePath(note)) - .filter(notePathArray => notePathArray.includes(hoistedNoteService.getHoistedNoteId())) - .map(notePathArray => new SearchResult(notePathArray)); - - // sort results by depth of the note. This is based on the assumption that more important results - // are closer to the note root. - searchResults.sort((a, b) => { - if (a.notePathArray.length === b.notePathArray.length) { - return a.notePathTitle < b.notePathTitle ? -1 : 1; - } - - return a.notePathArray.length < b.notePathArray.length ? -1 : 1; - }); - - return searchResults; -} - -async function findNotesForAutocomplete(query) { - if (!query.trim().length) { - return []; - } - - const tokens = query - .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc - .toLowerCase() - .split(/[ -]/) - .filter(token => token !== '/'); // '/' is used as separator - - const expression = new NoteCacheFulltextExp(tokens); - - let searchResults = await findNotesWithExpression(expression); - - searchResults = searchResults.slice(0, 200); - - highlightSearchResults(searchResults, tokens); - - return searchResults.map(result => { - return { - notePath: result.notePath, - notePathTitle: result.notePathTitle, - highlightedNotePathTitle: result.highlightedNotePathTitle - } - }); -} - -function highlightSearchResults(searchResults, tokens) { - // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks - // which would make the resulting HTML string invalid. - // { and } are used for marking and tag (to avoid matches on single 'b' character) - tokens = tokens.map(token => token.replace('/[<\{\}]/g', '')); - - // sort by the longest so we first highlight longest matches - tokens.sort((a, b) => a.length > b.length ? -1 : 1); - - for (const result of searchResults) { - const note = notes[result.noteId]; - - result.highlightedNotePathTitle = result.notePathTitle; - - for (const attr of note.attributes) { - if (tokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { - result.highlightedNotePathTitle += ` ${formatAttribute(attr)}`; - } - } - } - - for (const token of tokens) { - const tokenRegex = new RegExp("(" + utils.escapeRegExp(token) + ")", "gi"); - - for (const result of searchResults) { - result.highlightedNotePathTitle = result.highlightedNotePathTitle.replace(tokenRegex, "{$1}"); - } - } - - for (const result of searchResults) { - result.highlightedNotePathTitle = result.highlightedNotePathTitle - .replace(/{/g, "") - .replace(/}/g, ""); - } -} - -function decryptProtectedNote(note) { - if (note.isProtected && !note.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { - note.title = protectedSessionService.decryptString(note.title); - - note.isDecrypted = true; - } -} - -async function decryptProtectedNotes() { - for (const note of Object.values(notes)) { - decryptProtectedNote(note); - } -} - -function formatAttribute(attr) { - if (attr.type === 'relation') { - return '@' + utils.escapeHtml(attr.name) + "=…"; - } - else if (attr.type === 'label') { - let label = '#' + utils.escapeHtml(attr.name); - - if (attr.value) { - const val = /[^\w_-]/.test(attr.value) ? '"' + attr.value + '"' : attr.value; - - label += '=' + utils.escapeHtml(val); - } - - return label; - } -} - -function getBranch(childNoteId, parentNoteId) { - return childParentToBranch[`${childNoteId}-${parentNoteId}`]; -} - -function isNotePathArchived(notePath) { - const noteId = notePath[notePath.length - 1]; - const note = notes[noteId]; - - if (note.isArchived) { - return true; - } - - for (let i = 0; i < notePath.length - 1; i++) { - const note = notes[notePath[i]]; - - // this is going through parents so archived must be inheritable - if (note.hasInheritableOwnedArchivedLabel) { - return true; - } - } - - return false; -} - -/** - * This assumes that note is available. "archived" note means that there isn't a single non-archived note-path - * leading to this note. - * - * @param noteId - */ -function isArchived(noteId) { - const notePath = getSomePath(noteId); - - return isNotePathArchived(notePath); -} - -/** - * @param {string} noteId - * @param {string} ancestorNoteId - * @return {boolean} - true if given noteId has ancestorNoteId in any of its paths (even archived) - */ -function isInAncestor(noteId, ancestorNoteId) { - if (ancestorNoteId === 'root' || ancestorNoteId === noteId) { - return true; - } - - const note = notes[noteId]; - - for (const parentNote of note.parents) { - if (isInAncestor(parentNote.noteId, ancestorNoteId)) { - return true; - } - } - - return false; -} - -function getNoteTitle(childNoteId, parentNoteId) { - const childNote = notes[childNoteId]; - const parentNote = notes[parentNoteId]; - - let title; - - if (childNote.isProtected) { - title = protectedSessionService.isProtectedSessionAvailable() ? childNote.title : '[protected]'; - } - else { - title = childNote.title; - } - - const branch = parentNote ? getBranch(childNote.noteId, parentNote.noteId) : null; - - return ((branch && branch.prefix) ? `${branch.prefix} - ` : '') + title; -} - -function getNoteTitleArrayForPath(notePathArray) { - const titles = []; - - if (notePathArray[0] === hoistedNoteService.getHoistedNoteId() && notePathArray.length === 1) { - return [ getNoteTitle(hoistedNoteService.getHoistedNoteId()) ]; - } - - let parentNoteId = 'root'; - let hoistedNotePassed = false; - - for (const noteId of notePathArray) { - // start collecting path segment titles only after hoisted note - if (hoistedNotePassed) { - const title = getNoteTitle(noteId, parentNoteId); - - titles.push(title); - } - - if (noteId === hoistedNoteService.getHoistedNoteId()) { - hoistedNotePassed = true; - } - - parentNoteId = noteId; - } - - return titles; -} - -function getNoteTitleForPath(notePathArray) { - const titles = getNoteTitleArrayForPath(notePathArray); - - return titles.join(' / '); -} - -/** - * Returns notePath for noteId from cache. Note hoisting is respected. - * Archived notes are also returned, but non-archived paths are preferred if available - * - this means that archived paths is returned only if there's no non-archived path - * - you can check whether returned path is archived using isArchived() - */ -function getSomePath(note, path = []) { - if (note.noteId === 'root') { - path.push(note.noteId); - path.reverse(); - - if (!path.includes(hoistedNoteService.getHoistedNoteId())) { - return false; - } - - return path; - } - - const parents = note.parents; - if (parents.length === 0) { - return false; - } - - for (const parentNote of parents) { - const retPath = getSomePath(parentNote, path.concat([note.noteId])); - - if (retPath) { - return retPath; - } - } - - return false; -} - -function getNotePath(noteId) { - const note = notes[noteId]; - const retPath = getSomePath(note); - - if (retPath) { - const noteTitle = getNoteTitleForPath(retPath); - const parentNote = note.parents[0]; - - return { - noteId: noteId, - branchId: getBranch(noteId, parentNote.noteId).branchId, - title: noteTitle, - notePath: retPath, - path: retPath.join('/') - }; - } -} - -function evaluateSimilarity(sourceNote, candidateNote, results) { - let coeff = stringSimilarity.compareTwoStrings(sourceNote.flatText, candidateNote.flatText); - - if (coeff > 0.4) { - const notePath = getSomePath(candidateNote); - - // this takes care of note hoisting - if (!notePath) { - return; - } - - if (isNotePathArchived(notePath)) { - coeff -= 0.2; // archived penalization - } - - results.push({coeff, notePath, noteId: candidateNote.noteId}); - } -} - -/** - * Point of this is to break up long running sync process to avoid blocking - * see https://snyk.io/blog/nodejs-how-even-quick-async-functions-can-block-the-event-loop-starve-io/ - */ -function setImmediatePromise() { - return new Promise((resolve) => { - setTimeout(() => resolve(), 0); - }); -} - -async function findSimilarNotes(noteId) { - const results = []; - let i = 0; - - const origNote = notes[noteId]; - - if (!origNote) { - return []; - } - - for (const note of Object.values(notes)) { - if (note.isProtected && !note.isDecrypted) { - continue; - } - - evaluateSimilarity(origNote, note, results); - - i++; - - if (i % 200 === 0) { - await setImmediatePromise(); - } - } - - results.sort((a, b) => a.coeff > b.coeff ? -1 : 1); - - return results.length > 50 ? results.slice(0, 50) : results; -} - -eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { - // note that entity can also be just POJO without methods if coming from sync - - if (!loaded) { - return; - } - - if (entityName === 'notes') { - const {noteId} = entity; - - if (entity.isDeleted) { - delete notes[noteId]; - } - else if (noteId in notes) { - const note = notes[noteId]; - - // we can assume we have protected session since we managed to update - note.title = entity.title; - note.isProtected = entity.isProtected; - note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; - note.flatTextCache = null; - - decryptProtectedNote(note); - } - else { - const note = new Note(entity); - notes[noteId] = note; - - decryptProtectedNote(note); - } - } - else if (entityName === 'branches') { - const {branchId, noteId, parentNoteId} = entity; - const childNote = notes[noteId]; - - if (entity.isDeleted) { - if (childNote) { - childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); - childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); - - if (childNote.parents.length > 0) { - childNote.invalidateSubtreeCaches(); - } - } - - const parentNote = notes[parentNoteId]; - - if (parentNote) { - parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); - } - - delete childParentToBranch[`${noteId}-${parentNoteId}`]; - delete branches[branchId]; - } - else if (branchId in branches) { - // only relevant thing which can change in a branch is prefix - branches[branchId].prefix = entity.prefix; - - if (childNote) { - childNote.flatTextCache = null; - } - } - else { - branches[branchId] = new Branch(entity); - - if (childNote) { - childNote.resortParents(); - } - } - } - else if (entityName === 'attributes') { - const {attributeId, noteId} = entity; - const note = notes[noteId]; - const attr = attributes[attributeId]; - - if (entity.isDeleted) { - if (note && attr) { - // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeCaches(); - } - - note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); - - const targetNote = attr.targetNote; - - if (targetNote) { - targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); - } - } - - delete attributes[attributeId]; - delete attributeIndex[`${attr.type}-${attr.name}`]; - } - else if (attributeId in attributes) { - const attr = attributes[attributeId]; - - // attr name and isInheritable are immutable - attr.value = entity.value; - - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeFlatText(); - } - else { - note.flatTextCache = null; - } - } - else { - const attr = new Attribute(entity); - attributes[attributeId] = attr; - - if (note) { - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeCaches(); - } - else { - this.invalidateThisCache(); - } - } - } - } -}); - -/** - * @param noteId - * @returns {boolean} - true if note exists (is not deleted) and is available in current note hoisting - */ -function isAvailable(noteId) { - const notePath = getNotePath(noteId); - - return !!notePath; -} - -eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { - loadedPromise.then(() => decryptProtectedNotes()); -}); - -sqlInit.dbReady.then(() => utils.stopWatch("Note cache load", load)); - -module.exports = { - loadedPromise, - findNotesForAutocomplete, - getNotePath, - getNoteTitleForPath, - isAvailable, - isArchived, - isInAncestor, - load, - findSimilarNotes -}; diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js new file mode 100644 index 000000000..a276d47d6 --- /dev/null +++ b/src/services/note_cache/entities/attribute.js @@ -0,0 +1,43 @@ +class Attribute { + constructor(row) { + /** @param {string} */ + this.attributeId = row.attributeId; + /** @param {string} */ + this.noteId = row.noteId; + /** @param {string} */ + this.type = row.type; + /** @param {string} */ + this.name = row.name; + /** @param {string} */ + this.value = row.value; + /** @param {boolean} */ + this.isInheritable = !!row.isInheritable; + + notes[this.noteId].ownedAttributes.push(this); + + const key = `${this.type-this.name}`; + attributeIndex[key] = attributeIndex[key] || []; + attributeIndex[key].push(this); + + const targetNote = this.targetNote; + + if (targetNote) { + targetNote.targetRelations.push(this); + } + } + + get isAffectingSubtree() { + return this.isInheritable + || (this.type === 'relation' && this.name === 'template'); + } + + get note() { + return notes[this.noteId]; + } + + get targetNote() { + if (this.type === 'relation') { + return notes[this.value]; + } + } +} diff --git a/src/services/note_cache/entities/branch.js b/src/services/note_cache/entities/branch.js new file mode 100644 index 000000000..aaf075aa8 --- /dev/null +++ b/src/services/note_cache/entities/branch.js @@ -0,0 +1,42 @@ +export default class Branch { + constructor(row) { + /** @param {string} */ + this.branchId = row.branchId; + /** @param {string} */ + this.noteId = row.noteId; + /** @param {string} */ + this.parentNoteId = row.parentNoteId; + /** @param {string} */ + this.prefix = row.prefix; + + if (this.branchId === 'root') { + return; + } + + const childNote = notes[this.noteId]; + const parentNote = this.parentNote; + + if (!childNote) { + console.log(`Cannot find child note ${this.noteId} of a branch ${this.branchId}`); + return; + } + + childNote.parents.push(parentNote); + childNote.parentBranches.push(this); + + parentNote.children.push(childNote); + + childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; + } + + /** @return {Note} */ + get parentNote() { + const note = notes[this.parentNoteId]; + + if (!note) { + console.log(`Cannot find note ${this.parentNoteId}`); + } + + return note; + } +} diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js new file mode 100644 index 000000000..7f5dfe924 --- /dev/null +++ b/src/services/note_cache/entities/note.js @@ -0,0 +1,236 @@ +export default class Note { + constructor(row) { + /** @param {string} */ + this.noteId = row.noteId; + /** @param {string} */ + this.title = row.title; + /** @param {boolean} */ + this.isProtected = !!row.isProtected; + /** @param {boolean} */ + this.isDecrypted = !row.isProtected || !!row.isContentAvailable; + /** @param {Branch[]} */ + this.parentBranches = []; + /** @param {Note[]} */ + this.parents = []; + /** @param {Note[]} */ + this.children = []; + /** @param {Attribute[]} */ + this.ownedAttributes = []; + + /** @param {Attribute[]|null} */ + this.attributeCache = null; + /** @param {Attribute[]|null} */ + this.inheritableAttributeCache = null; + + /** @param {Attribute[]} */ + this.targetRelations = []; + + /** @param {string|null} */ + this.flatTextCache = null; + + if (protectedSessionService.isProtectedSessionAvailable()) { + decryptProtectedNote(this); + } + } + + /** @return {Attribute[]} */ + get attributes() { + if (!this.attributeCache) { + const parentAttributes = this.ownedAttributes.slice(); + + if (this.noteId !== 'root') { + for (const parentNote of this.parents) { + parentAttributes.push(...parentNote.inheritableAttributes); + } + } + + const templateAttributes = []; + + for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates + if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') { + const templateNote = notes[ownedAttr.value]; + + if (templateNote) { + templateAttributes.push(...templateNote.attributes); + } + } + } + + this.attributeCache = parentAttributes.concat(templateAttributes); + this.inheritableAttributeCache = []; + + for (const attr of this.attributeCache) { + if (attr.isInheritable) { + this.inheritableAttributeCache.push(attr); + } + } + } + + return this.attributeCache; + } + + /** @return {Attribute[]} */ + get inheritableAttributes() { + if (!this.inheritableAttributeCache) { + this.attributes; // will refresh also this.inheritableAttributeCache + } + + return this.inheritableAttributeCache; + } + + hasAttribute(type, name) { + return this.attributes.find(attr => attr.type === type && attr.name === name); + } + + get isArchived() { + return this.hasAttribute('label', 'archived'); + } + + get isHideInAutocompleteOrArchived() { + return this.attributes.find(attr => + attr.type === 'label' + && ["archived", "hideInAutocomplete"].includes(attr.name)); + } + + get hasInheritableOwnedArchivedLabel() { + return !!this.ownedAttributes.find(attr => attr.type === 'label' && attr.name === 'archived' && attr.isInheritable); + } + + // will sort the parents so that non-archived are first and archived at the end + // this is done so that non-archived paths are always explored as first when searching for note path + resortParents() { + this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1); + } + + /** + * @return {string} - returns flattened textual representation of note, prefixes and attributes usable for searching + */ + get flatText() { + if (!this.flatTextCache) { + if (this.isHideInAutocompleteOrArchived) { + this.flatTextCache = " "; // can't be empty + return this.flatTextCache; + } + + this.flatTextCache = ''; + + for (const branch of this.parentBranches) { + if (branch.prefix) { + this.flatTextCache += branch.prefix + ' - '; + } + } + + this.flatTextCache += this.title; + + for (const attr of this.attributes) { + // it's best to use space as separator since spaces are filtered from the search string by the tokenization into words + this.flatTextCache += (attr.type === 'label' ? '#' : '@') + attr.name; + + if (attr.value) { + this.flatTextCache += '=' + attr.value; + } + } + + this.flatTextCache = this.flatTextCache.toLowerCase(); + } + + return this.flatTextCache; + } + + invalidateThisCache() { + this.flatTextCache = null; + + this.attributeCache = null; + this.inheritableAttributeCache = null; + } + + invalidateSubtreeCaches() { + this.invalidateThisCache(); + + for (const childNote of this.children) { + childNote.invalidateSubtreeCaches(); + } + + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; + + if (note) { + note.invalidateSubtreeCaches(); + } + } + } + } + + invalidateSubtreeFlatText() { + this.flatTextCache = null; + + for (const childNote of this.children) { + childNote.invalidateSubtreeFlatText(); + } + + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; + + if (note) { + note.invalidateSubtreeFlatText(); + } + } + } + } + + get isTemplate() { + return !!this.targetRelations.find(rel => rel.name === 'template'); + } + + /** @return {Note[]} */ + get subtreeNotesIncludingTemplated() { + const arr = [[this]]; + + for (const childNote of this.children) { + arr.push(childNote.subtreeNotesIncludingTemplated); + } + + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; + + if (note) { + arr.push(note.subtreeNotesIncludingTemplated); + } + } + } + + return arr.flat(); + } + + /** @return {Note[]} */ + get subtreeNotes() { + const arr = [[this]]; + + for (const childNote of this.children) { + arr.push(childNote.subtreeNotes); + } + + return arr.flat(); + } + + /** @return {Note[]} - returns only notes which are templated, does not include their subtrees + * in effect returns notes which are influenced by note's non-inheritable attributes */ + get templatedNotes() { + const arr = [this]; + + for (const targetRelation of this.targetRelations) { + if (targetRelation.name === 'template') { + const note = targetRelation.note; + + if (note) { + arr.push(note); + } + } + } + + return arr; + } +} diff --git a/src/services/note_cache/expressions/and.js b/src/services/note_cache/expressions/and.js new file mode 100644 index 000000000..2cdb97f8f --- /dev/null +++ b/src/services/note_cache/expressions/and.js @@ -0,0 +1,13 @@ +export default class AndExp { + constructor(subExpressions) { + this.subExpressions = subExpressions; + } + + execute(noteSet, searchContext) { + for (const subExpression of this.subExpressions) { + noteSet = subExpression.execute(noteSet, searchContext); + } + + return noteSet; + } +} diff --git a/src/services/note_cache/expressions/equals.js b/src/services/note_cache/expressions/equals.js new file mode 100644 index 000000000..794fbc3f3 --- /dev/null +++ b/src/services/note_cache/expressions/equals.js @@ -0,0 +1,28 @@ +export default class EqualsExp { + constructor(attributeType, attributeName, attributeValue) { + this.attributeType = attributeType; + this.attributeName = attributeName; + this.attributeValue = attributeValue; + } + + execute(noteSet) { + const attrs = findAttributes(this.attributeType, this.attributeName); + const resultNoteSet = new NoteSet(); + + for (const attr of attrs) { + const note = attr.note; + + if (noteSet.hasNoteId(note.noteId) && attr.value === this.attributeValue) { + if (attr.isInheritable) { + resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); + } + else if (note.isTemplate) { + resultNoteSet.addAll(note.templatedNotes); + } + else { + resultNoteSet.add(note); + } + } + } + } +} diff --git a/src/services/note_cache/expressions/exists.js b/src/services/note_cache/expressions/exists.js new file mode 100644 index 000000000..f79f17af7 --- /dev/null +++ b/src/services/note_cache/expressions/exists.js @@ -0,0 +1,27 @@ +export default class ExistsExp { + constructor(attributeType, attributeName) { + this.attributeType = attributeType; + this.attributeName = attributeName; + } + + execute(noteSet) { + const attrs = findAttributes(this.attributeType, this.attributeName); + const resultNoteSet = new NoteSet(); + + for (const attr of attrs) { + const note = attr.note; + + if (noteSet.hasNoteId(note.noteId)) { + if (attr.isInheritable) { + resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); + } + else if (note.isTemplate) { + resultNoteSet.addAll(note.templatedNotes); + } + else { + resultNoteSet.add(note); + } + } + } + } +} diff --git a/src/services/note_cache/expressions/note_cache_fulltext.js b/src/services/note_cache/expressions/note_cache_fulltext.js new file mode 100644 index 000000000..5451d7255 --- /dev/null +++ b/src/services/note_cache/expressions/note_cache_fulltext.js @@ -0,0 +1,125 @@ +export default class NoteCacheFulltextExp { + constructor(tokens) { + this.tokens = tokens; + } + + execute(noteSet, searchContext) { + const resultNoteSet = new NoteSet(); + + const candidateNotes = this.getCandidateNotes(noteSet); + + for (const note of candidateNotes) { + // autocomplete should be able to find notes by their noteIds as well (only leafs) + if (this.tokens.length === 1 && note.noteId === this.tokens[0]) { + this.searchDownThePath(note, [], [], resultNoteSet, searchContext); + continue; + } + + // for leaf note it doesn't matter if "archived" label is inheritable or not + if (note.isArchived) { + continue; + } + + const foundAttrTokens = []; + + for (const attribute of note.ownedAttributes) { + for (const token of this.tokens) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { + foundAttrTokens.push(token); + } + } + } + + for (const parentNote of note.parents) { + const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const foundTokens = foundAttrTokens.slice(); + + for (const token of this.tokens) { + if (title.includes(token)) { + foundTokens.push(token); + } + } + + if (foundTokens.length > 0) { + const remainingTokens = this.tokens.filter(token => !foundTokens.includes(token)); + + this.searchDownThePath(parentNote, remainingTokens, [note.noteId], resultNoteSet, searchContext); + } + } + } + + return resultNoteSet; + } + + /** + * Returns noteIds which have at least one matching tokens + * + * @param {NoteSet} noteSet + * @return {String[]} + */ + getCandidateNotes(noteSet) { + const candidateNotes = []; + + for (const note of noteSet.notes) { + for (const token of this.tokens) { + if (note.flatText.includes(token)) { + candidateNotes.push(note); + break; + } + } + } + + return candidateNotes; + } + + searchDownThePath(note, tokens, path, resultNoteSet, searchContext) { + if (tokens.length === 0) { + const retPath = getSomePath(note, path); + + if (retPath) { + const noteId = retPath[retPath.length - 1]; + searchContext.noteIdToNotePath[noteId] = retPath; + + resultNoteSet.add(notes[noteId]); + } + + return; + } + + if (!note.parents.length === 0 || note.noteId === 'root') { + return; + } + + const foundAttrTokens = []; + + for (const attribute of note.ownedAttributes) { + for (const token of tokens) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { + foundAttrTokens.push(token); + } + } + } + + for (const parentNote of note.parents) { + const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const foundTokens = foundAttrTokens.slice(); + + for (const token of tokens) { + if (title.includes(token)) { + foundTokens.push(token); + } + } + + if (foundTokens.length > 0) { + const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); + + this.searchDownThePath(parentNote, remainingTokens, path.concat([note.noteId]), resultNoteSet, searchContext); + } + else { + this.searchDownThePath(parentNote, tokens, path.concat([note.noteId]), resultNoteSet, searchContext); + } + } + } +} diff --git a/src/services/note_cache/expressions/note_content_fulltext.js b/src/services/note_cache/expressions/note_content_fulltext.js new file mode 100644 index 000000000..9a8db4a40 --- /dev/null +++ b/src/services/note_cache/expressions/note_content_fulltext.js @@ -0,0 +1,26 @@ +export default class NoteContentFulltextExp { + constructor(tokens) { + this.tokens = tokens; + } + + async execute(noteSet) { + const resultNoteSet = new NoteSet(); + const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); + + const noteIds = await sql.getColumn(` + SELECT notes.noteId + FROM notes + JOIN note_contents ON notes.noteId = note_contents.noteId + WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); + + const results = []; + + for (const noteId of noteIds) { + if (noteSet.hasNoteId(noteId) && noteId in notes) { + resultNoteSet.add(notes[noteId]); + } + } + + return results; + } +} diff --git a/src/services/note_cache/expressions/or.js b/src/services/note_cache/expressions/or.js new file mode 100644 index 000000000..f5ba189c0 --- /dev/null +++ b/src/services/note_cache/expressions/or.js @@ -0,0 +1,15 @@ +export default class OrExp { + constructor(subExpressions) { + this.subExpressions = subExpressions; + } + + execute(noteSet, searchContext) { + const resultNoteSet = new NoteSet(); + + for (const subExpression of this.subExpressions) { + resultNoteSet.mergeIn(subExpression.execute(noteSet, searchContext)); + } + + return resultNoteSet; + } +} diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js new file mode 100644 index 000000000..0d6067235 --- /dev/null +++ b/src/services/note_cache/note_cache.js @@ -0,0 +1,224 @@ +import treeCache from "../../public/app/services/tree_cache.js"; + +const sql = require('../sql.js'); +const sqlInit = require('../sql_init.js'); +const eventService = require('../events.js'); +const protectedSessionService = require('../protected_session.js'); +const utils = require('../utils.js'); +const hoistedNoteService = require('../hoisted_note.js'); +const stringSimilarity = require('string-similarity'); + +class NoteCache { + constructor() { + /** @type {Object.} */ + this.notes = null; + /** @type {Object.} */ + this.branches = null; + /** @type {Object.} */ + this.childParentToBranch = {}; + /** @type {Object.} */ + this.attributes = null; + /** @type {Object.} Points from attribute type-name to list of attributes them */ + this.attributeIndex = null; + + this.loaded = false; + this.loadedPromiseResolve; + /** Is resolved after the initial load */ + this.loadedPromise = new Promise(res => this.loadedPromiseResolve = res); + } + + /** @return {Attribute[]} */ + findAttributes(type, name) { + return this.attributeIndex[`${type}-${name}`] || []; + } + + async load() { + this.notes = await this.getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, + row => new Note(row)); + + this.branches = await this.getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, + row => new Branch(row)); + + this.attributeIndex = []; + + this.attributes = await this.getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, + row => new Attribute(row)); + + this.loaded = true; + this.loadedPromiseResolve(); + } + + async getMappedRows(query, cb) { + const map = {}; + const results = await sql.getRows(query, []); + + for (const row of results) { + const keys = Object.keys(row); + + map[row[keys[0]]] = cb(row); + } + + return map; + } + + decryptProtectedNote(note) { + if (note.isProtected && !note.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { + note.title = protectedSessionService.decryptString(note.title); + + note.isDecrypted = true; + } + } + + decryptProtectedNotes() { + for (const note of Object.values(this.notes)) { + decryptProtectedNote(note); + } + } +} + +const noteCache = new NoteCache(); + +eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { + // note that entity can also be just POJO without methods if coming from sync + + if (!noteCache.loaded) { + return; + } + + if (entityName === 'notes') { + const {noteId} = entity; + + if (entity.isDeleted) { + delete noteCache.notes[noteId]; + } + else if (noteId in noteCache.notes) { + const note = noteCache.notes[noteId]; + + // we can assume we have protected session since we managed to update + note.title = entity.title; + note.isProtected = entity.isProtected; + note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; + note.flatTextCache = null; + + decryptProtectedNote(note); + } + else { + const note = new Note(entity); + noteCache.notes[noteId] = note; + + decryptProtectedNote(note); + } + } + else if (entityName === 'branches') { + const {branchId, noteId, parentNoteId} = entity; + const childNote = noteCache.notes[noteId]; + + if (entity.isDeleted) { + if (childNote) { + childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); + childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); + + if (childNote.parents.length > 0) { + childNote.invalidateSubtreeCaches(); + } + } + + const parentNote = noteCache.notes[parentNoteId]; + + if (parentNote) { + parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); + } + + delete noteCache.childParentToBranch[`${noteId}-${parentNoteId}`]; + delete noteCache.branches[branchId]; + } + else if (branchId in noteCache.branches) { + // only relevant thing which can change in a branch is prefix + noteCache.branches[branchId].prefix = entity.prefix; + + if (childNote) { + childNote.flatTextCache = null; + } + } + else { + noteCache.branches[branchId] = new Branch(entity); + + if (childNote) { + childNote.resortParents(); + } + } + } + else if (entityName === 'attributes') { + const {attributeId, noteId} = entity; + const note = noteCache.notes[noteId]; + const attr = noteCache.attributes[attributeId]; + + if (entity.isDeleted) { + if (note && attr) { + // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + + note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); + + const targetNote = attr.targetNote; + + if (targetNote) { + targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); + } + } + + delete noteCache.attributes[attributeId]; + delete noteCache.attributeIndex[`${attr.type}-${attr.name}`]; + } + else if (attributeId in noteCache.attributes) { + const attr = noteCache.attributes[attributeId]; + + // attr name and isInheritable are immutable + attr.value = entity.value; + + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeFlatText(); + } + else { + note.flatTextCache = null; + } + } + else { + const attr = new Attribute(entity); + noteCache.attributes[attributeId] = attr; + + if (note) { + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + else { + this.invalidateThisCache(); + } + } + } + } +}); + +function getBranch(childNoteId, parentNoteId) { + return noteCache.childParentToBranch[`${childNoteId}-${parentNoteId}`]; +} + +eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { + noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); +}); + +sqlInit.dbReady.then(() => utils.stopWatch("Note cache load", () => treeCache.load())); + +module.exports = { + loadedPromise, + findNotesForAutocomplete, + getNotePath, + getNoteTitleForPath, + isAvailable, + isArchived, + isInAncestor, + load, + findSimilarNotes +}; diff --git a/src/services/note_cache/note_cache_service.js b/src/services/note_cache/note_cache_service.js new file mode 100644 index 000000000..c5bebc305 --- /dev/null +++ b/src/services/note_cache/note_cache_service.js @@ -0,0 +1,233 @@ +function isNotePathArchived(notePath) { + const noteId = notePath[notePath.length - 1]; + const note = noteCache.notes[noteId]; + + if (note.isArchived) { + return true; + } + + for (let i = 0; i < notePath.length - 1; i++) { + const note = noteCache.notes[notePath[i]]; + + // this is going through parents so archived must be inheritable + if (note.hasInheritableOwnedArchivedLabel) { + return true; + } + } + + return false; +} + +/** + * This assumes that note is available. "archived" note means that there isn't a single non-archived note-path + * leading to this note. + * + * @param noteId + */ +function isArchived(noteId) { + const notePath = getSomePath(noteId); + + return isNotePathArchived(notePath); +} + +/** + * @param {string} noteId + * @param {string} ancestorNoteId + * @return {boolean} - true if given noteId has ancestorNoteId in any of its paths (even archived) + */ +function isInAncestor(noteId, ancestorNoteId) { + if (ancestorNoteId === 'root' || ancestorNoteId === noteId) { + return true; + } + + const note = noteCache.notes[noteId]; + + for (const parentNote of note.parents) { + if (isInAncestor(parentNote.noteId, ancestorNoteId)) { + return true; + } + } + + return false; +} + +function getNoteTitle(childNoteId, parentNoteId) { + const childNote = noteCache.notes[childNoteId]; + const parentNote = noteCache.notes[parentNoteId]; + + let title; + + if (childNote.isProtected) { + title = protectedSessionService.isProtectedSessionAvailable() ? childNote.title : '[protected]'; + } + else { + title = childNote.title; + } + + const branch = parentNote ? getBranch(childNote.noteId, parentNote.noteId) : null; + + return ((branch && branch.prefix) ? `${branch.prefix} - ` : '') + title; +} + +function getNoteTitleArrayForPath(notePathArray) { + const titles = []; + + if (notePathArray[0] === hoistedNoteService.getHoistedNoteId() && notePathArray.length === 1) { + return [ getNoteTitle(hoistedNoteService.getHoistedNoteId()) ]; + } + + let parentNoteId = 'root'; + let hoistedNotePassed = false; + + for (const noteId of notePathArray) { + // start collecting path segment titles only after hoisted note + if (hoistedNotePassed) { + const title = getNoteTitle(noteId, parentNoteId); + + titles.push(title); + } + + if (noteId === hoistedNoteService.getHoistedNoteId()) { + hoistedNotePassed = true; + } + + parentNoteId = noteId; + } + + return titles; +} + +function getNoteTitleForPath(notePathArray) { + const titles = getNoteTitleArrayForPath(notePathArray); + + return titles.join(' / '); +} + +/** + * Returns notePath for noteId from cache. Note hoisting is respected. + * Archived notes are also returned, but non-archived paths are preferred if available + * - this means that archived paths is returned only if there's no non-archived path + * - you can check whether returned path is archived using isArchived() + */ +function getSomePath(note, path = []) { + if (note.noteId === 'root') { + path.push(note.noteId); + path.reverse(); + + if (!path.includes(hoistedNoteService.getHoistedNoteId())) { + return false; + } + + return path; + } + + const parents = note.parents; + if (parents.length === 0) { + return false; + } + + for (const parentNote of parents) { + const retPath = getSomePath(parentNote, path.concat([note.noteId])); + + if (retPath) { + return retPath; + } + } + + return false; +} + +function getNotePath(noteId) { + const note = noteCache.notes[noteId]; + const retPath = getSomePath(note); + + if (retPath) { + const noteTitle = getNoteTitleForPath(retPath); + const parentNote = note.parents[0]; + + return { + noteId: noteId, + branchId: getBranch(noteId, parentNote.noteId).branchId, + title: noteTitle, + notePath: retPath, + path: retPath.join('/') + }; + } +} + +function evaluateSimilarity(sourceNote, candidateNote, results) { + let coeff = stringSimilarity.compareTwoStrings(sourceNote.flatText, candidateNote.flatText); + + if (coeff > 0.4) { + const notePath = getSomePath(candidateNote); + + // this takes care of note hoisting + if (!notePath) { + return; + } + + if (isNotePathArchived(notePath)) { + coeff -= 0.2; // archived penalization + } + + results.push({coeff, notePath, noteId: candidateNote.noteId}); + } +} + +/** + * Point of this is to break up long running sync process to avoid blocking + * see https://snyk.io/blog/nodejs-how-even-quick-async-functions-can-block-the-event-loop-starve-io/ + */ +function setImmediatePromise() { + return new Promise((resolve) => { + setTimeout(() => resolve(), 0); + }); +} + +async function findSimilarNotes(noteId) { + const results = []; + let i = 0; + + const origNote = noteCache.notes[noteId]; + + if (!origNote) { + return []; + } + + for (const note of Object.values(notes)) { + if (note.isProtected && !note.isDecrypted) { + continue; + } + + evaluateSimilarity(origNote, note, results); + + i++; + + if (i % 200 === 0) { + await setImmediatePromise(); + } + } + + results.sort((a, b) => a.coeff > b.coeff ? -1 : 1); + + return results.length > 50 ? results.slice(0, 50) : results; +} + +/** + * @param noteId + * @returns {boolean} - true if note exists (is not deleted) and is available in current note hoisting + */ +function isAvailable(noteId) { + const notePath = getNotePath(noteId); + + return !!notePath; +} + +module.exports = { + getNotePath, + getNoteTitleForPath, + isAvailable, + isArchived, + isInAncestor, + findSimilarNotes +}; diff --git a/src/services/note_cache/note_set.js b/src/services/note_cache/note_set.js new file mode 100644 index 000000000..bd088162a --- /dev/null +++ b/src/services/note_cache/note_set.js @@ -0,0 +1,22 @@ +export default class NoteSet { + constructor(notes = []) { + this.notes = notes; + } + + add(note) { + this.notes.push(note); + } + + addAll(notes) { + this.notes.push(...notes); + } + + hasNoteId(noteId) { + // TODO: optimize + return !!this.notes.find(note => note.noteId === noteId); + } + + mergeIn(anotherNoteSet) { + this.notes = this.notes.concat(anotherNoteSet.arr); + } +} diff --git a/src/services/note_cache/search.js b/src/services/note_cache/search.js new file mode 100644 index 000000000..993877f14 --- /dev/null +++ b/src/services/note_cache/search.js @@ -0,0 +1,113 @@ +async function findNotesWithExpression(expression) { + + const hoistedNote = notes[hoistedNoteService.getHoistedNoteId()]; + const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') + ? hoistedNote.subtreeNotes + : Object.values(notes); + + const allNoteSet = new NoteSet(allNotes); + + const searchContext = { + noteIdToNotePath: {} + }; + + const noteSet = await expression.execute(allNoteSet, searchContext); + + let searchResults = noteSet.notes + .map(note => searchContext.noteIdToNotePath[note.noteId] || getSomePath(note)) + .filter(notePathArray => notePathArray.includes(hoistedNoteService.getHoistedNoteId())) + .map(notePathArray => new SearchResult(notePathArray)); + + // sort results by depth of the note. This is based on the assumption that more important results + // are closer to the note root. + searchResults.sort((a, b) => { + if (a.notePathArray.length === b.notePathArray.length) { + return a.notePathTitle < b.notePathTitle ? -1 : 1; + } + + return a.notePathArray.length < b.notePathArray.length ? -1 : 1; + }); + + return searchResults; +} + +async function findNotesForAutocomplete(query) { + if (!query.trim().length) { + return []; + } + + const tokens = query + .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc + .toLowerCase() + .split(/[ -]/) + .filter(token => token !== '/'); // '/' is used as separator + + const expression = new NoteCacheFulltextExp(tokens); + + let searchResults = await findNotesWithExpression(expression); + + searchResults = searchResults.slice(0, 200); + + highlightSearchResults(searchResults, tokens); + + return searchResults.map(result => { + return { + notePath: result.notePath, + notePathTitle: result.notePathTitle, + highlightedNotePathTitle: result.highlightedNotePathTitle + } + }); +} + +function highlightSearchResults(searchResults, tokens) { + // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks + // which would make the resulting HTML string invalid. + // { and } are used for marking and tag (to avoid matches on single 'b' character) + tokens = tokens.map(token => token.replace('/[<\{\}]/g', '')); + + // sort by the longest so we first highlight longest matches + tokens.sort((a, b) => a.length > b.length ? -1 : 1); + + for (const result of searchResults) { + const note = notes[result.noteId]; + + result.highlightedNotePathTitle = result.notePathTitle; + + for (const attr of note.attributes) { + if (tokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { + result.highlightedNotePathTitle += ` ${formatAttribute(attr)}`; + } + } + } + + for (const token of tokens) { + const tokenRegex = new RegExp("(" + utils.escapeRegExp(token) + ")", "gi"); + + for (const result of searchResults) { + result.highlightedNotePathTitle = result.highlightedNotePathTitle.replace(tokenRegex, "{$1}"); + } + } + + for (const result of searchResults) { + result.highlightedNotePathTitle = result.highlightedNotePathTitle + .replace(/{/g, "") + .replace(/}/g, ""); + } +} + +function formatAttribute(attr) { + if (attr.type === 'relation') { + return '@' + utils.escapeHtml(attr.name) + "=…"; + } + else if (attr.type === 'label') { + let label = '#' + utils.escapeHtml(attr.name); + + if (attr.value) { + const val = /[^\w_-]/.test(attr.value) ? '"' + attr.value + '"' : attr.value; + + label += '=' + utils.escapeHtml(val); + } + + return label; + } +} diff --git a/src/services/note_cache/search_result.js b/src/services/note_cache/search_result.js new file mode 100644 index 000000000..3a92f4fbe --- /dev/null +++ b/src/services/note_cache/search_result.js @@ -0,0 +1,14 @@ +export default class SearchResult { + constructor(notePathArray) { + this.notePathArray = notePathArray; + this.notePathTitle = getNoteTitleForPath(notePathArray); + } + + get notePath() { + return this.notePathArray.join('/'); + } + + get noteId() { + return this.notePathArray[this.notePathArray.length - 1]; + } +} diff --git a/src/services/search.js b/src/services/search.js index c62c1dbda..72f9b3327 100644 --- a/src/services/search.js +++ b/src/services/search.js @@ -3,7 +3,7 @@ const sql = require('./sql'); const log = require('./log'); const parseFilters = require('./parse_filters'); const buildSearchQuery = require('./build_search_query'); -const noteCacheService = require('./note_cache'); +const noteCacheService = require('./note_cache/note_cache.js'); async function searchForNotes(searchString) { const noteIds = await searchForNoteIds(searchString); @@ -71,4 +71,4 @@ async function searchForNoteIds(searchString) { module.exports = { searchForNotes, searchForNoteIds -}; \ No newline at end of file +}; diff --git a/src/services/tree.js b/src/services/tree.js index 410d565fc..4f6c4bffe 100644 --- a/src/services/tree.js +++ b/src/services/tree.js @@ -5,7 +5,7 @@ const repository = require('./repository'); const Branch = require('../entities/branch'); const syncTableService = require('./sync_table'); const protectedSessionService = require('./protected_session'); -const noteCacheService = require('./note_cache'); +const noteCacheService = require('./note_cache/note_cache.js'); async function getNotes(noteIds) { // we return also deleted notes which have been specifically asked for @@ -197,4 +197,4 @@ module.exports = { validateParentChild, sortNotesAlphabetically, setNoteToParent -}; \ No newline at end of file +}; From 60c204972917e104e095de4edbce5e78138994f6 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 17 May 2020 09:48:24 +0200 Subject: [PATCH 16/47] note cache refactoring WIP --- src/services/note_cache/entities/attribute.js | 16 ++++-- src/services/note_cache/entities/branch.js | 14 +++-- src/services/note_cache/entities/note.js | 18 ++++++- src/services/note_cache/note_cache.js | 52 ++++++------------- src/services/note_cache/note_cache_service.js | 5 ++ src/services/note_cache/note_set.js | 6 ++- .../{note_cache => search}/expressions/and.js | 6 ++- .../expressions/equals.js | 6 ++- .../expressions/exists.js | 6 ++- .../expressions/note_cache_fulltext.js | 6 ++- .../expressions/note_content_fulltext.js | 6 ++- .../{note_cache => search}/expressions/or.js | 6 ++- src/services/{note_cache => search}/search.js | 4 ++ .../{note_cache => search}/search_result.js | 10 +++- 14 files changed, 104 insertions(+), 57 deletions(-) rename src/services/{note_cache => search}/expressions/and.js (84%) rename src/services/{note_cache => search}/expressions/equals.js (93%) rename src/services/{note_cache => search}/expressions/exists.js (92%) rename src/services/{note_cache => search}/expressions/note_cache_fulltext.js (97%) rename src/services/{note_cache => search}/expressions/note_content_fulltext.js (89%) rename src/services/{note_cache => search}/expressions/or.js (86%) rename src/services/{note_cache => search}/search.js (97%) rename src/services/{note_cache => search}/search_result.js (53%) diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index a276d47d6..beb605fbf 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -1,3 +1,7 @@ +"use strict"; + +const noteCache = require('../note_cache'); + class Attribute { constructor(row) { /** @param {string} */ @@ -13,11 +17,11 @@ class Attribute { /** @param {boolean} */ this.isInheritable = !!row.isInheritable; - notes[this.noteId].ownedAttributes.push(this); + noteCache.notes[this.noteId].ownedAttributes.push(this); const key = `${this.type-this.name}`; - attributeIndex[key] = attributeIndex[key] || []; - attributeIndex[key].push(this); + noteCache.attributeIndex[key] = noteCache.attributeIndex[key] || []; + noteCache.attributeIndex[key].push(this); const targetNote = this.targetNote; @@ -32,12 +36,14 @@ class Attribute { } get note() { - return notes[this.noteId]; + return noteCache.notes[this.noteId]; } get targetNote() { if (this.type === 'relation') { - return notes[this.value]; + return noteCache.notes[this.value]; } } } + +module.exports = Attribute; diff --git a/src/services/note_cache/entities/branch.js b/src/services/note_cache/entities/branch.js index aaf075aa8..07d4221f6 100644 --- a/src/services/note_cache/entities/branch.js +++ b/src/services/note_cache/entities/branch.js @@ -1,4 +1,8 @@ -export default class Branch { +"use strict"; + +const noteCache = require('../note_cache'); + +class Branch { constructor(row) { /** @param {string} */ this.branchId = row.branchId; @@ -13,7 +17,7 @@ export default class Branch { return; } - const childNote = notes[this.noteId]; + const childNote = noteCache.notes[this.noteId]; const parentNote = this.parentNote; if (!childNote) { @@ -26,12 +30,12 @@ export default class Branch { parentNote.children.push(childNote); - childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; + noteCache.childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; } /** @return {Note} */ get parentNote() { - const note = notes[this.parentNoteId]; + const note = noteCache.notes[this.parentNoteId]; if (!note) { console.log(`Cannot find note ${this.parentNoteId}`); @@ -40,3 +44,5 @@ export default class Branch { return note; } } + +module.exports = Branch; diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index 7f5dfe924..d86508e80 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -1,4 +1,8 @@ -export default class Note { +"use strict"; + +const noteCache = require('../note_cache'); + +class Note { constructor(row) { /** @param {string} */ this.noteId = row.noteId; @@ -29,7 +33,7 @@ export default class Note { this.flatTextCache = null; if (protectedSessionService.isProtectedSessionAvailable()) { - decryptProtectedNote(this); + noteCache.decryptProtectedNote(this); } } @@ -233,4 +237,14 @@ export default class Note { return arr; } + + decrypt() { + if (this.isProtected && !this.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { + this.title = protectedSessionService.decryptString(note.title); + + this.isDecrypted = true; + } + } } + +module.exports = Note; diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index 0d6067235..974377dc5 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -1,12 +1,11 @@ -import treeCache from "../../public/app/services/tree_cache.js"; +"use strict"; +const Note = require('./entities/note'); +const Branch = require('./entities/branch'); +const Attribute = require('./entities/attribute'); const sql = require('../sql.js'); const sqlInit = require('../sql_init.js'); const eventService = require('../events.js'); -const protectedSessionService = require('../protected_session.js'); -const utils = require('../utils.js'); -const hoistedNoteService = require('../hoisted_note.js'); -const stringSimilarity = require('string-similarity'); class NoteCache { constructor() { @@ -22,9 +21,7 @@ class NoteCache { this.attributeIndex = null; this.loaded = false; - this.loadedPromiseResolve; - /** Is resolved after the initial load */ - this.loadedPromise = new Promise(res => this.loadedPromiseResolve = res); + this.loadedPromise = this.load(); } /** @return {Attribute[]} */ @@ -33,6 +30,8 @@ class NoteCache { } async load() { + await sqlInit.dbReady; + this.notes = await this.getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, row => new Note(row)); @@ -45,7 +44,6 @@ class NoteCache { row => new Attribute(row)); this.loaded = true; - this.loadedPromiseResolve(); } async getMappedRows(query, cb) { @@ -61,18 +59,14 @@ class NoteCache { return map; } - decryptProtectedNote(note) { - if (note.isProtected && !note.isDecrypted && protectedSessionService.isProtectedSessionAvailable()) { - note.title = protectedSessionService.decryptString(note.title); - - note.isDecrypted = true; + decryptProtectedNotes() { + for (const note of Object.values(this.notes)) { + note.decrypt(); } } - decryptProtectedNotes() { - for (const note of Object.values(this.notes)) { - decryptProtectedNote(note); - } + getBranch(childNoteId, parentNoteId) { + return this.childParentToBranch[`${childNoteId}-${parentNoteId}`]; } } @@ -100,13 +94,13 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; note.flatTextCache = null; - decryptProtectedNote(note); + noteCache.decryptProtectedNote(note); } else { const note = new Note(entity); noteCache.notes[noteId] = note; - decryptProtectedNote(note); + noteCache.decryptProtectedNote(note); } } else if (entityName === 'branches') { @@ -201,24 +195,8 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED } }); -function getBranch(childNoteId, parentNoteId) { - return noteCache.childParentToBranch[`${childNoteId}-${parentNoteId}`]; -} - eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); }); -sqlInit.dbReady.then(() => utils.stopWatch("Note cache load", () => treeCache.load())); - -module.exports = { - loadedPromise, - findNotesForAutocomplete, - getNotePath, - getNoteTitleForPath, - isAvailable, - isArchived, - isInAncestor, - load, - findSimilarNotes -}; +module.exports = noteCache; diff --git a/src/services/note_cache/note_cache_service.js b/src/services/note_cache/note_cache_service.js index c5bebc305..3b567c2a9 100644 --- a/src/services/note_cache/note_cache_service.js +++ b/src/services/note_cache/note_cache_service.js @@ -1,3 +1,8 @@ +"use strict"; + +const noteCache = require('./note_cache'); +const hoistedNoteService = require('../hoisted_note'); + function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; const note = noteCache.notes[noteId]; diff --git a/src/services/note_cache/note_set.js b/src/services/note_cache/note_set.js index bd088162a..da768854d 100644 --- a/src/services/note_cache/note_set.js +++ b/src/services/note_cache/note_set.js @@ -1,4 +1,6 @@ -export default class NoteSet { +"use strict"; + +class NoteSet { constructor(notes = []) { this.notes = notes; } @@ -20,3 +22,5 @@ export default class NoteSet { this.notes = this.notes.concat(anotherNoteSet.arr); } } + +module.exports = NoteSet; diff --git a/src/services/note_cache/expressions/and.js b/src/services/search/expressions/and.js similarity index 84% rename from src/services/note_cache/expressions/and.js rename to src/services/search/expressions/and.js index 2cdb97f8f..f542d416c 100644 --- a/src/services/note_cache/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -1,4 +1,6 @@ -export default class AndExp { +"use strict"; + +class AndExp { constructor(subExpressions) { this.subExpressions = subExpressions; } @@ -11,3 +13,5 @@ export default class AndExp { return noteSet; } } + +module.exports = AndExp; diff --git a/src/services/note_cache/expressions/equals.js b/src/services/search/expressions/equals.js similarity index 93% rename from src/services/note_cache/expressions/equals.js rename to src/services/search/expressions/equals.js index 794fbc3f3..ab6e69b60 100644 --- a/src/services/note_cache/expressions/equals.js +++ b/src/services/search/expressions/equals.js @@ -1,4 +1,6 @@ -export default class EqualsExp { +"use strict"; + +class EqualsExp { constructor(attributeType, attributeName, attributeValue) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -26,3 +28,5 @@ export default class EqualsExp { } } } + +module.exports = EqualsExp; diff --git a/src/services/note_cache/expressions/exists.js b/src/services/search/expressions/exists.js similarity index 92% rename from src/services/note_cache/expressions/exists.js rename to src/services/search/expressions/exists.js index f79f17af7..781fd0604 100644 --- a/src/services/note_cache/expressions/exists.js +++ b/src/services/search/expressions/exists.js @@ -1,4 +1,6 @@ -export default class ExistsExp { +"use strict"; + +class ExistsExp { constructor(attributeType, attributeName) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -25,3 +27,5 @@ export default class ExistsExp { } } } + +module.exports = ExistsExp; diff --git a/src/services/note_cache/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js similarity index 97% rename from src/services/note_cache/expressions/note_cache_fulltext.js rename to src/services/search/expressions/note_cache_fulltext.js index 5451d7255..3d3e1bcff 100644 --- a/src/services/note_cache/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -1,4 +1,6 @@ -export default class NoteCacheFulltextExp { +"use strict"; + +class NoteCacheFulltextExp { constructor(tokens) { this.tokens = tokens; } @@ -123,3 +125,5 @@ export default class NoteCacheFulltextExp { } } } + +module.exports = NoteCacheFulltextExp; diff --git a/src/services/note_cache/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js similarity index 89% rename from src/services/note_cache/expressions/note_content_fulltext.js rename to src/services/search/expressions/note_content_fulltext.js index 9a8db4a40..a7a5a4f7a 100644 --- a/src/services/note_cache/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -1,4 +1,6 @@ -export default class NoteContentFulltextExp { +"use strict"; + +class NoteContentFulltextExp { constructor(tokens) { this.tokens = tokens; } @@ -24,3 +26,5 @@ export default class NoteContentFulltextExp { return results; } } + +module.exports = NoteContentFulltextExp; diff --git a/src/services/note_cache/expressions/or.js b/src/services/search/expressions/or.js similarity index 86% rename from src/services/note_cache/expressions/or.js rename to src/services/search/expressions/or.js index f5ba189c0..c17dd2210 100644 --- a/src/services/note_cache/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -1,4 +1,6 @@ -export default class OrExp { +"use strict"; + +class OrExp { constructor(subExpressions) { this.subExpressions = subExpressions; } @@ -13,3 +15,5 @@ export default class OrExp { return resultNoteSet; } } + +module.exports = OrExp; diff --git a/src/services/note_cache/search.js b/src/services/search/search.js similarity index 97% rename from src/services/note_cache/search.js rename to src/services/search/search.js index 993877f14..09f6f0031 100644 --- a/src/services/note_cache/search.js +++ b/src/services/search/search.js @@ -1,3 +1,7 @@ +"use strict"; + +import NoteCacheFulltextExp from "./expressions/note_cache_fulltext.js"; + async function findNotesWithExpression(expression) { const hoistedNote = notes[hoistedNoteService.getHoistedNoteId()]; diff --git a/src/services/note_cache/search_result.js b/src/services/search/search_result.js similarity index 53% rename from src/services/note_cache/search_result.js rename to src/services/search/search_result.js index 3a92f4fbe..83c395b10 100644 --- a/src/services/note_cache/search_result.js +++ b/src/services/search/search_result.js @@ -1,7 +1,11 @@ -export default class SearchResult { +"use strict"; + +const noteCacheService = require('../note_cache/note_cache_service'); + +class SearchResult { constructor(notePathArray) { this.notePathArray = notePathArray; - this.notePathTitle = getNoteTitleForPath(notePathArray); + this.notePathTitle = noteCacheService.getNoteTitleForPath(notePathArray); } get notePath() { @@ -12,3 +16,5 @@ export default class SearchResult { return this.notePathArray[this.notePathArray.length - 1]; } } + +module.exports = SearchResult; From 32eaafd024cb8df9fbb142070cec439477dda667 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 17 May 2020 10:11:19 +0200 Subject: [PATCH 17/47] note cache fixes after refactoring --- src/routes/api/autocomplete.js | 5 +++-- src/routes/api/similar_notes.js | 2 +- src/services/note_cache/entities/attribute.js | 16 ++++++++-------- src/services/note_cache/entities/branch.js | 12 ++++++------ src/services/note_cache/entities/note.js | 9 ++++++--- src/services/note_cache/note_cache.js | 10 +++++----- src/services/note_cache/note_cache_service.js | 7 +++++-- src/services/search/expressions/equals.js | 5 ++++- src/services/search/expressions/exists.js | 5 ++++- .../search/expressions/note_cache_fulltext.js | 12 ++++++++---- .../expressions/note_content_fulltext.js | 7 +++++-- .../{note_cache => search}/note_set.js | 0 src/services/search/search.js | 19 ++++++++++++++----- 13 files changed, 69 insertions(+), 40 deletions(-) rename src/services/{note_cache => search}/note_set.js (100%) diff --git a/src/routes/api/autocomplete.js b/src/routes/api/autocomplete.js index ef0b13724..28dfaf164 100644 --- a/src/routes/api/autocomplete.js +++ b/src/routes/api/autocomplete.js @@ -1,6 +1,7 @@ "use strict"; -const noteCacheService = require('../../services/note_cache/note_cache.js'); +const noteCacheService = require('../../services/note_cache/note_cache_service'); +const searchService = require('../../services/search/search'); const repository = require('../../services/repository'); const log = require('../../services/log'); const utils = require('../../services/utils'); @@ -18,7 +19,7 @@ async function getAutocomplete(req) { results = await getRecentNotes(activeNoteId); } else { - results = await noteCacheService.findNotesForAutocomplete(query); + results = await searchService.searchNotesForAutocomplete(query); } const msTaken = Date.now() - timestampStarted; diff --git a/src/routes/api/similar_notes.js b/src/routes/api/similar_notes.js index 22ddf427a..c04dc29f1 100644 --- a/src/routes/api/similar_notes.js +++ b/src/routes/api/similar_notes.js @@ -1,6 +1,6 @@ "use strict"; -const noteCacheService = require('../../services/note_cache/note_cache.js'); +const noteCacheService = require('../../services/note_cache/note_cache_service'); const repository = require('../../services/repository'); async function getSimilarNotes(req) { diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index beb605fbf..116c4b719 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -1,9 +1,9 @@ "use strict"; -const noteCache = require('../note_cache'); - class Attribute { - constructor(row) { + constructor(noteCache, row) { + /** @param {NoteCache} */ + this.noteCache = noteCache; /** @param {string} */ this.attributeId = row.attributeId; /** @param {string} */ @@ -17,11 +17,11 @@ class Attribute { /** @param {boolean} */ this.isInheritable = !!row.isInheritable; - noteCache.notes[this.noteId].ownedAttributes.push(this); + this.noteCache.notes[this.noteId].ownedAttributes.push(this); const key = `${this.type-this.name}`; - noteCache.attributeIndex[key] = noteCache.attributeIndex[key] || []; - noteCache.attributeIndex[key].push(this); + this.noteCache.attributeIndex[key] = this.noteCache.attributeIndex[key] || []; + this.noteCache.attributeIndex[key].push(this); const targetNote = this.targetNote; @@ -36,12 +36,12 @@ class Attribute { } get note() { - return noteCache.notes[this.noteId]; + return this.noteCache.notes[this.noteId]; } get targetNote() { if (this.type === 'relation') { - return noteCache.notes[this.value]; + return this.noteCache.notes[this.value]; } } } diff --git a/src/services/note_cache/entities/branch.js b/src/services/note_cache/entities/branch.js index 07d4221f6..b3a3af904 100644 --- a/src/services/note_cache/entities/branch.js +++ b/src/services/note_cache/entities/branch.js @@ -1,9 +1,9 @@ "use strict"; -const noteCache = require('../note_cache'); - class Branch { - constructor(row) { + constructor(noteCache, row) { + /** @param {NoteCache} */ + this.noteCache = noteCache; /** @param {string} */ this.branchId = row.branchId; /** @param {string} */ @@ -17,7 +17,7 @@ class Branch { return; } - const childNote = noteCache.notes[this.noteId]; + const childNote = this.noteCache.notes[this.noteId]; const parentNote = this.parentNote; if (!childNote) { @@ -30,12 +30,12 @@ class Branch { parentNote.children.push(childNote); - noteCache.childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; + this.noteCache.childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; } /** @return {Note} */ get parentNote() { - const note = noteCache.notes[this.parentNoteId]; + const note = this.noteCache.notes[this.parentNoteId]; if (!note) { console.log(`Cannot find note ${this.parentNoteId}`); diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index d86508e80..adc120d15 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -1,9 +1,12 @@ "use strict"; const noteCache = require('../note_cache'); +const protectedSessionService = require('../../protected_session'); class Note { - constructor(row) { + constructor(noteCache, row) { + /** @param {NoteCache} */ + this.noteCache = noteCache; /** @param {string} */ this.noteId = row.noteId; /** @param {string} */ @@ -33,7 +36,7 @@ class Note { this.flatTextCache = null; if (protectedSessionService.isProtectedSessionAvailable()) { - noteCache.decryptProtectedNote(this); + this.decrypt(); } } @@ -52,7 +55,7 @@ class Note { for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') { - const templateNote = notes[ownedAttr.value]; + const templateNote = this.noteCache.notes[ownedAttr.value]; if (templateNote) { templateAttributes.push(...templateNote.attributes); diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index 974377dc5..8aaa688bc 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -33,15 +33,15 @@ class NoteCache { await sqlInit.dbReady; this.notes = await this.getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, - row => new Note(row)); + row => new Note(this, row)); this.branches = await this.getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, - row => new Branch(row)); + row => new Branch(this, row)); this.attributeIndex = []; this.attributes = await this.getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, - row => new Attribute(row)); + row => new Attribute(this, row)); this.loaded = true; } @@ -94,13 +94,13 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; note.flatTextCache = null; - noteCache.decryptProtectedNote(note); + note.decrypt(); } else { const note = new Note(entity); noteCache.notes[noteId] = note; - noteCache.decryptProtectedNote(note); + note.decrypt(); } } else if (entityName === 'branches') { diff --git a/src/services/note_cache/note_cache_service.js b/src/services/note_cache/note_cache_service.js index 3b567c2a9..8007c9cb3 100644 --- a/src/services/note_cache/note_cache_service.js +++ b/src/services/note_cache/note_cache_service.js @@ -2,6 +2,7 @@ const noteCache = require('./note_cache'); const hoistedNoteService = require('../hoisted_note'); +const stringSimilarity = require('string-similarity'); function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; @@ -69,7 +70,7 @@ function getNoteTitle(childNoteId, parentNoteId) { title = childNote.title; } - const branch = parentNote ? getBranch(childNote.noteId, parentNote.noteId) : null; + const branch = parentNote ? noteCache.getBranch(childNote.noteId, parentNote.noteId) : null; return ((branch && branch.prefix) ? `${branch.prefix} - ` : '') + title; } @@ -199,7 +200,7 @@ async function findSimilarNotes(noteId) { return []; } - for (const note of Object.values(notes)) { + for (const note of Object.values(noteCache.notes)) { if (note.isProtected && !note.isDecrypted) { continue; } @@ -229,7 +230,9 @@ function isAvailable(noteId) { } module.exports = { + getSomePath, getNotePath, + getNoteTitle, getNoteTitleForPath, isAvailable, isArchived, diff --git a/src/services/search/expressions/equals.js b/src/services/search/expressions/equals.js index ab6e69b60..e5e04b3d8 100644 --- a/src/services/search/expressions/equals.js +++ b/src/services/search/expressions/equals.js @@ -1,5 +1,8 @@ "use strict"; +const NoteSet = require('../note_set'); +const noteCache = require('../../note_cache/note_cache'); + class EqualsExp { constructor(attributeType, attributeName, attributeValue) { this.attributeType = attributeType; @@ -8,7 +11,7 @@ class EqualsExp { } execute(noteSet) { - const attrs = findAttributes(this.attributeType, this.attributeName); + const attrs = noteCache.findAttributes(this.attributeType, this.attributeName); const resultNoteSet = new NoteSet(); for (const attr of attrs) { diff --git a/src/services/search/expressions/exists.js b/src/services/search/expressions/exists.js index 781fd0604..25c3a9245 100644 --- a/src/services/search/expressions/exists.js +++ b/src/services/search/expressions/exists.js @@ -1,5 +1,8 @@ "use strict"; +const NoteSet = require('../note_set'); +const noteCache = require('../../note_cache/note_cache'); + class ExistsExp { constructor(attributeType, attributeName) { this.attributeType = attributeType; @@ -7,7 +10,7 @@ class ExistsExp { } execute(noteSet) { - const attrs = findAttributes(this.attributeType, this.attributeName); + const attrs = noteCache.findAttributes(this.attributeType, this.attributeName); const resultNoteSet = new NoteSet(); for (const attr of attrs) { diff --git a/src/services/search/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js index 3d3e1bcff..f09df1e88 100644 --- a/src/services/search/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -1,5 +1,9 @@ "use strict"; +const NoteSet = require('../note_set'); +const noteCache = require('../../note_cache/note_cache'); +const noteCacheService = require('../../note_cache/note_cache_service'); + class NoteCacheFulltextExp { constructor(tokens) { this.tokens = tokens; @@ -34,7 +38,7 @@ class NoteCacheFulltextExp { } for (const parentNote of note.parents) { - const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const title = noteCacheService.getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); const foundTokens = foundAttrTokens.slice(); for (const token of this.tokens) { @@ -77,13 +81,13 @@ class NoteCacheFulltextExp { searchDownThePath(note, tokens, path, resultNoteSet, searchContext) { if (tokens.length === 0) { - const retPath = getSomePath(note, path); + const retPath = noteCacheService.getSomePath(note, path); if (retPath) { const noteId = retPath[retPath.length - 1]; searchContext.noteIdToNotePath[noteId] = retPath; - resultNoteSet.add(notes[noteId]); + resultNoteSet.add(noteCache.notes[noteId]); } return; @@ -105,7 +109,7 @@ class NoteCacheFulltextExp { } for (const parentNote of note.parents) { - const title = getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const title = noteCacheService.getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); const foundTokens = foundAttrTokens.slice(); for (const token of tokens) { diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index a7a5a4f7a..4d6900802 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -1,5 +1,8 @@ "use strict"; +const NoteSet = require('../note_set'); +const noteCache = require('../../note_cache/note_cache'); + class NoteContentFulltextExp { constructor(tokens) { this.tokens = tokens; @@ -18,8 +21,8 @@ class NoteContentFulltextExp { const results = []; for (const noteId of noteIds) { - if (noteSet.hasNoteId(noteId) && noteId in notes) { - resultNoteSet.add(notes[noteId]); + if (noteSet.hasNoteId(noteId) && noteId in noteCache.notes) { + resultNoteSet.add(noteCache.notes[noteId]); } } diff --git a/src/services/note_cache/note_set.js b/src/services/search/note_set.js similarity index 100% rename from src/services/note_cache/note_set.js rename to src/services/search/note_set.js diff --git a/src/services/search/search.js b/src/services/search/search.js index 09f6f0031..defc4cc7b 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -1,13 +1,18 @@ "use strict"; -import NoteCacheFulltextExp from "./expressions/note_cache_fulltext.js"; +const NoteCacheFulltextExp = require("./expressions/note_cache_fulltext"); +const NoteSet = require("./note_set"); +const SearchResult = require("./search_result"); +const noteCache = require('../note_cache/note_cache'); +const hoistedNoteService = require('../hoisted_note'); +const utils = require('../utils'); async function findNotesWithExpression(expression) { - const hoistedNote = notes[hoistedNoteService.getHoistedNoteId()]; + const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()]; const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') ? hoistedNote.subtreeNotes - : Object.values(notes); + : Object.values(noteCache.notes); const allNoteSet = new NoteSet(allNotes); @@ -35,7 +40,7 @@ async function findNotesWithExpression(expression) { return searchResults; } -async function findNotesForAutocomplete(query) { +async function searchNotesForAutocomplete(query) { if (!query.trim().length) { return []; } @@ -73,7 +78,7 @@ function highlightSearchResults(searchResults, tokens) { tokens.sort((a, b) => a.length > b.length ? -1 : 1); for (const result of searchResults) { - const note = notes[result.noteId]; + const note = noteCache.notes[result.noteId]; result.highlightedNotePathTitle = result.notePathTitle; @@ -115,3 +120,7 @@ function formatAttribute(attr) { return label; } } + +module.exports = { + searchNotesForAutocomplete +}; From e77e0ce675883136f7cfcb866dac2fa42242e12c Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 17 May 2020 19:43:37 +0200 Subject: [PATCH 18/47] lexer impl WIP + test --- package-lock.json | 413 +++++++++++---------- package.json | 14 +- spec/lexer.spec.js | 29 ++ spec/support/jasmine.json | 11 + src/services/search.js | 2 +- src/services/search/expressions/not.js | 15 + src/services/search/lexer.js | 94 +++++ src/services/search/note_set.js | 12 + src/services/{ => search}/parse_filters.js | 5 + 9 files changed, 384 insertions(+), 211 deletions(-) create mode 100644 spec/lexer.spec.js create mode 100644 spec/support/jasmine.json create mode 100644 src/services/search/expressions/not.js create mode 100644 src/services/search/lexer.js rename src/services/{ => search}/parse_filters.js (92%) diff --git a/package-lock.json b/package-lock.json index 554169239..0723c7057 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,9 +17,9 @@ "dev": true }, "@babel/runtime": { - "version": "7.9.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.2.tgz", - "integrity": "sha512-NE2DtOdufG7R5vnfQUTehdTfNycfUANEtCa9PssN9O/xmTzP4E08UI797ixaei6hBEVL9BI/PsdJS5x7mWoB9Q==", + "version": "7.9.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.6.tgz", + "integrity": "sha512-64AF1xY3OAkFHqOb9s4jpgk1Mm5vDZ4L3acHvAml+53nO1XbXLuDodsVpO4OIUsmemlUHMxNdYMNJmsvOwLrvQ==", "requires": { "regenerator-runtime": "^0.13.4" } @@ -198,26 +198,24 @@ } }, "@jimp/bmp": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.10.3.tgz", - "integrity": "sha512-keMOc5woiDmONXsB/6aXLR4Z5Q+v8lFq3EY2rcj2FmstbDMhRuGbmcBxlEgOqfRjwvtf/wOtJ3Of37oAWtVfLg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.12.0.tgz", + "integrity": "sha512-PjgGVaSQvPrepsD52aTQe6B8A1G/OOYIcpXt6K59AUHQE3s6oNo9lYfyUv96gInBBIMze9s8AgLhMLjU8ijw4Q==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "bmp-js": "^0.1.0", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0", + "bmp-js": "^0.1.0" } }, "@jimp/core": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.10.3.tgz", - "integrity": "sha512-Gd5IpL3U2bFIO57Fh/OA3HCpWm4uW/pU01E75rI03BXfTdz3T+J7TwvyG1XaqsQ7/DSlS99GXtLQPlfFIe28UA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.12.0.tgz", + "integrity": "sha512-xLF8gvRyJSCu08PI01b/MFijxoBoPusJFbSOOzMnP286qVDouxdXQy6CJB3mMosnlZRgp12I+ZgUvMsdJsL8ig==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", + "@jimp/utils": "^0.12.0", "any-base": "^1.1.0", "buffer": "^5.2.0", - "core-js": "^3.4.1", "exif-parser": "^0.1.12", "file-type": "^9.0.0", "load-bmfont": "^1.3.1", @@ -235,323 +233,294 @@ } }, "@jimp/custom": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.10.3.tgz", - "integrity": "sha512-nZmSI+jwTi5IRyNLbKSXQovoeqsw+D0Jn0SxW08wYQvdkiWA8bTlDQFgQ7HVwCAKBm8oKkDB/ZEo9qvHJ+1gAQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.12.0.tgz", + "integrity": "sha512-Rf3p50Jmvy9Aeovs0kyIpd0qbt2peLqDRq6f93AlDkUpB6OZ/rQwgJO8yysNMgI877a3xQz0Tda5j5Lv8AjWgA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/core": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/core": "^0.12.0" } }, "@jimp/gif": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.10.3.tgz", - "integrity": "sha512-vjlRodSfz1CrUvvrnUuD/DsLK1GHB/yDZXHthVdZu23zYJIW7/WrIiD1IgQ5wOMV7NocfrvPn2iqUfBP81/WWA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.12.0.tgz", + "integrity": "sha512-CMapyrH5LGXbl2jHgQA923wHUNbC0LajqMmMHfyFZE9GZFzXULqbTZdRemHXTXn++iruPSR37oVUYi67WG9qmQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/utils": "^0.12.0", "omggif": "^1.0.9" } }, "@jimp/jpeg": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.10.3.tgz", - "integrity": "sha512-AAANwgUZOt6f6P7LZxY9lyJ9xclqutYJlsxt3JbriXUGJgrrFAIkcKcqv1nObgmQASSAQKYaMV9KdHjMlWFKlQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.12.0.tgz", + "integrity": "sha512-jAC9gWPCBJ0ysTZDqDUOVUty3/tk2qStw3N5Vk9W3XZNSTNlLp5xWsiATlkAoSrwoBmdgjf6OfZwqmkFDFVMKw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/utils": "^0.12.0", "jpeg-js": "^0.3.4" } }, "@jimp/plugin-blit": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.10.3.tgz", - "integrity": "sha512-5zlKlCfx4JWw9qUVC7GI4DzXyxDWyFvgZLaoGFoT00mlXlN75SarlDwc9iZ/2e2kp4bJWxz3cGgG4G/WXrbg3Q==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.12.0.tgz", + "integrity": "sha512-csSxB/ZOljGLtvRne+nF1EGpcHZ/6mdGc+trcihClTTLAS5FzX+tySpQj9sHrIzzHtEcILYPMOKaf6KC4LOrfw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-blur": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.10.3.tgz", - "integrity": "sha512-cTOK3rjh1Yjh23jSfA6EHCHjsPJDEGLC8K2y9gM7dnTUK1y9NNmkFS23uHpyjgsWFIoH9oRh2SpEs3INjCpZhQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.12.0.tgz", + "integrity": "sha512-HCL570HvZxhT7Yn/Qqow00sRK0J/E4j1Clwp78vMnQWQ38PONi/Ipyjqp0RLdvCj3tJ3mzrKDqlnN2bbLcKsjQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-circle": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.10.3.tgz", - "integrity": "sha512-51GAPIVelqAcfuUpaM5JWJ0iWl4vEjNXB7p4P7SX5udugK5bxXUjO6KA2qgWmdpHuCKtoNgkzWU9fNSuYp7tCA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.12.0.tgz", + "integrity": "sha512-d+cRlyrM4ylXKk6TuFZcoFz8xsXqLHGfZcX+BDFe9HPz+TTW7AoL5eq8I0uLpTHRD1dLdBPScMejkn3ppLKnjg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-color": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.10.3.tgz", - "integrity": "sha512-RgeHUElmlTH7vpI4WyQrz6u59spiKfVQbsG/XUzfWGamFSixa24ZDwX/yV/Ts+eNaz7pZeIuv533qmKPvw2ujg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.12.0.tgz", + "integrity": "sha512-RmPSwryrmLLtsNluQ9hT73EovM+KcthacDmF7VN/xnJMD/r+vXfgUcDLZDx8yQsd5kdezhtPh7wKihH2+voOwg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/utils": "^0.12.0", "tinycolor2": "^1.4.1" } }, "@jimp/plugin-contain": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.10.3.tgz", - "integrity": "sha512-bYJKW9dqzcB0Ihc6u7jSyKa3juStzbLs2LFr6fu8TzA2WkMS/R8h+ddkiO36+F9ILTWHP0CIA3HFe5OdOGcigw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.12.0.tgz", + "integrity": "sha512-mA1l2GbtmY2uLdCiwzdSJa9tZSyL5uvQwT3UrKDWaPiyhT4+VrCgQVD4CBbOFztI8ToxPcGM9GG4oRuRR2cKDQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-cover": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.10.3.tgz", - "integrity": "sha512-pOxu0cM0BRPzdV468n4dMocJXoMbTnARDY/EpC3ZW15SpMuc/dr1KhWQHgoQX5kVW1Wt8zgqREAJJCQ5KuPKDA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.12.0.tgz", + "integrity": "sha512-rTrGxCBr1dn6DOVF+g8IFUCXHpfOaZCC4kvOyx/GIE3861GqKyOWzjLRcWTVBfgyiuOx+S6kBwpadmnsFO8tHg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-crop": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.10.3.tgz", - "integrity": "sha512-nB7HgOjjl9PgdHr076xZ3Sr6qHYzeBYBs9qvs3tfEEUeYMNnvzgCCGtUl6eMakazZFCMk3mhKmcB9zQuHFOvkg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.12.0.tgz", + "integrity": "sha512-sEz1T7waD5c+nB0aJERipc8/LSaRo4IxPzemOuzWaXxvwdUVRPtM7Rk7XOZmJyc2nW8qPNet+2JkaSjg848xgQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-displace": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.10.3.tgz", - "integrity": "sha512-8t3fVKCH5IVqI4lewe4lFFjpxxr69SQCz5/tlpDLQZsrNScNJivHdQ09zljTrVTCSgeCqQJIKgH2Q7Sk/pAZ0w==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.12.0.tgz", + "integrity": "sha512-VWlTF6TEDdGoN56tnOfsHVNNtsWBHCBmT77G+2k2agbXWAPD5A++bye0y4XP/icAS//sAd7UFzvNQlnT7sIAdg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-dither": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.10.3.tgz", - "integrity": "sha512-JCX/oNSnEg1kGQ8ffZ66bEgQOLCY3Rn+lrd6v1jjLy/mn9YVZTMsxLtGCXpiCDC2wG/KTmi4862ysmP9do9dAQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.12.0.tgz", + "integrity": "sha512-EzhHugll52ngdV1RBh1wmRUjf1jgo2GfU+Zh/a05uLxKGZEDWqGcsfFlI4lZnJbiKUhHCTNwZRCkV2w9gNJ6uw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-fisheye": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.10.3.tgz", - "integrity": "sha512-RRZb1wqe+xdocGcFtj2xHU7sF7xmEZmIa6BmrfSchjyA2b32TGPWKnP3qyj7p6LWEsXn+19hRYbjfyzyebPElQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.12.0.tgz", + "integrity": "sha512-Rz/gboWtY6sow6FC4tg9kG/fNBLopjGRoMmzHVcoQK1XXI2O/tH6nrliHHv3s3AvBBrQ5qPyO5VyCb1vm9xOmA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-flip": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.10.3.tgz", - "integrity": "sha512-0epbi8XEzp0wmSjoW9IB0iMu0yNF17aZOxLdURCN3Zr+8nWPs5VNIMqSVa1Y62GSyiMDpVpKF/ITiXre+EqrPg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.12.0.tgz", + "integrity": "sha512-NFeIHWU95rSmIUnUdHVAYU4dYE3X10qY2peTgbMJ+q1J2qsrUO7w6Gepfd26tA9lh41zwDD5UzuAorpHQ3z27g==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-gaussian": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.10.3.tgz", - "integrity": "sha512-25eHlFbHUDnMMGpgRBBeQ2AMI4wsqCg46sue0KklI+c2BaZ+dGXmJA5uT8RTOrt64/K9Wz5E+2n7eBnny4dfpQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.12.0.tgz", + "integrity": "sha512-LXus5pMzUaIYGTCoWDxRiMb5AW0gJMqet3U6+mQIP7OtSnBL2Vimz9WBbzZuEfKRMCc1l6oDwD/o/fH5ehv+TQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-invert": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.10.3.tgz", - "integrity": "sha512-effYSApWY/FbtlzqsKXlTLkgloKUiHBKjkQnqh5RL4oQxh/33j6aX+HFdDyQKtsXb8CMd4xd7wyiD2YYabTa0g==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.12.0.tgz", + "integrity": "sha512-fkOBCFg9P3Nkc0aFgWt5WgRP41KOs9m8OOnIi4jLnvCamv/Fv8GJLMeDS3gIXuzb/XkS0W/WpMQJmvI1+Zj2xg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-mask": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.10.3.tgz", - "integrity": "sha512-twrg8q8TIhM9Z6Jcu9/5f+OCAPaECb0eKrrbbIajJqJ3bCUlj5zbfgIhiQIzjPJ6KjpnFPSqHQfHkU1Vvk/nVw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.12.0.tgz", + "integrity": "sha512-BWe0n6EB5/b5H062Vybyd2rTkC7yV/DNtNgJiVseZiqJCwmOjZDq+Gx+gKmB3959Th9ipwdEt3nzwBwmyDBVwA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-normalize": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.10.3.tgz", - "integrity": "sha512-xkb5eZI/mMlbwKkDN79+1/t/+DBo8bBXZUMsT4gkFgMRKNRZ6NQPxlv1d3QpRzlocsl6UMxrHnhgnXdLAcgrXw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.12.0.tgz", + "integrity": "sha512-w66beKgxBI1Psv7BmKDxCFJOqAxzn4whVcHgsQ31627HFTelDAf9kSTUOdcIPEwWfWg0tzkKmnHGYQhfJUHYRw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-print": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.10.3.tgz", - "integrity": "sha512-wjRiI6yjXsAgMe6kVjizP+RgleUCLkH256dskjoNvJzmzbEfO7xQw9g6M02VET+emnbY0CO83IkrGm2q43VRyg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.12.0.tgz", + "integrity": "sha512-DdPAmPlTc0rNXRD7efLnCUD2VhYe9kx6h+2mCobGA3AHakrAdJ8qndkWF6UsYxlyrLXxVTftfWKhHOTGOGyA7Q==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/utils": "^0.12.0", "load-bmfont": "^1.4.0" } }, "@jimp/plugin-resize": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.10.3.tgz", - "integrity": "sha512-rf8YmEB1d7Sg+g4LpqF0Mp+dfXfb6JFJkwlAIWPUOR7lGsPWALavEwTW91c0etEdnp0+JB9AFpy6zqq7Lwkq6w==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.12.0.tgz", + "integrity": "sha512-5qqrYmMeSyfNvFb+hdL1XDdGC2Db+/1KwWH9Zw3IxaAB4pXVPmZYMfBi9cJXd1mVHafl+FQWAEy5Ii3hXA32aw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-rotate": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.10.3.tgz", - "integrity": "sha512-YXLlRjm18fkW9MOHUaVAxWjvgZM851ofOipytz5FyKp4KZWDLk+dZK1JNmVmK7MyVmAzZ5jsgSLhIgj+GgN0Eg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.12.0.tgz", + "integrity": "sha512-tOgn86RoFyDm+BJOfdhPXNjaUiaotKcvMzfdR/o4kL/55y+x7xfVj7v7CJbvudnG29bDwEM+3r8HwfaQsezosg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-scale": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.10.3.tgz", - "integrity": "sha512-5DXD7x7WVcX1gUgnlFXQa8F+Q3ThRYwJm+aesgrYvDOY+xzRoRSdQvhmdd4JEEue3lyX44DvBSgCIHPtGcEPaw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.12.0.tgz", + "integrity": "sha512-FS8MWgUcCZ1nwFX4YupTK59nuTqK8seo2CXJeHXgGjl8UU6c/EPBD9SrAuqSNbngcDY9fZ65i6srUyqrQ8kk7w==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-shadow": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.10.3.tgz", - "integrity": "sha512-/nkFXpt2zVcdP4ETdkAUL0fSzyrC5ZFxdcphbYBodqD7fXNqChS/Un1eD4xCXWEpW8cnG9dixZgQgStjywH0Mg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.12.0.tgz", + "integrity": "sha512-FzzTVccC6BkL9Y0rFxI5Di4JEZvCxKq7AyyK6qI7OwBrwxoAmtUodkxGDZTUvYfpmtMZeLWG9TUVrJ/sBQ+NWA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugin-threshold": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.10.3.tgz", - "integrity": "sha512-Dzh0Yq2wXP2SOnxcbbiyA4LJ2luwrdf1MghNIt9H+NX7B+IWw/N8qA2GuSm9n4BPGSLluuhdAWJqHcTiREriVA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.12.0.tgz", + "integrity": "sha512-Sqf2MFDQY/kz0sAPtfjjG4BUcrF58lT09h2EJ75Rdc3hiAWrB7XizLvnI1J8rooHci8Ablbkb/E6xu+52KOGuw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1" + "@jimp/utils": "^0.12.0" } }, "@jimp/plugins": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.10.3.tgz", - "integrity": "sha512-jTT3/7hOScf0EIKiAXmxwayHhryhc1wWuIe3FrchjDjr9wgIGNN2a7XwCgPl3fML17DXK1x8EzDneCdh261bkw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.12.0.tgz", + "integrity": "sha512-P/1vKex4P697ayzVysMSjckcHE2Ii61tyNkq9t1RSZuERgyE616llVKMcil0aVYTnoqapjOwEW36c/fWY8Zj6g==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/plugin-blit": "^0.10.3", - "@jimp/plugin-blur": "^0.10.3", - "@jimp/plugin-circle": "^0.10.3", - "@jimp/plugin-color": "^0.10.3", - "@jimp/plugin-contain": "^0.10.3", - "@jimp/plugin-cover": "^0.10.3", - "@jimp/plugin-crop": "^0.10.3", - "@jimp/plugin-displace": "^0.10.3", - "@jimp/plugin-dither": "^0.10.3", - "@jimp/plugin-fisheye": "^0.10.3", - "@jimp/plugin-flip": "^0.10.3", - "@jimp/plugin-gaussian": "^0.10.3", - "@jimp/plugin-invert": "^0.10.3", - "@jimp/plugin-mask": "^0.10.3", - "@jimp/plugin-normalize": "^0.10.3", - "@jimp/plugin-print": "^0.10.3", - "@jimp/plugin-resize": "^0.10.3", - "@jimp/plugin-rotate": "^0.10.3", - "@jimp/plugin-scale": "^0.10.3", - "@jimp/plugin-shadow": "^0.10.3", - "@jimp/plugin-threshold": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/plugin-blit": "^0.12.0", + "@jimp/plugin-blur": "^0.12.0", + "@jimp/plugin-circle": "^0.12.0", + "@jimp/plugin-color": "^0.12.0", + "@jimp/plugin-contain": "^0.12.0", + "@jimp/plugin-cover": "^0.12.0", + "@jimp/plugin-crop": "^0.12.0", + "@jimp/plugin-displace": "^0.12.0", + "@jimp/plugin-dither": "^0.12.0", + "@jimp/plugin-fisheye": "^0.12.0", + "@jimp/plugin-flip": "^0.12.0", + "@jimp/plugin-gaussian": "^0.12.0", + "@jimp/plugin-invert": "^0.12.0", + "@jimp/plugin-mask": "^0.12.0", + "@jimp/plugin-normalize": "^0.12.0", + "@jimp/plugin-print": "^0.12.0", + "@jimp/plugin-resize": "^0.12.0", + "@jimp/plugin-rotate": "^0.12.0", + "@jimp/plugin-scale": "^0.12.0", + "@jimp/plugin-shadow": "^0.12.0", + "@jimp/plugin-threshold": "^0.12.0", "timm": "^1.6.1" } }, "@jimp/png": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.10.3.tgz", - "integrity": "sha512-YKqk/dkl+nGZxSYIDQrqhmaP8tC3IK8H7dFPnnzFVvbhDnyYunqBZZO3SaZUKTichClRw8k/CjBhbc+hifSGWg==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.12.0.tgz", + "integrity": "sha512-5MgVBRhjkivIHy7cJ6QnU4CygndSde0ZMcaVkfBIyh6gd8pCcIG/XbY2TcR9lSkflgw3tUVzLrFR1xWUYr2trg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/utils": "^0.12.0", "pngjs": "^3.3.3" } }, "@jimp/tiff": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.10.3.tgz", - "integrity": "sha512-7EsJzZ5Y/EtinkBGuwX3Bi4S+zgbKouxjt9c82VJTRJOQgLWsE/RHqcyRCOQBhHAZ9QexYmDz34medfLKdoX0g==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.12.0.tgz", + "integrity": "sha512-h7HBCSjTA4YlnWx66qxQh9YxuzxMoBSGkTiUDEhao2BIhYa2pRmRwtMfqp1EdeRYcXkswWpn4qZAr7zY1TlIGw==", "requires": { "@babel/runtime": "^7.7.2", - "core-js": "^3.4.1", "utif": "^2.0.1" } }, "@jimp/types": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.10.3.tgz", - "integrity": "sha512-XGmBakiHZqseSWr/puGN+CHzx0IKBSpsKlmEmsNV96HKDiP6eu8NSnwdGCEq2mmIHe0JNcg1hqg59hpwtQ7Tiw==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.12.0.tgz", + "integrity": "sha512-6avU1n9lY4vpAHjKSQqrLbk6L5PCNFORre+T1Rcyvv/CGQKxVIAuRj1w+RzXClob8MEmvI17OI3R2w5RCbYpQw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/bmp": "^0.10.3", - "@jimp/gif": "^0.10.3", - "@jimp/jpeg": "^0.10.3", - "@jimp/png": "^0.10.3", - "@jimp/tiff": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/bmp": "^0.12.0", + "@jimp/gif": "^0.12.0", + "@jimp/jpeg": "^0.12.0", + "@jimp/png": "^0.12.0", + "@jimp/tiff": "^0.12.0", "timm": "^1.6.1" } }, "@jimp/utils": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.10.3.tgz", - "integrity": "sha512-VcSlQhkil4ReYmg1KkN+WqHyYfZ2XfZxDsKAHSfST1GEz/RQHxKZbX+KhFKtKflnL0F4e6DlNQj3vznMNXCR2w==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.12.0.tgz", + "integrity": "sha512-MVoR31cQ6QRXHQI+qS9po7sr1LQTOOpQHE9I2oVeakcDkVX80xrRBif3WoNPvq3BG2+BDxt09CFwwHFHHFY49Q==", "requires": { "@babel/runtime": "^7.7.2", - "core-js": "^3.4.1", "regenerator-runtime": "^0.13.3" } }, @@ -2644,7 +2613,9 @@ "core-js": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.4.4.tgz", - "integrity": "sha512-vKea1DrcLA80Hrfc7AQgfoGvEaByfR5mH08t+zuWOWY94TFBmabdEL56mUbcijvadG9RxsXR2gUUFrfj4/iTcA==" + "integrity": "sha512-vKea1DrcLA80Hrfc7AQgfoGvEaByfR5mH08t+zuWOWY94TFBmabdEL56mUbcijvadG9RxsXR2gUUFrfj4/iTcA==", + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -2832,9 +2803,9 @@ "integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI=" }, "dayjs": { - "version": "1.8.26", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.26.tgz", - "integrity": "sha512-KqtAuIfdNfZR5sJY1Dixr2Is4ZvcCqhb0dZpCOt5dGEFiMzoIbjkTSzUb4QKTCsP+WNpGwUjAFIZrnZvUxxkhw==" + "version": "1.8.27", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.27.tgz", + "integrity": "sha512-Jpa2acjWIeOkg8KURUHICk0EqnEFSSF5eMEscsOgyJ92ZukXwmpmRkPSUka7KHSfbj5eKH30ieosYip+ky9emQ==" }, "debug": { "version": "4.1.1", @@ -3337,9 +3308,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "ejs": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.2.tgz", - "integrity": "sha512-zFuywxrAWtX5Mk2KAuoJNkXXbfezpNA0v7i+YC971QORguPekpjpAgeOv99YWSdKXwj7JxI2QAWDeDkE8fWtXw==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.3.tgz", + "integrity": "sha512-wmtrUGyfSC23GC/B1SMv2ogAUgbQEtDmTIhfqielrG5ExIM9TP4UoYdi90jLF1aTcsWCJNEO0UrgKzP0y3nTSg==", "requires": { "jake": "^10.6.1" } @@ -4448,9 +4419,9 @@ } }, "file-type": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-14.3.0.tgz", - "integrity": "sha512-s71v6jMkbfwVdj87csLeNpL5K93mv4lN+lzgzifoICtPHhnXokDwBa3jrzfg+z6FK872iYJ0vS0i74v8XmoFDA==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-14.4.0.tgz", + "integrity": "sha512-U5Q2lHPcERmBsg+DpS/+0r+g7PCsJmyW+aggHnGbMimCyNCpIerLv/VzHJHqtc0O91AXr4Puz4DL7LzA5hMdwA==", "requires": { "readable-web-to-node-stream": "^2.0.0", "strtok3": "^6.0.0", @@ -6114,9 +6085,12 @@ "integrity": "sha1-1LVcafUYhvm2XHDWwmItN+KfSP4=" }, "is-wsl": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.1.1.tgz", - "integrity": "sha512-umZHcSrwlDHo2TGMXv0DZ8dIUGunZ2Iv68YZnrmCiBPkZ4aaOhtv7pXJKeki9k3qJ3RJr0cDyitcl5wEH3AYog==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "requires": { + "is-docker": "^2.0.0" + } }, "is-yarn-global": { "version": "0.3.0", @@ -6216,6 +6190,38 @@ } } }, + "jasmine": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.5.0.tgz", + "integrity": "sha512-DYypSryORqzsGoMazemIHUfMkXM7I7easFaxAvNM3Mr6Xz3Fy36TupTrAOxZWN8MVKEU5xECv22J4tUQf3uBzQ==", + "dev": true, + "requires": { + "glob": "^7.1.4", + "jasmine-core": "~3.5.0" + }, + "dependencies": { + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, + "jasmine-core": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.5.0.tgz", + "integrity": "sha512-nCeAiw37MIMA9w9IXso7bRaLl+c/ef3wnxsoSAlYrzS+Ot0zTG6nU8G/cIfGkqpkjX2wNaIW9RFG0TwIFnG6bA==", + "dev": true + }, "jest-worker": { "version": "25.5.0", "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-25.5.0.tgz", @@ -6244,15 +6250,14 @@ } }, "jimp": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.10.3.tgz", - "integrity": "sha512-meVWmDMtyUG5uYjFkmzu0zBgnCvvxwWNi27c4cg55vWNVC9ES4Lcwb+ogx+uBBQE3Q+dLKjXaLl0JVW+nUNwbQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.12.0.tgz", + "integrity": "sha512-8QD1QNk2ZpoSFLDEQn4rlQ0sDAO1z6UagIqUsH6YjopHCExcAbk3q2hJFXk6wSf+LMHHkic44PhdVTZ0drER2w==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/custom": "^0.10.3", - "@jimp/plugins": "^0.10.3", - "@jimp/types": "^0.10.3", - "core-js": "^3.4.1", + "@jimp/custom": "^0.12.0", + "@jimp/plugins": "^0.12.0", + "@jimp/types": "^0.12.0", "regenerator-runtime": "^0.13.3" } }, @@ -8231,9 +8236,9 @@ "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=" }, "open": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/open/-/open-7.0.3.tgz", - "integrity": "sha512-sP2ru2v0P290WFfv49Ap8MF6PkzGNnGlAwHweB4WR4mr5d2d0woiCluUeJ218w7/+PmoBy9JmYgD5A4mLcWOFA==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/open/-/open-7.0.4.tgz", + "integrity": "sha512-brSA+/yq+b08Hsr4c8fsEW2CRzk1BmfN3SAK/5VCHQ9bdoZJ4qa/+AfR0xHjlbbZUyPkUHs1b8x1RqdyZdkVqQ==", "requires": { "is-docker": "^2.0.0", "is-wsl": "^2.1.1" diff --git a/package.json b/package.json index 65fd8a95c..d9aaaff44 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "build-backend-docs": "./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/backend_api src/entities/*.js src/services/backend_script_api.js", "build-frontend-docs": "./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/collapsible_widget.js", "build-docs": "npm run build-backend-docs && npm run build-frontend-docs", - "webpack": "npx webpack -c webpack-desktop.config.js && npx webpack -c webpack-mobile.config.js && npx webpack -c webpack-setup.config.js" + "webpack": "npx webpack -c webpack-desktop.config.js && npx webpack -c webpack-mobile.config.js && npx webpack -c webpack-setup.config.js", + "test": "jasmine" }, "dependencies": { "async-mutex": "0.2.2", @@ -28,16 +29,16 @@ "commonmark": "0.29.1", "cookie-parser": "1.4.5", "csurf": "1.11.0", - "dayjs": "1.8.26", + "dayjs": "1.8.27", "debug": "4.1.1", - "ejs": "3.1.2", + "ejs": "3.1.3", "electron-debug": "3.0.1", "electron-dl": "3.0.0", "electron-find": "1.0.6", "electron-window-state": "5.0.3", "express": "4.17.1", "express-session": "1.17.1", - "file-type": "14.3.0", + "file-type": "14.4.0", "fs-extra": "9.0.0", "helmet": "3.22.0", "html": "1.0.0", @@ -51,11 +52,11 @@ "imagemin-pngquant": "8.0.0", "ini": "1.3.5", "is-svg": "4.2.1", - "jimp": "0.10.3", + "jimp": "0.12.0", "mime-types": "2.1.27", "multer": "1.4.2", "node-abi": "2.16.0", - "open": "7.0.3", + "open": "7.0.4", "portscanner": "2.2.0", "rand-token": "1.0.1", "rcedit": "2.1.1", @@ -82,6 +83,7 @@ "electron-builder": "22.6.0", "electron-packager": "14.2.1", "electron-rebuild": "1.11.0", + "jasmine": "^3.5.0", "jsdoc": "3.6.4", "lorem-ipsum": "2.0.3", "webpack": "5.0.0-beta.16", diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js new file mode 100644 index 000000000..de0f81370 --- /dev/null +++ b/spec/lexer.spec.js @@ -0,0 +1,29 @@ +const lexerSpec = require('../src/services/search/lexer.js'); + +describe("Lexer", function() { + it("simple lexing", () => { + expect(lexerSpec("hello world").fulltextTokens) + .toEqual(["hello", "world"]); + }); + + it("use quotes to keep words together", () => { + expect(lexerSpec("'hello world' my friend").fulltextTokens) + .toEqual(["hello world", "my", "friend"]); + + expect(lexerSpec('"hello world" my friend').fulltextTokens) + .toEqual(["hello world", "my", "friend"]); + + expect(lexerSpec('`hello world` my friend').fulltextTokens) + .toEqual(["hello world", "my", "friend"]); + }); + + it("you can use different quotes and other special characters inside quotes", () => { + expect(lexerSpec("'I can use \" or ` or #@=*' without problem").fulltextTokens) + .toEqual(["I can use \" or ` or #@=*", "without", "problem"]); + }); + + it("if quote is not ended then it's just one long token", () => { + expect(lexerSpec("'unfinished quote").fulltextTokens) + .toEqual(["unfinished quote"]); + }); +}); diff --git a/spec/support/jasmine.json b/spec/support/jasmine.json new file mode 100644 index 000000000..370fc4464 --- /dev/null +++ b/spec/support/jasmine.json @@ -0,0 +1,11 @@ +{ + "spec_dir": "spec", + "spec_files": [ + "**/*[sS]pec.js" + ], + "helpers": [ + "helpers/**/*.js" + ], + "stopSpecOnExpectationFailure": false, + "random": true +} diff --git a/src/services/search.js b/src/services/search.js index 72f9b3327..774614ca8 100644 --- a/src/services/search.js +++ b/src/services/search.js @@ -1,7 +1,7 @@ const repository = require('./repository'); const sql = require('./sql'); const log = require('./log'); -const parseFilters = require('./parse_filters'); +const parseFilters = require('./search/parse_filters.js'); const buildSearchQuery = require('./build_search_query'); const noteCacheService = require('./note_cache/note_cache.js'); diff --git a/src/services/search/expressions/not.js b/src/services/search/expressions/not.js new file mode 100644 index 000000000..22d8ebee4 --- /dev/null +++ b/src/services/search/expressions/not.js @@ -0,0 +1,15 @@ +"use strict"; + +class NotExp { + constructor(subExpression) { + this.subExpression = subExpression; + } + + execute(noteSet, searchContext) { + const subNoteSet = this.subExpression.execute(noteSet, searchContext); + + return noteSet.minus(subNoteSet); + } +} + +module.exports = NotExp; diff --git a/src/services/search/lexer.js b/src/services/search/lexer.js new file mode 100644 index 000000000..02510079d --- /dev/null +++ b/src/services/search/lexer.js @@ -0,0 +1,94 @@ +function lexer(str) { + const fulltextTokens = []; + const expressionTokens = []; + + let quotes = false; + let fulltextEnded = false; + let currentWord = ''; + let symbol = false; + + function isSymbol(chr) { + return ['=', '*', '>', '<', '!'].includes(chr); + } + + function finishWord() { + if (currentWord === '') { + return; + } + + if (fulltextEnded) { + expressionTokens.push(currentWord); + } else { + fulltextTokens.push(currentWord); + } + + currentWord = ''; + } + + for (let i = 0; i < str.length; i++) { + const chr = str[i]; + + if (chr === '\\') { + if ((i + 1) < str.length) { + i++; + + currentWord += str[i]; + } + else { + currentWord += chr; + } + + continue; + } + else if (['"', "'", '`'].includes(chr)) { + if (!quotes) { + if (currentWord.length === 0) { + quotes = chr; + } + else { + // quote inside a word does not have special meening and does not break word + // e.g. d'Artagnan is kept as a single token + currentWord += chr; + } + } + else if (quotes === chr) { + quotes = false; + + finishWord(); + } + else { + // it's a quote but within other kind of quotes so it's valid as a literal character + currentWord += chr; + } + continue; + } + else if (!quotes) { + if (chr === '#' || chr === '@') { + fulltextEnded = true; + continue; + } + else if (chr === ' ') { + finishWord(); + continue; + } + else if (fulltextEnded && symbol !== isSymbol(chr)) { + finishWord(); + + currentWord += chr; + symbol = isSymbol(chr); + continue; + } + } + + currentWord += chr; + } + + finishWord(); + + return { + fulltextTokens, + expressionTokens + } +} + +module.exports = lexer; diff --git a/src/services/search/note_set.js b/src/services/search/note_set.js index da768854d..287f2f6bd 100644 --- a/src/services/search/note_set.js +++ b/src/services/search/note_set.js @@ -21,6 +21,18 @@ class NoteSet { mergeIn(anotherNoteSet) { this.notes = this.notes.concat(anotherNoteSet.arr); } + + minus(anotherNoteSet) { + const newNoteSet = new NoteSet(); + + for (const note of this.notes) { + if (!anotherNoteSet.hasNoteId(note.noteId)) { + newNoteSet.add(note); + } + } + + return newNoteSet; + } } module.exports = NoteSet; diff --git a/src/services/parse_filters.js b/src/services/search/parse_filters.js similarity index 92% rename from src/services/parse_filters.js rename to src/services/search/parse_filters.js index e21b67c09..472a938d0 100644 --- a/src/services/parse_filters.js +++ b/src/services/search/parse_filters.js @@ -1,4 +1,9 @@ const dayjs = require("dayjs"); +const AndExp = require('./expressions/and'); +const OrExp = require('./expressions/or'); +const NotExp = require('./expressions/not'); +const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); +const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); const filterRegex = /(\b(AND|OR)\s+)?@(!?)([\p{L}\p{Number}_]+|"[^"]+")\s*((=|!=|<|<=|>|>=|!?\*=|!?=\*|!?\*=\*)\s*([^\s=*"]+|"[^"]+"))?/igu; const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i; From 81bf84f2deea721bd0e92a0f56f242ec21691174 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 17 May 2020 23:14:24 +0200 Subject: [PATCH 19/47] lexer fixes + tests --- spec/lexer.spec.js | 36 ++++++++++++++++++++++++++++++++++-- src/services/search/lexer.js | 31 +++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js index de0f81370..e24235594 100644 --- a/spec/lexer.spec.js +++ b/spec/lexer.spec.js @@ -1,6 +1,6 @@ -const lexerSpec = require('../src/services/search/lexer.js'); +const lexerSpec = require('../src/services/search/lexer'); -describe("Lexer", function() { +describe("Lexer fulltext", () => { it("simple lexing", () => { expect(lexerSpec("hello world").fulltextTokens) .toEqual(["hello", "world"]); @@ -26,4 +26,36 @@ describe("Lexer", function() { expect(lexerSpec("'unfinished quote").fulltextTokens) .toEqual(["unfinished quote"]); }); + + it("parenthesis and symbols in fulltext section are just normal characters", () => { + expect(lexerSpec("what's u=p ").fulltextTokens) + .toEqual(["what's", "u=p", ""]); + }); + + it("escaping special characters", () => { + expect(lexerSpec("hello \\#\\@\\'").fulltextTokens) + .toEqual(["hello", "#@'"]); + }); +}); + +describe("Lexer expression", () => { + it("simple attribute existence", () => { + expect(lexerSpec("#label @relation").expressionTokens) + .toEqual(["#label", "@relation"]); + }); + + it("simple label operators", () => { + expect(lexerSpec("#label*=*text").expressionTokens) + .toEqual(["#label", "*=*", "text"]); + }); + + it("spaces in attribute names and values", () => { + expect(lexerSpec(`#'long label'="hello o' world" @'long relation'`).expressionTokens) + .toEqual(["#long label", "=", "hello o' world", "@long relation"]); + }); + + it("complex expressions with and, or and parenthesis", () => { + expect(lexerSpec(`# (#label=text OR #second=text) AND @relation`).expressionTokens) + .toEqual(["#", "(", "#label", "=", "text", "OR", "#second", "=", "text", ")", "AND", "@relation"]); + }); }); diff --git a/src/services/search/lexer.js b/src/services/search/lexer.js index 02510079d..301355d30 100644 --- a/src/services/search/lexer.js +++ b/src/services/search/lexer.js @@ -5,12 +5,20 @@ function lexer(str) { let quotes = false; let fulltextEnded = false; let currentWord = ''; - let symbol = false; - function isSymbol(chr) { + function isOperatorSymbol(chr) { return ['=', '*', '>', '<', '!'].includes(chr); } + function previusOperatorSymbol() { + if (currentWord.length === 0) { + return false; + } + else { + return isOperatorSymbol(currentWord[currentWord.length - 1]); + } + } + function finishWord() { if (currentWord === '') { return; @@ -42,7 +50,11 @@ function lexer(str) { } else if (['"', "'", '`'].includes(chr)) { if (!quotes) { - if (currentWord.length === 0) { + if (currentWord.length === 0 || fulltextEnded) { + if (previusOperatorSymbol()) { + finishWord(); + } + quotes = chr; } else { @@ -63,19 +75,26 @@ function lexer(str) { continue; } else if (!quotes) { - if (chr === '#' || chr === '@') { + if (currentWord.length === 0 && (chr === '#' || chr === '@')) { fulltextEnded = true; + currentWord = chr; + continue; } else if (chr === ' ') { finishWord(); continue; } - else if (fulltextEnded && symbol !== isSymbol(chr)) { + else if (fulltextEnded && ['(', ')'].includes(chr)) { + finishWord(); + currentWord += chr; + finishWord(); + continue; + } + else if (fulltextEnded && previusOperatorSymbol() !== isOperatorSymbol(chr)) { finishWord(); currentWord += chr; - symbol = isSymbol(chr); continue; } } From b72dc977e63b37135fd1702ac04d129ecc95cd4e Mon Sep 17 00:00:00 2001 From: zadam Date: Tue, 19 May 2020 00:00:35 +0200 Subject: [PATCH 20/47] parens handler + parser in progress --- spec/lexer.spec.js | 26 ++++---- spec/parens.spec.js | 21 ++++++ src/services/search/expressions/and.js | 9 +++ src/services/search/expressions/equals.js | 3 +- src/services/search/expressions/or.js | 2 + src/services/search/parens.js | 43 ++++++++++++ src/services/search/parser.js | 81 +++++++++++++++++++++++ 7 files changed, 171 insertions(+), 14 deletions(-) create mode 100644 spec/parens.spec.js create mode 100644 src/services/search/parens.js create mode 100644 src/services/search/parser.js diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js index e24235594..14f4314fb 100644 --- a/spec/lexer.spec.js +++ b/spec/lexer.spec.js @@ -1,61 +1,61 @@ -const lexerSpec = require('../src/services/search/lexer'); +const lexer = require('../src/services/search/lexer'); describe("Lexer fulltext", () => { it("simple lexing", () => { - expect(lexerSpec("hello world").fulltextTokens) + expect(lexer("hello world").fulltextTokens) .toEqual(["hello", "world"]); }); it("use quotes to keep words together", () => { - expect(lexerSpec("'hello world' my friend").fulltextTokens) + expect(lexer("'hello world' my friend").fulltextTokens) .toEqual(["hello world", "my", "friend"]); - expect(lexerSpec('"hello world" my friend').fulltextTokens) + expect(lexer('"hello world" my friend').fulltextTokens) .toEqual(["hello world", "my", "friend"]); - expect(lexerSpec('`hello world` my friend').fulltextTokens) + expect(lexer('`hello world` my friend').fulltextTokens) .toEqual(["hello world", "my", "friend"]); }); it("you can use different quotes and other special characters inside quotes", () => { - expect(lexerSpec("'I can use \" or ` or #@=*' without problem").fulltextTokens) + expect(lexer("'I can use \" or ` or #@=*' without problem").fulltextTokens) .toEqual(["I can use \" or ` or #@=*", "without", "problem"]); }); it("if quote is not ended then it's just one long token", () => { - expect(lexerSpec("'unfinished quote").fulltextTokens) + expect(lexer("'unfinished quote").fulltextTokens) .toEqual(["unfinished quote"]); }); it("parenthesis and symbols in fulltext section are just normal characters", () => { - expect(lexerSpec("what's u=p ").fulltextTokens) + expect(lexer("what's u=p ").fulltextTokens) .toEqual(["what's", "u=p", ""]); }); it("escaping special characters", () => { - expect(lexerSpec("hello \\#\\@\\'").fulltextTokens) + expect(lexer("hello \\#\\@\\'").fulltextTokens) .toEqual(["hello", "#@'"]); }); }); describe("Lexer expression", () => { it("simple attribute existence", () => { - expect(lexerSpec("#label @relation").expressionTokens) + expect(lexer("#label @relation").expressionTokens) .toEqual(["#label", "@relation"]); }); it("simple label operators", () => { - expect(lexerSpec("#label*=*text").expressionTokens) + expect(lexer("#label*=*text").expressionTokens) .toEqual(["#label", "*=*", "text"]); }); it("spaces in attribute names and values", () => { - expect(lexerSpec(`#'long label'="hello o' world" @'long relation'`).expressionTokens) + expect(lexer(`#'long label'="hello o' world" @'long relation'`).expressionTokens) .toEqual(["#long label", "=", "hello o' world", "@long relation"]); }); it("complex expressions with and, or and parenthesis", () => { - expect(lexerSpec(`# (#label=text OR #second=text) AND @relation`).expressionTokens) + expect(lexer(`# (#label=text OR #second=text) AND @relation`).expressionTokens) .toEqual(["#", "(", "#label", "=", "text", "OR", "#second", "=", "text", ")", "AND", "@relation"]); }); }); diff --git a/spec/parens.spec.js b/spec/parens.spec.js new file mode 100644 index 000000000..8c7db6d56 --- /dev/null +++ b/spec/parens.spec.js @@ -0,0 +1,21 @@ +const parens = require('../src/services/search/parens'); + +describe("Parens handler", () => { + it("handles parens", () => {console.log(parens(["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"])) + expect(parens(["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"])) + .toEqual([ + [ + "hello" + ], + "and", + [ + [ + "pick", + "one" + ], + "and", + "another" + ] + ]); + }); +}); diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index f542d416c..44e19c0af 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -5,6 +5,15 @@ class AndExp { this.subExpressions = subExpressions; } + static of(subExpressions) { + if (subExpressions.length === 1) { + return subExpressions[0]; + } + else { + return new AndExp(subExpressions); + } + } + execute(noteSet, searchContext) { for (const subExpression of this.subExpressions) { noteSet = subExpression.execute(noteSet, searchContext); diff --git a/src/services/search/expressions/equals.js b/src/services/search/expressions/equals.js index e5e04b3d8..ecae22241 100644 --- a/src/services/search/expressions/equals.js +++ b/src/services/search/expressions/equals.js @@ -4,9 +4,10 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); class EqualsExp { - constructor(attributeType, attributeName, attributeValue) { + constructor(attributeType, attributeName, operator, attributeValue) { this.attributeType = attributeType; this.attributeName = attributeName; + this.operator = operator; this.attributeValue = attributeValue; } diff --git a/src/services/search/expressions/or.js b/src/services/search/expressions/or.js index c17dd2210..a48bb8bc8 100644 --- a/src/services/search/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -1,5 +1,7 @@ "use strict"; +const NoteSet = require('../note_set'); + class OrExp { constructor(subExpressions) { this.subExpressions = subExpressions; diff --git a/src/services/search/parens.js b/src/services/search/parens.js new file mode 100644 index 000000000..7f402d2b5 --- /dev/null +++ b/src/services/search/parens.js @@ -0,0 +1,43 @@ +/** + * This will create a recursive object from list of tokens - tokens between parenthesis are grouped in a single array + */ +function parens(tokens) { + if (tokens.length === 0) { + throw new Error("Empty expression."); + } + + while (true) { + const leftIdx = tokens.findIndex(token => token === '('); + + if (leftIdx === -1) { + return tokens; + } + + let rightIdx; + let parensLevel = 0 + + for (rightIdx = leftIdx; rightIdx < tokens.length; rightIdx++) { + if (tokens[rightIdx] === ')') { + parensLevel--; + + if (parensLevel === 0) { + break; + } + } else if (tokens[rightIdx] === '(') { + parensLevel++; + } + } + + if (rightIdx >= tokens.length) { + throw new Error("Did not find matching right parenthesis."); + } + + tokens = [ + ...tokens.slice(0, leftIdx), + parens(tokens.slice(leftIdx + 1, rightIdx)), + ...tokens.slice(rightIdx + 1) + ]; + } +} + +module.exports = parens; diff --git a/src/services/search/parser.js b/src/services/search/parser.js new file mode 100644 index 000000000..8ad021f89 --- /dev/null +++ b/src/services/search/parser.js @@ -0,0 +1,81 @@ +const AndExp = require('./expressions/and'); +const OrExp = require('./expressions/or'); +const NotExp = require('./expressions/not'); +const ExistsExp = require('./expressions/exists'); +const EqualsExp = require('./expressions/equals'); +const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); +const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); + +function getFulltext(tokens, includingNoteContent) { + if (includingNoteContent) { + return [ + new OrExp([ + new NoteCacheFulltextExp(tokens), + new NoteContentFulltextExp(tokens) + ]) + ] + } + else { + return [ + new NoteCacheFulltextExp(tokens) + ] + } +} + +function isOperator(str) { + return str.matches(/^[=<>*]+$/); +} + +function getExpressions(tokens) { + const expressions = []; + let op = null; + + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]; + + if (token === '#' || token === '@') { + continue; + } + + if (Array.isArray(token)) { + expressions.push(getExpressions(token)); + } + else if (token.startsWith('#') || token.startsWith('@')) { + const type = token.startsWith('#') ? 'label' : 'relation'; + + if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { + expressions.push(new EqualsExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); + + i += 2; + } + else { + expressions.push(new ExistsExp(type, token.substr(1))); + } + } + else if (['and', 'or'].includes(token.toLowerCase())) { + if (!op) { + op = token.toLowerCase(); + } + else if (op !== token.toLowerCase()) { + throw new Error('Mixed usage of AND/OR - always use parenthesis to group AND/OR expressions.'); + } + } + else if (isOperator(token)) { + throw new Error(`Misplaced or incomplete expression "${token}"`); + } + else { + throw new Error(`Unrecognized expression "${token}"`); + } + + if (!op && expressions.length > 1) { + op = 'and'; + } + } +} + +function parse(fulltextTokens, expressionTokens, includingNoteContent) { + return AndExp.of([ + ...getFulltext(fulltextTokens, includingNoteContent), + ...getExpressions(expressionTokens) + ]); +} From 99aa481acea6634892b467e228c40cf2bb3f87ee Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 20 May 2020 00:03:33 +0200 Subject: [PATCH 21/47] refactoring for testing parser --- spec/parens.spec.js | 2 +- spec/parser.spec.js | 10 ++ src/public/app/services/utils.js | 4 +- src/services/note_cache/entities/note.js | 1 - src/services/note_cache/note_cache.js | 163 +---------------- src/services/note_cache/note_cache_loader.js | 169 ++++++++++++++++++ src/services/note_cache/note_cache_service.js | 2 + .../{exists.js => attribute_exists.js} | 4 +- .../{equals.js => field_comparison.js} | 4 +- .../search/expressions/note_cache_fulltext.js | 4 +- .../expressions/note_content_fulltext.js | 2 + src/services/search/parser.js | 12 +- src/services/sql_init.js | 2 +- 13 files changed, 204 insertions(+), 175 deletions(-) create mode 100644 spec/parser.spec.js create mode 100644 src/services/note_cache/note_cache_loader.js rename src/services/search/expressions/{exists.js => attribute_exists.js} (93%) rename src/services/search/expressions/{equals.js => field_comparison.js} (94%) diff --git a/spec/parens.spec.js b/spec/parens.spec.js index 8c7db6d56..c55896e44 100644 --- a/spec/parens.spec.js +++ b/spec/parens.spec.js @@ -1,7 +1,7 @@ const parens = require('../src/services/search/parens'); describe("Parens handler", () => { - it("handles parens", () => {console.log(parens(["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"])) + it("handles parens", () => { expect(parens(["(", "hello", ")", "and", "(", "(", "pick", "one", ")", "and", "another", ")"])) .toEqual([ [ diff --git a/spec/parser.spec.js b/spec/parser.spec.js new file mode 100644 index 000000000..7d9ad2a55 --- /dev/null +++ b/spec/parser.spec.js @@ -0,0 +1,10 @@ +const parser = require('../src/services/search/parser'); + +describe("Parser", () => { + it("fulltext parser without content", () => { + const exps = parser(["hello", "hi"], [], false); + + expect(exps.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(exps.tokens).toEqual(["hello", "hi"]); + }); +}); diff --git a/src/public/app/services/utils.js b/src/public/app/services/utils.js index 0fa8e9642..504f5ae45 100644 --- a/src/public/app/services/utils.js +++ b/src/public/app/services/utils.js @@ -187,7 +187,7 @@ function setCookie(name, value) { } function setSessionCookie(name, value) { - document.cookie = name + "=" + (value || "") + ";"; + document.cookie = name + "=" + (value || "") + "; SameSite=Strict"; } function getCookie(name) { @@ -356,4 +356,4 @@ export default { copySelectionToClipboard, isCKEditorInitialized, dynamicRequire -}; \ No newline at end of file +}; diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index adc120d15..8d3a7abc2 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -1,6 +1,5 @@ "use strict"; -const noteCache = require('../note_cache'); const protectedSessionService = require('../../protected_session'); class Note { diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index 8aaa688bc..12942ee7f 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -3,9 +3,6 @@ const Note = require('./entities/note'); const Branch = require('./entities/branch'); const Attribute = require('./entities/attribute'); -const sql = require('../sql.js'); -const sqlInit = require('../sql_init.js'); -const eventService = require('../events.js'); class NoteCache { constructor() { @@ -21,7 +18,8 @@ class NoteCache { this.attributeIndex = null; this.loaded = false; - this.loadedPromise = this.load(); + this.loadedResolve = null; + this.loadedPromise = new Promise(res => {this.loadedResolve = res;}); } /** @return {Attribute[]} */ @@ -29,36 +27,6 @@ class NoteCache { return this.attributeIndex[`${type}-${name}`] || []; } - async load() { - await sqlInit.dbReady; - - this.notes = await this.getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, - row => new Note(this, row)); - - this.branches = await this.getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, - row => new Branch(this, row)); - - this.attributeIndex = []; - - this.attributes = await this.getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, - row => new Attribute(this, row)); - - this.loaded = true; - } - - async getMappedRows(query, cb) { - const map = {}; - const results = await sql.getRows(query, []); - - for (const row of results) { - const keys = Object.keys(row); - - map[row[keys[0]]] = cb(row); - } - - return map; - } - decryptProtectedNotes() { for (const note of Object.values(this.notes)) { note.decrypt(); @@ -72,131 +40,4 @@ class NoteCache { const noteCache = new NoteCache(); -eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { - // note that entity can also be just POJO without methods if coming from sync - - if (!noteCache.loaded) { - return; - } - - if (entityName === 'notes') { - const {noteId} = entity; - - if (entity.isDeleted) { - delete noteCache.notes[noteId]; - } - else if (noteId in noteCache.notes) { - const note = noteCache.notes[noteId]; - - // we can assume we have protected session since we managed to update - note.title = entity.title; - note.isProtected = entity.isProtected; - note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; - note.flatTextCache = null; - - note.decrypt(); - } - else { - const note = new Note(entity); - noteCache.notes[noteId] = note; - - note.decrypt(); - } - } - else if (entityName === 'branches') { - const {branchId, noteId, parentNoteId} = entity; - const childNote = noteCache.notes[noteId]; - - if (entity.isDeleted) { - if (childNote) { - childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); - childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); - - if (childNote.parents.length > 0) { - childNote.invalidateSubtreeCaches(); - } - } - - const parentNote = noteCache.notes[parentNoteId]; - - if (parentNote) { - parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); - } - - delete noteCache.childParentToBranch[`${noteId}-${parentNoteId}`]; - delete noteCache.branches[branchId]; - } - else if (branchId in noteCache.branches) { - // only relevant thing which can change in a branch is prefix - noteCache.branches[branchId].prefix = entity.prefix; - - if (childNote) { - childNote.flatTextCache = null; - } - } - else { - noteCache.branches[branchId] = new Branch(entity); - - if (childNote) { - childNote.resortParents(); - } - } - } - else if (entityName === 'attributes') { - const {attributeId, noteId} = entity; - const note = noteCache.notes[noteId]; - const attr = noteCache.attributes[attributeId]; - - if (entity.isDeleted) { - if (note && attr) { - // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeCaches(); - } - - note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); - - const targetNote = attr.targetNote; - - if (targetNote) { - targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); - } - } - - delete noteCache.attributes[attributeId]; - delete noteCache.attributeIndex[`${attr.type}-${attr.name}`]; - } - else if (attributeId in noteCache.attributes) { - const attr = noteCache.attributes[attributeId]; - - // attr name and isInheritable are immutable - attr.value = entity.value; - - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeFlatText(); - } - else { - note.flatTextCache = null; - } - } - else { - const attr = new Attribute(entity); - noteCache.attributes[attributeId] = attr; - - if (note) { - if (attr.isAffectingSubtree || note.isTemplate) { - note.invalidateSubtreeCaches(); - } - else { - this.invalidateThisCache(); - } - } - } - } -}); - -eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { - noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); -}); - module.exports = noteCache; diff --git a/src/services/note_cache/note_cache_loader.js b/src/services/note_cache/note_cache_loader.js new file mode 100644 index 000000000..33d387c3e --- /dev/null +++ b/src/services/note_cache/note_cache_loader.js @@ -0,0 +1,169 @@ +"use strict"; + +const sql = require('../sql.js'); +const sqlInit = require('../sql_init.js'); +const eventService = require('../events.js'); +const noteCache = require('./note_cache'); +const Note = require('./entities/note'); +const Branch = require('./entities/branch'); +const Attribute = require('./entities/attribute'); + +async function load() { + await sqlInit.dbReady; + + noteCache.notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, + row => new Note(noteCache, row)); + + noteCache.branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, + row => new Branch(noteCache, row)); + + noteCache.attributeIndex = []; + + noteCache.attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, + row => new Attribute(noteCache, row)); + + noteCache.loaded = true; + noteCache.loadedResolve(); +} + +async function getMappedRows(query, cb) { + const map = {}; + const results = await sql.getRows(query, []); + + for (const row of results) { + const keys = Object.keys(row); + + map[row[keys[0]]] = cb(row); + } + + return map; +} + +eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { + // note that entity can also be just POJO without methods if coming from sync + + if (!noteCache.loaded) { + return; + } + + if (entityName === 'notes') { + const {noteId} = entity; + + if (entity.isDeleted) { + delete noteCache.notes[noteId]; + } + else if (noteId in noteCache.notes) { + const note = noteCache.notes[noteId]; + + // we can assume we have protected session since we managed to update + note.title = entity.title; + note.isProtected = entity.isProtected; + note.isDecrypted = !entity.isProtected || !!entity.isContentAvailable; + note.flatTextCache = null; + + note.decrypt(); + } + else { + const note = new Note(entity); + noteCache.notes[noteId] = note; + + note.decrypt(); + } + } + else if (entityName === 'branches') { + const {branchId, noteId, parentNoteId} = entity; + const childNote = noteCache.notes[noteId]; + + if (entity.isDeleted) { + if (childNote) { + childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId); + childNote.parentBranches = childNote.parentBranches.filter(branch => branch.branchId !== branchId); + + if (childNote.parents.length > 0) { + childNote.invalidateSubtreeCaches(); + } + } + + const parentNote = noteCache.notes[parentNoteId]; + + if (parentNote) { + parentNote.children = parentNote.children.filter(child => child.noteId !== noteId); + } + + delete noteCache.childParentToBranch[`${noteId}-${parentNoteId}`]; + delete noteCache.branches[branchId]; + } + else if (branchId in noteCache.branches) { + // only relevant thing which can change in a branch is prefix + noteCache.branches[branchId].prefix = entity.prefix; + + if (childNote) { + childNote.flatTextCache = null; + } + } + else { + noteCache.branches[branchId] = new Branch(entity); + + if (childNote) { + childNote.resortParents(); + } + } + } + else if (entityName === 'attributes') { + const {attributeId, noteId} = entity; + const note = noteCache.notes[noteId]; + const attr = noteCache.attributes[attributeId]; + + if (entity.isDeleted) { + if (note && attr) { + // first invalidate and only then remove the attribute (otherwise invalidation wouldn't be complete) + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + + note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId); + + const targetNote = attr.targetNote; + + if (targetNote) { + targetNote.targetRelations = targetNote.targetRelations.filter(rel => rel.attributeId !== attributeId); + } + } + + delete noteCache.attributes[attributeId]; + delete noteCache.attributeIndex[`${attr.type}-${attr.name}`]; + } + else if (attributeId in noteCache.attributes) { + const attr = noteCache.attributes[attributeId]; + + // attr name and isInheritable are immutable + attr.value = entity.value; + + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeFlatText(); + } + else { + note.flatTextCache = null; + } + } + else { + const attr = new Attribute(entity); + noteCache.attributes[attributeId] = attr; + + if (note) { + if (attr.isAffectingSubtree || note.isTemplate) { + note.invalidateSubtreeCaches(); + } + else { + note.invalidateThisCache(); + } + } + } + } +}); + +eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { + noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); +}); + +module.exports = load; diff --git a/src/services/note_cache/note_cache_service.js b/src/services/note_cache/note_cache_service.js index 8007c9cb3..35b078be1 100644 --- a/src/services/note_cache/note_cache_service.js +++ b/src/services/note_cache/note_cache_service.js @@ -4,6 +4,8 @@ const noteCache = require('./note_cache'); const hoistedNoteService = require('../hoisted_note'); const stringSimilarity = require('string-similarity'); +require('./note_cache_loader')(); + function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; const note = noteCache.notes[noteId]; diff --git a/src/services/search/expressions/exists.js b/src/services/search/expressions/attribute_exists.js similarity index 93% rename from src/services/search/expressions/exists.js rename to src/services/search/expressions/attribute_exists.js index 25c3a9245..e80f8a9a8 100644 --- a/src/services/search/expressions/exists.js +++ b/src/services/search/expressions/attribute_exists.js @@ -3,7 +3,7 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class ExistsExp { +class AttributeExistsExp { constructor(attributeType, attributeName) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -31,4 +31,4 @@ class ExistsExp { } } -module.exports = ExistsExp; +module.exports = AttributeExistsExp; diff --git a/src/services/search/expressions/equals.js b/src/services/search/expressions/field_comparison.js similarity index 94% rename from src/services/search/expressions/equals.js rename to src/services/search/expressions/field_comparison.js index ecae22241..8c95f2169 100644 --- a/src/services/search/expressions/equals.js +++ b/src/services/search/expressions/field_comparison.js @@ -3,7 +3,7 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class EqualsExp { +class FieldComparisonExp { constructor(attributeType, attributeName, operator, attributeValue) { this.attributeType = attributeType; this.attributeName = attributeName; @@ -33,4 +33,4 @@ class EqualsExp { } } -module.exports = EqualsExp; +module.exports = FieldComparisonExp; diff --git a/src/services/search/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js index f09df1e88..c3cbc7b64 100644 --- a/src/services/search/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -2,7 +2,6 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -const noteCacheService = require('../../note_cache/note_cache_service'); class NoteCacheFulltextExp { constructor(tokens) { @@ -10,6 +9,9 @@ class NoteCacheFulltextExp { } execute(noteSet, searchContext) { + // has deps on SQL which breaks unit test so needs to be dynamically required + const noteCacheService = require('../../note_cache/note_cache_service'); + const resultNoteSet = new NoteSet(); const candidateNotes = this.getCandidateNotes(noteSet); diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index 4d6900802..d7591c7ef 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -12,6 +12,8 @@ class NoteContentFulltextExp { const resultNoteSet = new NoteSet(); const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); + const sql = require('../../sql'); + const noteIds = await sql.getColumn(` SELECT notes.noteId FROM notes diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 8ad021f89..1c692eaaf 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -1,8 +1,8 @@ const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); -const ExistsExp = require('./expressions/exists'); -const EqualsExp = require('./expressions/equals'); +const AttributeExistsExp = require('./expressions/attribute_exists'); +const FieldComparisonExp = require('./expressions/field_comparison'); const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); @@ -44,12 +44,12 @@ function getExpressions(tokens) { const type = token.startsWith('#') ? 'label' : 'relation'; if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { - expressions.push(new EqualsExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); + expressions.push(new FieldComparisonExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); i += 2; } else { - expressions.push(new ExistsExp(type, token.substr(1))); + expressions.push(new AttributeExistsExp(type, token.substr(1))); } } else if (['and', 'or'].includes(token.toLowerCase())) { @@ -71,6 +71,8 @@ function getExpressions(tokens) { op = 'and'; } } + + return expressions; } function parse(fulltextTokens, expressionTokens, includingNoteContent) { @@ -79,3 +81,5 @@ function parse(fulltextTokens, expressionTokens, includingNoteContent) { ...getExpressions(expressionTokens) ]); } + +module.exports = parse; diff --git a/src/services/sql_init.js b/src/services/sql_init.js index eaf0aaf23..6a50492e3 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -197,4 +197,4 @@ module.exports = { createInitialDatabase, createDatabaseForSync, dbInitialized -}; \ No newline at end of file +}; From faf4daa577c8f65fe2aee99a2b6b918bb751529f Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 20 May 2020 19:14:07 +0200 Subject: [PATCH 22/47] shadow on hover for title bar buttons --- src/public/app/widgets/title_bar_buttons.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/public/app/widgets/title_bar_buttons.js b/src/public/app/widgets/title_bar_buttons.js index 7c3084353..6be2e2dae 100644 --- a/src/public/app/widgets/title_bar_buttons.js +++ b/src/public/app/widgets/title_bar_buttons.js @@ -17,6 +17,10 @@ const TPL = ` padding-left: 10px; padding-right: 10px; } + + .title-bar-buttons button:hover { + background-color: var(--accented-background-color) !important; + } @@ -62,4 +66,4 @@ export default class TitleBarButtonsWidget extends BasicWidget { return this.$widget; } -} \ No newline at end of file +} From b26100479d15ba2a93019440de10d37bca305a26 Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 20 May 2020 23:20:39 +0200 Subject: [PATCH 23/47] parser tests added --- spec/parser.spec.js | 99 ++++++++++++++++++- src/services/note_cache/entities/attribute.js | 4 +- src/services/search/comparator_builder.js | 66 +++++++++++++ src/services/search/expressions/and.js | 13 +-- .../search/expressions/field_comparison.js | 7 +- src/services/search/expressions/or.js | 11 +++ src/services/search/parser.js | 52 ++++++---- 7 files changed, 220 insertions(+), 32 deletions(-) create mode 100644 src/services/search/comparator_builder.js diff --git a/spec/parser.spec.js b/spec/parser.spec.js index 7d9ad2a55..1fd939031 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -2,9 +2,102 @@ const parser = require('../src/services/search/parser'); describe("Parser", () => { it("fulltext parser without content", () => { - const exps = parser(["hello", "hi"], [], false); + const rootExp = parser(["hello", "hi"], [], false); - expect(exps.constructor.name).toEqual("NoteCacheFulltextExp"); - expect(exps.tokens).toEqual(["hello", "hi"]); + expect(rootExp.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(rootExp.tokens).toEqual(["hello", "hi"]); + }); + + it("fulltext parser with content", () => { + const rootExp = parser(["hello", "hi"], [], true); + + expect(rootExp.constructor.name).toEqual("OrExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(firstSub.tokens).toEqual(["hello", "hi"]); + + expect(secondSub.constructor.name).toEqual("NoteContentFulltextExp"); + expect(secondSub.tokens).toEqual(["hello", "hi"]); + }); + + it("simple label comparison", () => { + const rootExp = parser([], ["#mylabel", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("FieldComparisonExp"); + expect(rootExp.attributeType).toEqual("label"); + expect(rootExp.attributeName).toEqual("mylabel"); + expect(rootExp.comparator).toBeTruthy(); + }); + + it("simple label AND", () => { + const rootExp = parser([], ["#first", "=", "text", "AND", "#second", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("AndExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("second"); + }); + + it("simple label AND without explicit AND", () => { + const rootExp = parser([], ["#first", "=", "text", "#second", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("AndExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("second"); + }); + + it("simple label OR", () => { + const rootExp = parser([], ["#first", "=", "text", "OR", "#second", "=", "text"], true); + + expect(rootExp.constructor.name).toEqual("OrExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("second"); + }); + + it("fulltext and simple label", () => { + const rootExp = parser(["hello"], ["#mylabel", "=", "text"], false); + + expect(rootExp.constructor.name).toEqual("AndExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("NoteCacheFulltextExp"); + expect(firstSub.tokens).toEqual(["hello"]); + + expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.attributeName).toEqual("mylabel"); + }); + + it("label sub-expression", () => { + const rootExp = parser([], ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], false); + + expect(rootExp.constructor.name).toEqual("OrExp"); + const [firstSub, secondSub] = rootExp.subExpressions; + + expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.attributeName).toEqual("first"); + + expect(secondSub.constructor.name).toEqual("AndExp"); + const [firstSubSub, secondSubSub] = secondSub.subExpressions; + + expect(firstSubSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSubSub.attributeName).toEqual("second"); + + expect(secondSubSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSubSub.attributeName).toEqual("third"); }); }); diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index 116c4b719..e66b9c771 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -11,9 +11,9 @@ class Attribute { /** @param {string} */ this.type = row.type; /** @param {string} */ - this.name = row.name; + this.name = row.name.toLowerCase(); /** @param {string} */ - this.value = row.value; + this.value = row.value.toLowerCase(); /** @param {boolean} */ this.isInheritable = !!row.isInheritable; diff --git a/src/services/search/comparator_builder.js b/src/services/search/comparator_builder.js new file mode 100644 index 000000000..72c1103a8 --- /dev/null +++ b/src/services/search/comparator_builder.js @@ -0,0 +1,66 @@ +const dayjs = require("dayjs"); + +const comparators = { + "=": comparedValue => (val => val === comparedValue), + "!=": comparedValue => (val => val !== comparedValue), + ">": comparedValue => (val => val > comparedValue), + ">=": comparedValue => (val => val >= comparedValue), + "<": comparedValue => (val => val < comparedValue), + "<=": comparedValue => (val => val <= comparedValue), + "*=": comparedValue => (val => val.endsWith(comparedValue)), + "=*": comparedValue => (val => val.startsWith(comparedValue)), + "*=*": comparedValue => (val => val.includes(comparedValue)), +} + +const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i; + +function calculateSmartValue(v) { + const match = smartValueRegex.exec(v); + if (match === null) { + return v; + } + + const keyword = match[1].toUpperCase(); + const num = match[2] ? parseInt(match[2].replace(/ /g, "")) : 0; // can contain spaces between sign and digits + + let format, date; + + if (keyword === 'NOW') { + date = dayjs().add(num, 'second'); + format = "YYYY-MM-DD HH:mm:ss"; + } + else if (keyword === 'TODAY') { + date = dayjs().add(num, 'day'); + format = "YYYY-MM-DD"; + } + else if (keyword === 'WEEK') { + // FIXME: this will always use sunday as start of the week + date = dayjs().startOf('week').add(7 * num, 'day'); + format = "YYYY-MM-DD"; + } + else if (keyword === 'MONTH') { + date = dayjs().add(num, 'month'); + format = "YYYY-MM"; + } + else if (keyword === 'YEAR') { + date = dayjs().add(num, 'year'); + format = "YYYY"; + } + else { + throw new Error("Unrecognized keyword: " + keyword); + } + + return date.format(format); +} + +function buildComparator(operator, comparedValue) { + comparedValue = comparedValue.toLowerCase(); + + comparedValue = calculateSmartValue(comparedValue); + + if (operator in comparators) { + return comparators[operator](comparedValue); + } +} + +module.exports = buildComparator; diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index 44e19c0af..cbdd4e108 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -1,19 +1,20 @@ "use strict"; class AndExp { - constructor(subExpressions) { - this.subExpressions = subExpressions; - } - static of(subExpressions) { + subExpressions = subExpressions.filter(exp => !!exp); + if (subExpressions.length === 1) { return subExpressions[0]; - } - else { + } else if (subExpressions.length > 0) { return new AndExp(subExpressions); } } + constructor(subExpressions) { + this.subExpressions = subExpressions; + } + execute(noteSet, searchContext) { for (const subExpression of this.subExpressions) { noteSet = subExpression.execute(noteSet, searchContext); diff --git a/src/services/search/expressions/field_comparison.js b/src/services/search/expressions/field_comparison.js index 8c95f2169..4630087ce 100644 --- a/src/services/search/expressions/field_comparison.js +++ b/src/services/search/expressions/field_comparison.js @@ -4,11 +4,10 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); class FieldComparisonExp { - constructor(attributeType, attributeName, operator, attributeValue) { + constructor(attributeType, attributeName, comparator) { this.attributeType = attributeType; this.attributeName = attributeName; - this.operator = operator; - this.attributeValue = attributeValue; + this.comparator = comparator; } execute(noteSet) { @@ -18,7 +17,7 @@ class FieldComparisonExp { for (const attr of attrs) { const note = attr.note; - if (noteSet.hasNoteId(note.noteId) && attr.value === this.attributeValue) { + if (noteSet.hasNoteId(note.noteId) && this.comparator(attr.value)) { if (attr.isInheritable) { resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); } diff --git a/src/services/search/expressions/or.js b/src/services/search/expressions/or.js index a48bb8bc8..51406bcfc 100644 --- a/src/services/search/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -3,6 +3,17 @@ const NoteSet = require('../note_set'); class OrExp { + static of(subExpressions) { + subExpressions = subExpressions.filter(exp => !!exp); + + if (subExpressions.length === 1) { + return subExpressions[0]; + } + else if (subExpressions.length > 0) { + return new OrExp(subExpressions); + } + } + constructor(subExpressions) { this.subExpressions = subExpressions; } diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 1c692eaaf..bf82e4b21 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -5,28 +5,32 @@ const AttributeExistsExp = require('./expressions/attribute_exists'); const FieldComparisonExp = require('./expressions/field_comparison'); const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); +const comparatorBuilder = require('./comparator_builder'); function getFulltext(tokens, includingNoteContent) { - if (includingNoteContent) { - return [ - new OrExp([ - new NoteCacheFulltextExp(tokens), - new NoteContentFulltextExp(tokens) - ]) - ] + if (tokens.length === 0) { + return null; + } + else if (includingNoteContent) { + return new OrExp([ + new NoteCacheFulltextExp(tokens), + new NoteContentFulltextExp(tokens) + ]); } else { - return [ - new NoteCacheFulltextExp(tokens) - ] + return new NoteCacheFulltextExp(tokens); } } function isOperator(str) { - return str.matches(/^[=<>*]+$/); + return str.match(/^[=<>*]+$/); } -function getExpressions(tokens) { +function getExpression(tokens) { + if (tokens.length === 0) { + return null; + } + const expressions = []; let op = null; @@ -38,13 +42,22 @@ function getExpressions(tokens) { } if (Array.isArray(token)) { - expressions.push(getExpressions(token)); + expressions.push(getExpression(token)); } else if (token.startsWith('#') || token.startsWith('@')) { const type = token.startsWith('#') ? 'label' : 'relation'; if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { - expressions.push(new FieldComparisonExp(type, token.substr(1), tokens[i + 1], tokens[i + 2])); + const operator = tokens[i + 1]; + const comparedValue = tokens[i + 2]; + + const comparator = comparatorBuilder(operator, comparedValue); + + if (!comparator) { + throw new Error(`Can't find operator '${operator}'`); + } + + expressions.push(new FieldComparisonExp(type, token.substr(1), comparator)); i += 2; } @@ -72,13 +85,18 @@ function getExpressions(tokens) { } } - return expressions; + if (op === null || op === 'and') { + return AndExp.of(expressions); + } + else if (op === 'or') { + return OrExp.of(expressions); + } } function parse(fulltextTokens, expressionTokens, includingNoteContent) { return AndExp.of([ - ...getFulltext(fulltextTokens, includingNoteContent), - ...getExpressions(expressionTokens) + getFulltext(fulltextTokens, includingNoteContent), + getExpression(expressionTokens) ]); } From 32dde426fd0f2d573e154b1ea20feb2c1539146c Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 00:39:17 +0200 Subject: [PATCH 24/47] apply new query parsing to note autocomplete --- src/services/note_cache/entities/attribute.js | 2 +- .../search/expressions/attribute_exists.js | 2 + .../search/expressions/field_comparison.js | 2 + .../search/expressions/note_cache_fulltext.js | 105 +++++++++--------- .../expressions/note_content_fulltext.js | 4 +- src/services/search/parens.js | 2 +- src/services/search/search.js | 36 ++++-- 7 files changed, 84 insertions(+), 69 deletions(-) diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index e66b9c771..173f80071 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -19,7 +19,7 @@ class Attribute { this.noteCache.notes[this.noteId].ownedAttributes.push(this); - const key = `${this.type-this.name}`; + const key = `${this.type}-${this.name}`; this.noteCache.attributeIndex[key] = this.noteCache.attributeIndex[key] || []; this.noteCache.attributeIndex[key].push(this); diff --git a/src/services/search/expressions/attribute_exists.js b/src/services/search/expressions/attribute_exists.js index e80f8a9a8..196937777 100644 --- a/src/services/search/expressions/attribute_exists.js +++ b/src/services/search/expressions/attribute_exists.js @@ -28,6 +28,8 @@ class AttributeExistsExp { } } } + + return resultNoteSet; } } diff --git a/src/services/search/expressions/field_comparison.js b/src/services/search/expressions/field_comparison.js index 4630087ce..cf3523d20 100644 --- a/src/services/search/expressions/field_comparison.js +++ b/src/services/search/expressions/field_comparison.js @@ -29,6 +29,8 @@ class FieldComparisonExp { } } } + + return resultNoteSet; } } diff --git a/src/services/search/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js index c3cbc7b64..7a816f11b 100644 --- a/src/services/search/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -11,15 +11,64 @@ class NoteCacheFulltextExp { execute(noteSet, searchContext) { // has deps on SQL which breaks unit test so needs to be dynamically required const noteCacheService = require('../../note_cache/note_cache_service'); - const resultNoteSet = new NoteSet(); + function searchDownThePath(note, tokens, path) { + if (tokens.length === 0) { + const retPath = noteCacheService.getSomePath(note, path); + + if (retPath) { + const noteId = retPath[retPath.length - 1]; + searchContext.noteIdToNotePath[noteId] = retPath; + + resultNoteSet.add(noteCache.notes[noteId]); + } + + return; + } + + if (!note.parents.length === 0 || note.noteId === 'root') { + return; + } + + const foundAttrTokens = []; + + for (const attribute of note.ownedAttributes) { + for (const token of tokens) { + if (attribute.name.toLowerCase().includes(token) + || attribute.value.toLowerCase().includes(token)) { + foundAttrTokens.push(token); + } + } + } + + for (const parentNote of note.parents) { + const title = noteCacheService.getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); + const foundTokens = foundAttrTokens.slice(); + + for (const token of tokens) { + if (title.includes(token)) { + foundTokens.push(token); + } + } + + if (foundTokens.length > 0) { + const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); + + searchDownThePath(parentNote, remainingTokens, path.concat([note.noteId])); + } + else { + searchDownThePath(parentNote, tokens, path.concat([note.noteId])); + } + } + } + const candidateNotes = this.getCandidateNotes(noteSet); for (const note of candidateNotes) { // autocomplete should be able to find notes by their noteIds as well (only leafs) if (this.tokens.length === 1 && note.noteId === this.tokens[0]) { - this.searchDownThePath(note, [], [], resultNoteSet, searchContext); + searchDownThePath(note, [], []); continue; } @@ -52,7 +101,7 @@ class NoteCacheFulltextExp { if (foundTokens.length > 0) { const remainingTokens = this.tokens.filter(token => !foundTokens.includes(token)); - this.searchDownThePath(parentNote, remainingTokens, [note.noteId], resultNoteSet, searchContext); + searchDownThePath(parentNote, remainingTokens, [note.noteId]); } } } @@ -80,56 +129,6 @@ class NoteCacheFulltextExp { return candidateNotes; } - - searchDownThePath(note, tokens, path, resultNoteSet, searchContext) { - if (tokens.length === 0) { - const retPath = noteCacheService.getSomePath(note, path); - - if (retPath) { - const noteId = retPath[retPath.length - 1]; - searchContext.noteIdToNotePath[noteId] = retPath; - - resultNoteSet.add(noteCache.notes[noteId]); - } - - return; - } - - if (!note.parents.length === 0 || note.noteId === 'root') { - return; - } - - const foundAttrTokens = []; - - for (const attribute of note.ownedAttributes) { - for (const token of tokens) { - if (attribute.name.toLowerCase().includes(token) - || attribute.value.toLowerCase().includes(token)) { - foundAttrTokens.push(token); - } - } - } - - for (const parentNote of note.parents) { - const title = noteCacheService.getNoteTitle(note.noteId, parentNote.noteId).toLowerCase(); - const foundTokens = foundAttrTokens.slice(); - - for (const token of tokens) { - if (title.includes(token)) { - foundTokens.push(token); - } - } - - if (foundTokens.length > 0) { - const remainingTokens = tokens.filter(token => !foundTokens.includes(token)); - - this.searchDownThePath(parentNote, remainingTokens, path.concat([note.noteId]), resultNoteSet, searchContext); - } - else { - this.searchDownThePath(parentNote, tokens, path.concat([note.noteId]), resultNoteSet, searchContext); - } - } - } } module.exports = NoteCacheFulltextExp; diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index d7591c7ef..606dc780c 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -20,15 +20,13 @@ class NoteContentFulltextExp { JOIN note_contents ON notes.noteId = note_contents.noteId WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); - const results = []; - for (const noteId of noteIds) { if (noteSet.hasNoteId(noteId) && noteId in noteCache.notes) { resultNoteSet.add(noteCache.notes[noteId]); } } - return results; + return resultNoteSet; } } diff --git a/src/services/search/parens.js b/src/services/search/parens.js index 7f402d2b5..ba5fdc7c1 100644 --- a/src/services/search/parens.js +++ b/src/services/search/parens.js @@ -3,7 +3,7 @@ */ function parens(tokens) { if (tokens.length === 0) { - throw new Error("Empty expression."); + return []; } while (true) { diff --git a/src/services/search/search.js b/src/services/search/search.js index defc4cc7b..6a51f5abc 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -1,14 +1,16 @@ "use strict"; -const NoteCacheFulltextExp = require("./expressions/note_cache_fulltext"); +const lexer = require('./lexer'); +const parens = require('./parens'); +const parser = require('./parser'); const NoteSet = require("./note_set"); const SearchResult = require("./search_result"); const noteCache = require('../note_cache/note_cache'); +const noteCacheService = require('../note_cache/note_cache_service'); const hoistedNoteService = require('../hoisted_note'); const utils = require('../utils'); async function findNotesWithExpression(expression) { - const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()]; const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') ? hoistedNote.subtreeNotes @@ -23,7 +25,7 @@ async function findNotesWithExpression(expression) { const noteSet = await expression.execute(allNoteSet, searchContext); let searchResults = noteSet.notes - .map(note => searchContext.noteIdToNotePath[note.noteId] || getSomePath(note)) + .map(note => searchContext.noteIdToNotePath[note.noteId] || noteCacheService.getSomePath(note)) .filter(notePathArray => notePathArray.includes(hoistedNoteService.getHoistedNoteId())) .map(notePathArray => new SearchResult(notePathArray)); @@ -40,24 +42,30 @@ async function findNotesWithExpression(expression) { return searchResults; } +function parseQueryToExpression(query) { + const {fulltextTokens, expressionTokens} = lexer(query); + const structuredExpressionTokens = parens(expressionTokens); + const expression = parser(fulltextTokens, structuredExpressionTokens, false); + + return expression; +} + async function searchNotesForAutocomplete(query) { if (!query.trim().length) { return []; } - const tokens = query - .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc - .toLowerCase() - .split(/[ -]/) - .filter(token => token !== '/'); // '/' is used as separator + const expression = parseQueryToExpression(query); - const expression = new NoteCacheFulltextExp(tokens); + if (!expression) { + return []; + } let searchResults = await findNotesWithExpression(expression); searchResults = searchResults.slice(0, 200); - highlightSearchResults(searchResults, tokens); + highlightSearchResults(searchResults, query); return searchResults.map(result => { return { @@ -68,7 +76,13 @@ async function searchNotesForAutocomplete(query) { }); } -function highlightSearchResults(searchResults, tokens) { +function highlightSearchResults(searchResults, query) { + let tokens = query + .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc + .toLowerCase() + .split(/[ -]/) + .filter(token => token !== '/'); + // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks // which would make the resulting HTML string invalid. // { and } are used for marking and tag (to avoid matches on single 'b' character) From 08dbf90a8c40222ebf8b02f9a5f2ab9740036d69 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 09:42:25 +0200 Subject: [PATCH 25/47] hide body during startup to reduce flicker --- .idea/dataSources.xml | 2 +- package.json | 10 +++++----- src/views/desktop.ejs | 9 +++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index fa5ae48bf..3ecf47dd8 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -5,7 +5,7 @@ sqlite.xerial true org.sqlite.JDBC - jdbc:sqlite:$PROJECT_DIR$/../trilium-data/document.db + jdbc:sqlite:$USER_HOME$/trilium-data/document.db \ No newline at end of file diff --git a/package.json b/package.json index df0b1c810..892152425 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "electron-window-state": "5.0.3", "express": "4.17.1", "express-session": "1.17.1", - "file-type": "14.4.0", + "file-type": "14.5.0", "fs-extra": "9.0.0", "helmet": "3.22.0", "html": "1.0.0", @@ -52,14 +52,14 @@ "imagemin-pngquant": "8.0.0", "ini": "1.3.5", "is-svg": "4.2.1", - "jimp": "0.12.0", + "jimp": "0.12.1", "mime-types": "2.1.27", "multer": "1.4.2", - "node-abi": "2.16.0", + "node-abi": "2.17.0", "open": "7.0.4", "portscanner": "2.2.0", "rand-token": "1.0.1", - "rcedit": "2.1.1", + "rcedit": "2.2.0", "rimraf": "3.0.2", "sanitize-filename": "1.6.3", "sax": "1.2.4", @@ -80,7 +80,7 @@ }, "devDependencies": { "electron": "9.0.0", - "electron-builder": "22.6.0", + "electron-builder": "22.6.1", "electron-packager": "14.2.1", "electron-rebuild": "1.11.0", "jasmine": "^3.5.0", diff --git a/src/views/desktop.ejs b/src/views/desktop.ejs index 837e8d8d4..8f5560d1e 100644 --- a/src/views/desktop.ejs +++ b/src/views/desktop.ejs @@ -8,6 +8,11 @@ + +
@@ -78,6 +83,10 @@ + + From a8d12f723fb9fe4bf35fa08cabcb769dc6b99692 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 11:18:15 +0200 Subject: [PATCH 26/47] fix highlighting --- package-lock.json | 490 +++++++++++++++++----------------- spec/parser.spec.js | 48 +++- src/services/search/parser.js | 20 +- src/services/search/search.js | 32 +-- 4 files changed, 311 insertions(+), 279 deletions(-) diff --git a/package-lock.json b/package-lock.json index f408c38bf..4e7f1cb65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -198,22 +198,22 @@ } }, "@jimp/bmp": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.12.0.tgz", - "integrity": "sha512-PjgGVaSQvPrepsD52aTQe6B8A1G/OOYIcpXt6K59AUHQE3s6oNo9lYfyUv96gInBBIMze9s8AgLhMLjU8ijw4Q==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.12.1.tgz", + "integrity": "sha512-t16IamuBMv4GiGa1VAMzsgrVKVANxXG81wXECzbikOUkUv7pKJ2vHZDgkLBEsZQ9sAvFCneM1+yoSRpuENrfVQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", + "@jimp/utils": "^0.12.1", "bmp-js": "^0.1.0" } }, "@jimp/core": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.12.0.tgz", - "integrity": "sha512-xLF8gvRyJSCu08PI01b/MFijxoBoPusJFbSOOzMnP286qVDouxdXQy6CJB3mMosnlZRgp12I+ZgUvMsdJsL8ig==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.12.1.tgz", + "integrity": "sha512-mWfjExYEjHxBal+1gPesGChOQBSpxO7WUQkrO9KM7orboitOdQ15G5UA75ce7XVZ+5t+FQPOLmVkVZzzTQSEJA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", + "@jimp/utils": "^0.12.1", "any-base": "^1.1.0", "buffer": "^5.2.0", "exif-parser": "^0.1.12", @@ -233,292 +233,292 @@ } }, "@jimp/custom": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.12.0.tgz", - "integrity": "sha512-Rf3p50Jmvy9Aeovs0kyIpd0qbt2peLqDRq6f93AlDkUpB6OZ/rQwgJO8yysNMgI877a3xQz0Tda5j5Lv8AjWgA==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.12.1.tgz", + "integrity": "sha512-bVClp8FEJ/11GFTKeRTrfH7NgUWvVO5/tQzO/68aOwMIhbz9BOYQGh533K9+mSy29VjZJo8jxZ0C9ZwYHuFwfA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/core": "^0.12.0" + "@jimp/core": "^0.12.1" } }, "@jimp/gif": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.12.0.tgz", - "integrity": "sha512-CMapyrH5LGXbl2jHgQA923wHUNbC0LajqMmMHfyFZE9GZFzXULqbTZdRemHXTXn++iruPSR37oVUYi67WG9qmQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.12.1.tgz", + "integrity": "sha512-cGn/AcvMGUGcqR6ByClGSnrja4AYmTwsGVXTQ1+EmfAdTiy6ztGgZCTDpZ/tq4SpdHXwm9wDHez7damKhTrH0g==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", + "@jimp/utils": "^0.12.1", "omggif": "^1.0.9" } }, "@jimp/jpeg": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.12.0.tgz", - "integrity": "sha512-jAC9gWPCBJ0ysTZDqDUOVUty3/tk2qStw3N5Vk9W3XZNSTNlLp5xWsiATlkAoSrwoBmdgjf6OfZwqmkFDFVMKw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.12.1.tgz", + "integrity": "sha512-UoCUHbKLj2CDCETd7LrJnmK/ExDsSfJXmc1pKkfgomvepjXogdl2KTHf141wL6D+9CfSD2VBWQLC5TvjMvcr9A==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", - "jpeg-js": "^0.3.4" + "@jimp/utils": "^0.12.1", + "jpeg-js": "^0.4.0" } }, "@jimp/plugin-blit": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.12.0.tgz", - "integrity": "sha512-csSxB/ZOljGLtvRne+nF1EGpcHZ/6mdGc+trcihClTTLAS5FzX+tySpQj9sHrIzzHtEcILYPMOKaf6KC4LOrfw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.12.1.tgz", + "integrity": "sha512-VRBB6bx6EpQuaH0WX8ytlGNqUQcmuxXBbzL3e+cD0W6MluYibzQy089okvXcyUS72Q+qpSMmUDCVr3pDqLAsSA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-blur": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.12.0.tgz", - "integrity": "sha512-HCL570HvZxhT7Yn/Qqow00sRK0J/E4j1Clwp78vMnQWQ38PONi/Ipyjqp0RLdvCj3tJ3mzrKDqlnN2bbLcKsjQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.12.1.tgz", + "integrity": "sha512-rTFY0yrwVJFNgNsAlYGn2GYCRLVEcPQ6cqAuhNylXuR/7oH3Acul+ZWafeKtvN8D8uMlth/6VP74gruXvwffZw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-circle": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.12.0.tgz", - "integrity": "sha512-d+cRlyrM4ylXKk6TuFZcoFz8xsXqLHGfZcX+BDFe9HPz+TTW7AoL5eq8I0uLpTHRD1dLdBPScMejkn3ppLKnjg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.12.1.tgz", + "integrity": "sha512-+/OiBDjby7RBbQoDX8ZsqJRr1PaGPdTaaKUVGAsrE7KCNO9ODYNFAizB9lpidXkGgJ4Wx5R4mJy21i22oY/a4Q==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-color": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.12.0.tgz", - "integrity": "sha512-RmPSwryrmLLtsNluQ9hT73EovM+KcthacDmF7VN/xnJMD/r+vXfgUcDLZDx8yQsd5kdezhtPh7wKihH2+voOwg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.12.1.tgz", + "integrity": "sha512-xlnK/msWN4uZ+Bu7+UrCs9oMzTSA9QE0jWFnF3h0aBsD8t1LGxozkckHe8nHtC/y/sxIa8BGKSfkiaW+r6FbnA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", + "@jimp/utils": "^0.12.1", "tinycolor2": "^1.4.1" } }, "@jimp/plugin-contain": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.12.0.tgz", - "integrity": "sha512-mA1l2GbtmY2uLdCiwzdSJa9tZSyL5uvQwT3UrKDWaPiyhT4+VrCgQVD4CBbOFztI8ToxPcGM9GG4oRuRR2cKDQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.12.1.tgz", + "integrity": "sha512-WZ/D6G0jhnBh2bkBh610PEh/caGhAUIAxYLsQsfSSlOxPsDhbj3S6hMbFKRgnDvf0hsd5zTIA0j1B0UG4kh18A==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-cover": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.12.0.tgz", - "integrity": "sha512-rTrGxCBr1dn6DOVF+g8IFUCXHpfOaZCC4kvOyx/GIE3861GqKyOWzjLRcWTVBfgyiuOx+S6kBwpadmnsFO8tHg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.12.1.tgz", + "integrity": "sha512-ddWwTQO40GcabJ2UwUYCeuNxnjV4rBTiLprnjGMqAJCzdz3q3Sp20FkRf+H+E22k2v2LHss8dIOFOF4i6ycr9Q==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-crop": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.12.0.tgz", - "integrity": "sha512-sEz1T7waD5c+nB0aJERipc8/LSaRo4IxPzemOuzWaXxvwdUVRPtM7Rk7XOZmJyc2nW8qPNet+2JkaSjg848xgQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.12.1.tgz", + "integrity": "sha512-CKjVkrNO8FDZKYVpMireQW4SgKBSOdF+Ip/1sWssHHe77+jGEKqOjhYju+VhT3dZJ3+75rJNI9II7Kethp+rTw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-displace": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.12.0.tgz", - "integrity": "sha512-VWlTF6TEDdGoN56tnOfsHVNNtsWBHCBmT77G+2k2agbXWAPD5A++bye0y4XP/icAS//sAd7UFzvNQlnT7sIAdg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.12.1.tgz", + "integrity": "sha512-MQAw2iuf1/bVJ6P95WWTLA+WBjvIZ7TeGBerkvBaTK8oWdj+NSLNRIYOIoyPbZ7DTL8f1SN4Vd6KD6BZaoWrwg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-dither": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.12.0.tgz", - "integrity": "sha512-EzhHugll52ngdV1RBh1wmRUjf1jgo2GfU+Zh/a05uLxKGZEDWqGcsfFlI4lZnJbiKUhHCTNwZRCkV2w9gNJ6uw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.12.1.tgz", + "integrity": "sha512-mCrBHdx2ViTLJDLcrobqGLlGhZF/Mq41bURWlElQ2ArvrQ3/xR52We9DNDfC08oQ2JVb6q3v1GnCCdn0KNojGQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-fisheye": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.12.0.tgz", - "integrity": "sha512-Rz/gboWtY6sow6FC4tg9kG/fNBLopjGRoMmzHVcoQK1XXI2O/tH6nrliHHv3s3AvBBrQ5qPyO5VyCb1vm9xOmA==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.12.1.tgz", + "integrity": "sha512-CHvYSXtHNplzkkYzB44tENPDmvfUHiYCnAETTY+Hx58kZ0w8ERZ+OiLhUmiBcvH/QHm/US1iiNjgGUAfeQX6dg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-flip": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.12.0.tgz", - "integrity": "sha512-NFeIHWU95rSmIUnUdHVAYU4dYE3X10qY2peTgbMJ+q1J2qsrUO7w6Gepfd26tA9lh41zwDD5UzuAorpHQ3z27g==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.12.1.tgz", + "integrity": "sha512-xi+Yayrnln8A/C9E3yQBExjxwBSeCkt/ZQg1CxLgszVyX/3Zo8+nkV8MJYpkTpj8LCZGTOKlsE05mxu/a3lbJQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-gaussian": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.12.0.tgz", - "integrity": "sha512-LXus5pMzUaIYGTCoWDxRiMb5AW0gJMqet3U6+mQIP7OtSnBL2Vimz9WBbzZuEfKRMCc1l6oDwD/o/fH5ehv+TQ==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.12.1.tgz", + "integrity": "sha512-7O6eKlhL37hsLfV6WAX1Cvce7vOqSwL1oWbBveC1agutDlrtvcTh1s2mQ4Pde654hCJu55mq1Ur10+ote5j3qw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-invert": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.12.0.tgz", - "integrity": "sha512-fkOBCFg9P3Nkc0aFgWt5WgRP41KOs9m8OOnIi4jLnvCamv/Fv8GJLMeDS3gIXuzb/XkS0W/WpMQJmvI1+Zj2xg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.12.1.tgz", + "integrity": "sha512-JTAs7A1Erbxwl+7ph7tgcb2PZ4WzB+3nb2WbfiWU8iCrKj17mMDSc5soaCCycn8wfwqvgB1vhRfGpseOLWxsuQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-mask": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.12.0.tgz", - "integrity": "sha512-BWe0n6EB5/b5H062Vybyd2rTkC7yV/DNtNgJiVseZiqJCwmOjZDq+Gx+gKmB3959Th9ipwdEt3nzwBwmyDBVwA==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.12.1.tgz", + "integrity": "sha512-bnDdY0RO/x5Mhqoy+056SN1wEj++sD4muAKqLD2CIT8Zq5M/0TA4hkdf/+lwFy3H2C0YTK39PSE9xyb4jPX3kA==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-normalize": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.12.0.tgz", - "integrity": "sha512-w66beKgxBI1Psv7BmKDxCFJOqAxzn4whVcHgsQ31627HFTelDAf9kSTUOdcIPEwWfWg0tzkKmnHGYQhfJUHYRw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.12.1.tgz", + "integrity": "sha512-4kSaI4JLM/PNjHwbnAHgyh51V5IlPfPxYvsZyZ1US32pebWtocxSMaSuOaJUg7OGSkwSDBv81UR2h5D+Dz1b5A==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-print": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.12.0.tgz", - "integrity": "sha512-DdPAmPlTc0rNXRD7efLnCUD2VhYe9kx6h+2mCobGA3AHakrAdJ8qndkWF6UsYxlyrLXxVTftfWKhHOTGOGyA7Q==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.12.1.tgz", + "integrity": "sha512-T0lNS3qU9SwCHOEz7AGrdp50+gqiWGZibOL3350/X/dqoFs1EvGDjKVeWncsGCyLlpfd7M/AibHZgu8Fx2bWng==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", + "@jimp/utils": "^0.12.1", "load-bmfont": "^1.4.0" } }, "@jimp/plugin-resize": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.12.0.tgz", - "integrity": "sha512-5qqrYmMeSyfNvFb+hdL1XDdGC2Db+/1KwWH9Zw3IxaAB4pXVPmZYMfBi9cJXd1mVHafl+FQWAEy5Ii3hXA32aw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.12.1.tgz", + "integrity": "sha512-sbNn4tdBGcgGlPt9XFxCuDl4ZOoxa8/Re8nAikyxYhRss2Dqz91ARbBQxOf1vlUGeicQMsjEuWbPQAogTSJRug==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-rotate": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.12.0.tgz", - "integrity": "sha512-tOgn86RoFyDm+BJOfdhPXNjaUiaotKcvMzfdR/o4kL/55y+x7xfVj7v7CJbvudnG29bDwEM+3r8HwfaQsezosg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.12.1.tgz", + "integrity": "sha512-RYkLzwG2ervG6hHy8iepbIVeWdT1kz4Qz044eloqo6c66MK0KAqp228YI8+CAKm0joQnVDC/A0FgRIj/K8uyAw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-scale": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.12.0.tgz", - "integrity": "sha512-FS8MWgUcCZ1nwFX4YupTK59nuTqK8seo2CXJeHXgGjl8UU6c/EPBD9SrAuqSNbngcDY9fZ65i6srUyqrQ8kk7w==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.12.1.tgz", + "integrity": "sha512-zjNVI1fUj+ywfG78T1ZU33g9a5sk4rhEQkkhtny8koAscnVsDN2YaZEKoFli54kqaWh5kSS5DDL7a/9pEfXnFQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-shadow": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.12.0.tgz", - "integrity": "sha512-FzzTVccC6BkL9Y0rFxI5Di4JEZvCxKq7AyyK6qI7OwBrwxoAmtUodkxGDZTUvYfpmtMZeLWG9TUVrJ/sBQ+NWA==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.12.1.tgz", + "integrity": "sha512-Z82IwvunXWQ2jXegd3W3TYUXpfJcEvNbHodr7Z+oVnwhM1OoQ5QC6RSRQwsj2qXIhbGffQjH8eguHgEgAV+u5w==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugin-threshold": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.12.0.tgz", - "integrity": "sha512-Sqf2MFDQY/kz0sAPtfjjG4BUcrF58lT09h2EJ75Rdc3hiAWrB7XizLvnI1J8rooHci8Ablbkb/E6xu+52KOGuw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.12.1.tgz", + "integrity": "sha512-PFezt5fSk0q+xKvdpuv0eLggy2I7EgYotrK8TRZOT0jimuYFXPF0Z514c6szumoW5kEsRz04L1HkPT1FqI97Yg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0" + "@jimp/utils": "^0.12.1" } }, "@jimp/plugins": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.12.0.tgz", - "integrity": "sha512-P/1vKex4P697ayzVysMSjckcHE2Ii61tyNkq9t1RSZuERgyE616llVKMcil0aVYTnoqapjOwEW36c/fWY8Zj6g==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.12.1.tgz", + "integrity": "sha512-7+Yp29T6BbYo+Oqnc+m7A5AH+O+Oy5xnxvxlfmsp48+SuwEZ4akJp13Gu2PSmRlylENzR7MlWOxzhas5ERNlIg==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/plugin-blit": "^0.12.0", - "@jimp/plugin-blur": "^0.12.0", - "@jimp/plugin-circle": "^0.12.0", - "@jimp/plugin-color": "^0.12.0", - "@jimp/plugin-contain": "^0.12.0", - "@jimp/plugin-cover": "^0.12.0", - "@jimp/plugin-crop": "^0.12.0", - "@jimp/plugin-displace": "^0.12.0", - "@jimp/plugin-dither": "^0.12.0", - "@jimp/plugin-fisheye": "^0.12.0", - "@jimp/plugin-flip": "^0.12.0", - "@jimp/plugin-gaussian": "^0.12.0", - "@jimp/plugin-invert": "^0.12.0", - "@jimp/plugin-mask": "^0.12.0", - "@jimp/plugin-normalize": "^0.12.0", - "@jimp/plugin-print": "^0.12.0", - "@jimp/plugin-resize": "^0.12.0", - "@jimp/plugin-rotate": "^0.12.0", - "@jimp/plugin-scale": "^0.12.0", - "@jimp/plugin-shadow": "^0.12.0", - "@jimp/plugin-threshold": "^0.12.0", + "@jimp/plugin-blit": "^0.12.1", + "@jimp/plugin-blur": "^0.12.1", + "@jimp/plugin-circle": "^0.12.1", + "@jimp/plugin-color": "^0.12.1", + "@jimp/plugin-contain": "^0.12.1", + "@jimp/plugin-cover": "^0.12.1", + "@jimp/plugin-crop": "^0.12.1", + "@jimp/plugin-displace": "^0.12.1", + "@jimp/plugin-dither": "^0.12.1", + "@jimp/plugin-fisheye": "^0.12.1", + "@jimp/plugin-flip": "^0.12.1", + "@jimp/plugin-gaussian": "^0.12.1", + "@jimp/plugin-invert": "^0.12.1", + "@jimp/plugin-mask": "^0.12.1", + "@jimp/plugin-normalize": "^0.12.1", + "@jimp/plugin-print": "^0.12.1", + "@jimp/plugin-resize": "^0.12.1", + "@jimp/plugin-rotate": "^0.12.1", + "@jimp/plugin-scale": "^0.12.1", + "@jimp/plugin-shadow": "^0.12.1", + "@jimp/plugin-threshold": "^0.12.1", "timm": "^1.6.1" } }, "@jimp/png": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.12.0.tgz", - "integrity": "sha512-5MgVBRhjkivIHy7cJ6QnU4CygndSde0ZMcaVkfBIyh6gd8pCcIG/XbY2TcR9lSkflgw3tUVzLrFR1xWUYr2trg==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.12.1.tgz", + "integrity": "sha512-tOUSJMJzcMAN82F9/Q20IToquIVWzvOe/7NIpVQJn6m+Lq6TtVmd7d8gdcna9AEFm2FIza5lhq2Kta6Xj0KXhQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/utils": "^0.12.0", + "@jimp/utils": "^0.12.1", "pngjs": "^3.3.3" } }, "@jimp/tiff": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.12.0.tgz", - "integrity": "sha512-h7HBCSjTA4YlnWx66qxQh9YxuzxMoBSGkTiUDEhao2BIhYa2pRmRwtMfqp1EdeRYcXkswWpn4qZAr7zY1TlIGw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.12.1.tgz", + "integrity": "sha512-bzWDgv3202TKhaBGzV9OFF0PVQWEb4194h9kv5js348SSnbCusz/tzTE1EwKrnbDZThZPgTB1ryKs7D+Q9Mhmg==", "requires": { "@babel/runtime": "^7.7.2", "utif": "^2.0.1" } }, "@jimp/types": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.12.0.tgz", - "integrity": "sha512-6avU1n9lY4vpAHjKSQqrLbk6L5PCNFORre+T1Rcyvv/CGQKxVIAuRj1w+RzXClob8MEmvI17OI3R2w5RCbYpQw==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.12.1.tgz", + "integrity": "sha512-hg5OKXpWWeKGuDrfibrjWWhr7hqb7f552wqnPWSLQpVrdWgjH+hpOv6cOzdo9bsU78qGTelZJPxr0ERRoc+MhQ==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/bmp": "^0.12.0", - "@jimp/gif": "^0.12.0", - "@jimp/jpeg": "^0.12.0", - "@jimp/png": "^0.12.0", - "@jimp/tiff": "^0.12.0", + "@jimp/bmp": "^0.12.1", + "@jimp/gif": "^0.12.1", + "@jimp/jpeg": "^0.12.1", + "@jimp/png": "^0.12.1", + "@jimp/tiff": "^0.12.1", "timm": "^1.6.1" } }, "@jimp/utils": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.12.0.tgz", - "integrity": "sha512-MVoR31cQ6QRXHQI+qS9po7sr1LQTOOpQHE9I2oVeakcDkVX80xrRBif3WoNPvq3BG2+BDxt09CFwwHFHHFY49Q==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.12.1.tgz", + "integrity": "sha512-EjPkDQOzV/oZfbolEUgFT6SE++PtCccVBvjuACkttyCfl0P2jnpR49SwstyVLc2u8AwBAZEHHAw9lPYaMjtbXQ==", "requires": { "@babel/runtime": "^7.7.2", "regenerator-runtime": "^0.13.3" @@ -588,9 +588,9 @@ "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==" }, "@types/fs-extra": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.0.tgz", - "integrity": "sha512-UoOfVEzAUpeSPmjm7h1uk5MH6KZma2z2O7a75onTGjnNvAvMVrPzPL/vBbT65iIGHWj6rokwfmYcmxmlSf2uwg==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.1.tgz", + "integrity": "sha512-TcUlBem321DFQzBNuz8p0CLLKp0VvF/XH9E4KHNmgwyp4E3AfgI5cjiIVZWlbfThBop2qxFIh4+LeY6hVWWZ2w==", "dev": true, "requires": { "@types/node": "*" @@ -617,9 +617,9 @@ "integrity": "sha512-KPYGmfD0/b1eXurQ59fXD1GBzhSQfz6/lKBxkaHX9dKTzjXbK68Zt7yGUxUsCS1jeTy/8aL+d9JEr+S54mpkWQ==" }, "@types/yargs": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.4.tgz", - "integrity": "sha512-9T1auFmbPZoxHz0enUFlUuKRy3it01R+hlggyVUMtnCTQRunsQYifnSGb8hET4Xo8yiC0o0r1paW3ud5+rbURg==", + "version": "15.0.5", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.5.tgz", + "integrity": "sha512-Dk/IDOPtOgubt/IaevIUbTgV7doaKkoorvOyYM2CMwuDyP89bekI7H4xLIwunNYiK9jhCkmc6pUrJk3cj2AB9w==", "dev": true, "requires": { "@types/yargs-parser": "*" @@ -1161,27 +1161,27 @@ "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==" }, "app-builder-bin": { - "version": "3.5.8", - "resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-3.5.8.tgz", - "integrity": "sha512-ni3q7QTfQNWHNWuyn5x3FZu6GnQZv+TFnfgk5++svqleKEhHGqS1mIaKsh7x5pBX6NFXU3/+ktk98wA/AW4EXw==", + "version": "3.5.9", + "resolved": "https://registry.npmjs.org/app-builder-bin/-/app-builder-bin-3.5.9.tgz", + "integrity": "sha512-NSjtqZ3x2kYiDp3Qezsgukx/AUzKPr3Xgf9by4cYt05ILWGAptepeeu0Uv+7MO+41o6ujhLixTou8979JGg2Kg==", "dev": true }, "app-builder-lib": { - "version": "22.6.0", - "resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-22.6.0.tgz", - "integrity": "sha512-ky2aLYy92U+Gh6dKq/e8/bNmCotp6/GMhnX8tDZPv9detLg9WuBnWWi1ktBPlpbl1DREusy+TIh+9rgvfduQoA==", + "version": "22.6.1", + "resolved": "https://registry.npmjs.org/app-builder-lib/-/app-builder-lib-22.6.1.tgz", + "integrity": "sha512-ENL7r+H7IBfDb4faeLASgndsXrAT7AV7m7yJjcpbFDXYma6an7ZWGFIvR0HJrsfiC5TIB8kdLJ/aMSImrrSi/Q==", "dev": true, "requires": { "7zip-bin": "~5.0.3", "@develar/schema-utils": "~2.6.5", "async-exit-hook": "^2.0.1", "bluebird-lst": "^1.0.9", - "builder-util": "22.6.0", + "builder-util": "22.6.1", "builder-util-runtime": "8.7.0", "chromium-pickle-js": "^0.2.0", "debug": "^4.1.1", "ejs": "^3.1.2", - "electron-publish": "22.6.0", + "electron-publish": "22.6.1", "fs-extra": "^9.0.0", "hosted-git-info": "^3.0.4", "is-ci": "^2.0.0", @@ -1232,7 +1232,7 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" } } @@ -1508,7 +1508,7 @@ }, "uuid": { "version": "2.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" } } @@ -1542,7 +1542,7 @@ "dependencies": { "semver": { "version": "4.3.6", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz", "integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=" } } @@ -1562,7 +1562,7 @@ }, "bl": { "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "resolved": "http://registry.npmjs.org/bl/-/bl-1.2.2.tgz", "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", "requires": { "readable-stream": "^2.3.5", @@ -1822,26 +1822,26 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" }, "uuid": { "version": "2.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" } } }, "builder-util": { - "version": "22.6.0", - "resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.6.0.tgz", - "integrity": "sha512-jgdES2ExJYkuXC3DEaGAjFctKNA81C4QDy8zdoc+rqdSqheTizuDNtZg02uMFklmUES4V4fggmqds+Y7wraqng==", + "version": "22.6.1", + "resolved": "https://registry.npmjs.org/builder-util/-/builder-util-22.6.1.tgz", + "integrity": "sha512-A9cF+bSHqRTSKIUHEyE92Tl0Uh12N7yZRH9bccIL3gRUwtp6ulF28LsjNIWTSQ1clZo2M895cT5PCrKzjPQFVg==", "dev": true, "requires": { "7zip-bin": "~5.0.3", "@types/debug": "^4.1.5", "@types/fs-extra": "^8.1.0", - "app-builder-bin": "3.5.8", + "app-builder-bin": "3.5.9", "bluebird-lst": "^1.0.9", "builder-util-runtime": "8.7.0", "chalk": "^4.0.0", @@ -1889,16 +1889,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "source-map-support": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", - "dev": true, - "requires": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "stat-mode": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stat-mode/-/stat-mode-1.0.0.tgz", @@ -1942,7 +1932,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -2117,7 +2107,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -2434,7 +2424,7 @@ }, "commander": { "version": "2.8.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", + "resolved": "http://registry.npmjs.org/commander/-/commander-2.8.1.tgz", "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", "requires": { "graceful-readlink": ">= 1.0.0" @@ -3099,7 +3089,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -3124,13 +3114,13 @@ } }, "dmg-builder": { - "version": "22.6.0", - "resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-22.6.0.tgz", - "integrity": "sha512-rJxuGhHIpcuDGBtWZMM8aLxkbZNgYO2MO5dUerDIBXebhX1K8DA23iz/uZ8ahcRNgWEv57b8GDqJbXKEfr5T0A==", + "version": "22.6.1", + "resolved": "https://registry.npmjs.org/dmg-builder/-/dmg-builder-22.6.1.tgz", + "integrity": "sha512-jUTN0acP15puzevtQASj7QEPgUGpedWSuSnOwR/++JbeYRTwU2oro09h/KZnaeMcxgxjdmT3tYLJeY1XUfPbRg==", "dev": true, "requires": { - "app-builder-lib": "22.6.0", - "builder-util": "22.6.0", + "app-builder-lib": "22.6.1", + "builder-util": "22.6.1", "fs-extra": "^9.0.0", "iconv-lite": "^0.5.1", "js-yaml": "^3.13.1", @@ -3327,18 +3317,18 @@ } }, "electron-builder": { - "version": "22.6.0", - "resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-22.6.0.tgz", - "integrity": "sha512-aLHlB6DTfjJ3MI4AUIFeWnwIozNgNlbOk2c2sTHxB10cAKp0dBVSPZ7xF5NK0uwDhElvRzJQubnHtJD6zKg42Q==", + "version": "22.6.1", + "resolved": "https://registry.npmjs.org/electron-builder/-/electron-builder-22.6.1.tgz", + "integrity": "sha512-3/VNg9GfXKHM53TilFtfF1+bsAR8THK1XHgeqCpsiequa02J9jTPc/DhpCUKQPkrs6/EIGxP7uboop7XYoew0Q==", "dev": true, "requires": { - "@types/yargs": "^15.0.4", - "app-builder-lib": "22.6.0", + "@types/yargs": "^15.0.5", + "app-builder-lib": "22.6.1", "bluebird-lst": "^1.0.9", - "builder-util": "22.6.0", + "builder-util": "22.6.1", "builder-util-runtime": "8.7.0", "chalk": "^4.0.0", - "dmg-builder": "22.6.0", + "dmg-builder": "22.6.1", "fs-extra": "^9.0.0", "is-ci": "^2.0.0", "lazy-val": "^1.0.4", @@ -3705,19 +3695,19 @@ } }, "electron-publish": { - "version": "22.6.0", - "resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-22.6.0.tgz", - "integrity": "sha512-+v05SBf9qR7Os5au+fifloNHy5QxHQkUGudBj68YaTb43Pn37UkwRxSc49Lf13s4wW32ohM45g8BOVInPJEdnA==", + "version": "22.6.1", + "resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-22.6.1.tgz", + "integrity": "sha512-/MkS47ospdSfAFW5Jp52OzYou14HhGJpZ51uAc3GJ5rCfACeqpimC/n1ajRLE3hcXxTWfd3t9MCuClq5jrUO5w==", "dev": true, "requires": { "@types/fs-extra": "^8.1.0", "bluebird-lst": "^1.0.9", - "builder-util": "22.6.0", + "builder-util": "22.6.1", "builder-util-runtime": "8.7.0", "chalk": "^4.0.0", "fs-extra": "^9.0.0", "lazy-val": "^1.0.4", - "mime": "^2.4.4" + "mime": "^2.4.5" }, "dependencies": { "ansi-styles": { @@ -4419,9 +4409,9 @@ } }, "file-type": { - "version": "14.4.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-14.4.0.tgz", - "integrity": "sha512-U5Q2lHPcERmBsg+DpS/+0r+g7PCsJmyW+aggHnGbMimCyNCpIerLv/VzHJHqtc0O91AXr4Puz4DL7LzA5hMdwA==", + "version": "14.5.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-14.5.0.tgz", + "integrity": "sha512-hIxIT/8DPClkKbC+IEoZvcQ5aBhsivh4aWzLMvmkp9Uabzey7gFNNPmTOwp8O/b2DkJ8a4FkFMkyFzkyRVsJXg==", "requires": { "readable-web-to-node-stream": "^2.0.0", "strtok3": "^6.0.0", @@ -4928,7 +4918,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "getpass": { @@ -5192,7 +5182,7 @@ }, "got": { "version": "5.7.1", - "resolved": "https://registry.npmjs.org/got/-/got-5.7.1.tgz", + "resolved": "http://registry.npmjs.org/got/-/got-5.7.1.tgz", "integrity": "sha1-X4FjWmHkplifGAVp6k44FoClHzU=", "requires": { "create-error-class": "^3.0.1", @@ -5840,7 +5830,7 @@ }, "into-stream": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", + "resolved": "http://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", "integrity": "sha1-lvsKk2wSur1v8XUqF9BWFqvQlMY=", "requires": { "from2": "^2.1.1", @@ -5992,7 +5982,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, "is-object": { @@ -6250,21 +6240,21 @@ } }, "jimp": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.12.0.tgz", - "integrity": "sha512-8QD1QNk2ZpoSFLDEQn4rlQ0sDAO1z6UagIqUsH6YjopHCExcAbk3q2hJFXk6wSf+LMHHkic44PhdVTZ0drER2w==", + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.12.1.tgz", + "integrity": "sha512-0soPJif+yjmzmOF+4cF2hyhxUWWpXpQntsm2joJXFFoRcQiPzsG4dbLKYqYPT3Fc6PjZ8MaLtCkDqqckVSfmRw==", "requires": { "@babel/runtime": "^7.7.2", - "@jimp/custom": "^0.12.0", - "@jimp/plugins": "^0.12.0", - "@jimp/types": "^0.12.0", + "@jimp/custom": "^0.12.1", + "@jimp/plugins": "^0.12.1", + "@jimp/types": "^0.12.1", "regenerator-runtime": "^0.13.3" } }, "jpeg-js": { - "version": "0.3.7", - "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz", - "integrity": "sha512-9IXdWudL61npZjvLuVe/ktHiA41iE8qFyLB+4VDTblEsWBzeg8WQTlktdUK4CdncUqtUgUg0bbOmTE2bKBKaBQ==" + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.0.tgz", + "integrity": "sha512-960VHmtN1vTpasX/1LupLohdP5odwAT7oK/VSm6mW0M58LbrBnowLAPWAZhWGhDAGjzbMnPXZxzB/QYgBwkN0w==" }, "js-yaml": { "version": "3.13.1", @@ -6626,7 +6616,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "requires": { "graceful-fs": "^4.1.2", @@ -7135,7 +7125,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, "minipass": { @@ -7235,7 +7225,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -7243,7 +7233,7 @@ "dependencies": { "minimist": { "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" } } @@ -7437,7 +7427,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "got": { @@ -7473,7 +7463,7 @@ }, "p-cancelable": { "version": "0.4.1", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", + "resolved": "http://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", "integrity": "sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ==" }, "p-event": { @@ -7597,7 +7587,7 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" } } @@ -7622,7 +7612,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "pify": { @@ -7679,7 +7669,7 @@ }, "get-stream": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", "integrity": "sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=", "requires": { "object-assign": "^4.0.1", @@ -7709,7 +7699,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -7749,7 +7739,7 @@ }, "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" }, "prepend-http": { @@ -7854,7 +7844,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "requires": { "core-util-is": "~1.0.0", @@ -7928,9 +7918,9 @@ "integrity": "sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==" }, "node-abi": { - "version": "2.16.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.16.0.tgz", - "integrity": "sha512-+sa0XNlWDA6T+bDLmkCUYn6W5k5W6BPRL6mqzSCs6H/xUgtl4D5x2fORKDzopKiU6wsyn/+wXlRXwXeSp+mtoA==", + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.17.0.tgz", + "integrity": "sha512-dFRAA0ACk/aBo0TIXQMEWMLUTyWYYT8OBYIzLmEUrQTElGRjxDCvyBZIsDL0QA7QCaj9PrawhOmTEdsuLY4uOQ==", "requires": { "semver": "^5.4.1" }, @@ -8232,7 +8222,7 @@ }, "onetime": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=" }, "open": { @@ -8384,7 +8374,7 @@ }, "p-is-promise": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=" }, "p-limit": { @@ -8865,7 +8855,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -9149,7 +9139,7 @@ "dependencies": { "file-type": { "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "resolved": "http://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" } } @@ -9174,7 +9164,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" }, "pify": { @@ -9212,7 +9202,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -9264,7 +9254,7 @@ }, "get-stream": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", "integrity": "sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=", "requires": { "object-assign": "^4.0.1", @@ -9294,7 +9284,7 @@ "dependencies": { "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" } } @@ -9482,7 +9472,7 @@ }, "query-string": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", + "resolved": "http://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", "integrity": "sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==", "requires": { "decode-uri-component": "^0.2.0", @@ -9557,9 +9547,9 @@ } }, "rcedit": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/rcedit/-/rcedit-2.1.1.tgz", - "integrity": "sha512-N1JyXxHD2zpqqW4A77RNK1d/M+tyed9JkvL/lnUI5cf4igF/8B9FNLFCtDUhGrk2GWEPxC+RF0WXWWB3I8QC7w==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/rcedit/-/rcedit-2.2.0.tgz", + "integrity": "sha512-dhFtYmQS+V8qQIANyX6zDK+sO50ayDePKApi46ZPK8I6QeyyTDD6LManMa7a3p3c9mLM4zi9QBP41pfhQ9p7Sg==" }, "read-all-stream": { "version": "3.1.0", @@ -9621,7 +9611,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -10489,7 +10479,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -10514,7 +10504,7 @@ }, "strip-dirs": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/strip-dirs/-/strip-dirs-1.1.1.tgz", "integrity": "sha1-lgu9EoeETzl1pFWKoQOoJV4kVqA=", "requires": { "chalk": "^1.0.0", @@ -10772,7 +10762,7 @@ }, "through": { "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { @@ -10791,7 +10781,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "requires": { "core-util-is": "~1.0.0", diff --git a/spec/parser.spec.js b/spec/parser.spec.js index 1fd939031..05fabea01 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -2,14 +2,22 @@ const parser = require('../src/services/search/parser'); describe("Parser", () => { it("fulltext parser without content", () => { - const rootExp = parser(["hello", "hi"], [], false); + const rootExp = parser({ + fulltextTokens: ["hello", "hi"], + expressionTokens: [], + includingNoteContent: false + }); expect(rootExp.constructor.name).toEqual("NoteCacheFulltextExp"); expect(rootExp.tokens).toEqual(["hello", "hi"]); }); it("fulltext parser with content", () => { - const rootExp = parser(["hello", "hi"], [], true); + const rootExp = parser({ + fulltextTokens: ["hello", "hi"], + expressionTokens: [], + includingNoteContent: true + }); expect(rootExp.constructor.name).toEqual("OrExp"); const [firstSub, secondSub] = rootExp.subExpressions; @@ -22,7 +30,11 @@ describe("Parser", () => { }); it("simple label comparison", () => { - const rootExp = parser([], ["#mylabel", "=", "text"], true); + const rootExp = parser({ + fulltextTokens: [], + expressionTokens: ["#mylabel", "=", "text"], + includingNoteContent: true + }); expect(rootExp.constructor.name).toEqual("FieldComparisonExp"); expect(rootExp.attributeType).toEqual("label"); @@ -31,7 +43,11 @@ describe("Parser", () => { }); it("simple label AND", () => { - const rootExp = parser([], ["#first", "=", "text", "AND", "#second", "=", "text"], true); + const rootExp = parser({ + fulltextTokens: [], + expressionTokens: ["#first", "=", "text", "AND", "#second", "=", "text"], + includingNoteContent: true + }); expect(rootExp.constructor.name).toEqual("AndExp"); const [firstSub, secondSub] = rootExp.subExpressions; @@ -44,7 +60,11 @@ describe("Parser", () => { }); it("simple label AND without explicit AND", () => { - const rootExp = parser([], ["#first", "=", "text", "#second", "=", "text"], true); + const rootExp = parser({ + fulltextTokens: [], + expressionTokens: ["#first", "=", "text", "#second", "=", "text"], + includingNoteContent: true + }); expect(rootExp.constructor.name).toEqual("AndExp"); const [firstSub, secondSub] = rootExp.subExpressions; @@ -57,7 +77,11 @@ describe("Parser", () => { }); it("simple label OR", () => { - const rootExp = parser([], ["#first", "=", "text", "OR", "#second", "=", "text"], true); + const rootExp = parser({ + fulltextTokens: [], + expressionTokens: ["#first", "=", "text", "OR", "#second", "=", "text"], + includingNoteContent: true + }); expect(rootExp.constructor.name).toEqual("OrExp"); const [firstSub, secondSub] = rootExp.subExpressions; @@ -70,7 +94,11 @@ describe("Parser", () => { }); it("fulltext and simple label", () => { - const rootExp = parser(["hello"], ["#mylabel", "=", "text"], false); + const rootExp = parser({ + fulltextTokens: ["hello"], + expressionTokens: ["#mylabel", "=", "text"], + includingNoteContent: false + }); expect(rootExp.constructor.name).toEqual("AndExp"); const [firstSub, secondSub] = rootExp.subExpressions; @@ -83,7 +111,11 @@ describe("Parser", () => { }); it("label sub-expression", () => { - const rootExp = parser([], ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], false); + const rootExp = parser({ + fulltextTokens: [], + expressionTokens: ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], + includingNoteContent: false + }); expect(rootExp.constructor.name).toEqual("OrExp"); const [firstSub, secondSub] = rootExp.subExpressions; diff --git a/src/services/search/parser.js b/src/services/search/parser.js index bf82e4b21..d63d14374 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -7,7 +7,9 @@ const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); const comparatorBuilder = require('./comparator_builder'); -function getFulltext(tokens, includingNoteContent) { +function getFulltext(tokens, includingNoteContent, highlightedTokens) { + highlightedTokens.push(...tokens); + if (tokens.length === 0) { return null; } @@ -26,7 +28,7 @@ function isOperator(str) { return str.match(/^[=<>*]+$/); } -function getExpression(tokens) { +function getExpression(tokens, highlightedTokens) { if (tokens.length === 0) { return null; } @@ -42,15 +44,19 @@ function getExpression(tokens) { } if (Array.isArray(token)) { - expressions.push(getExpression(token)); + expressions.push(getExpression(token, highlightedTokens)); } else if (token.startsWith('#') || token.startsWith('@')) { const type = token.startsWith('#') ? 'label' : 'relation'; + highlightedTokens.push(token.substr(1)); + if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { const operator = tokens[i + 1]; const comparedValue = tokens[i + 2]; + highlightedTokens.push(comparedValue); + const comparator = comparatorBuilder(operator, comparedValue); if (!comparator) { @@ -93,10 +99,12 @@ function getExpression(tokens) { } } -function parse(fulltextTokens, expressionTokens, includingNoteContent) { +function parse({fulltextTokens, expressionTokens, includingNoteContent, highlightedTokens}) { + highlightedTokens = highlightedTokens || []; + return AndExp.of([ - getFulltext(fulltextTokens, includingNoteContent), - getExpression(expressionTokens) + getFulltext(fulltextTokens, includingNoteContent, highlightedTokens), + getExpression(expressionTokens, highlightedTokens) ]); } diff --git a/src/services/search/search.js b/src/services/search/search.js index 6a51f5abc..a4d86802f 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -42,10 +42,16 @@ async function findNotesWithExpression(expression) { return searchResults; } -function parseQueryToExpression(query) { +function parseQueryToExpression(query, highlightedTokens) { const {fulltextTokens, expressionTokens} = lexer(query); const structuredExpressionTokens = parens(expressionTokens); - const expression = parser(fulltextTokens, structuredExpressionTokens, false); + + const expression = parser({ + fulltextTokens, + expressionTokens: structuredExpressionTokens, + includingNoteContent: false, + highlightedTokens + }); return expression; } @@ -55,7 +61,9 @@ async function searchNotesForAutocomplete(query) { return []; } - const expression = parseQueryToExpression(query); + const highlightedTokens = []; + + const expression = parseQueryToExpression(query, highlightedTokens); if (!expression) { return []; @@ -65,7 +73,7 @@ async function searchNotesForAutocomplete(query) { searchResults = searchResults.slice(0, 200); - highlightSearchResults(searchResults, query); + highlightSearchResults(searchResults, highlightedTokens); return searchResults.map(result => { return { @@ -76,20 +84,14 @@ async function searchNotesForAutocomplete(query) { }); } -function highlightSearchResults(searchResults, query) { - let tokens = query - .trim() // necessary because even with .split() trailing spaces are tokens which causes havoc - .toLowerCase() - .split(/[ -]/) - .filter(token => token !== '/'); - +function highlightSearchResults(searchResults, highlightedTokens) { // we remove < signs because they can cause trouble in matching and overwriting existing highlighted chunks // which would make the resulting HTML string invalid. // { and } are used for marking and tag (to avoid matches on single 'b' character) - tokens = tokens.map(token => token.replace('/[<\{\}]/g', '')); + highlightedTokens = highlightedTokens.map(token => token.replace('/[<\{\}]/g', '')); // sort by the longest so we first highlight longest matches - tokens.sort((a, b) => a.length > b.length ? -1 : 1); + highlightedTokens.sort((a, b) => a.length > b.length ? -1 : 1); for (const result of searchResults) { const note = noteCache.notes[result.noteId]; @@ -97,13 +99,13 @@ function highlightSearchResults(searchResults, query) { result.highlightedNotePathTitle = result.notePathTitle; for (const attr of note.attributes) { - if (tokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { + if (highlightedTokens.find(token => attr.name.includes(token) || attr.value.includes(token))) { result.highlightedNotePathTitle += ` ${formatAttribute(attr)}`; } } } - for (const token of tokens) { + for (const token of highlightedTokens) { const tokenRegex = new RegExp("(" + utils.escapeRegExp(token) + ")", "gi"); for (const result of searchResults) { From 75d8627f1cf6f4ab8d34dfbadf92536d523da546 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 11:46:01 +0200 Subject: [PATCH 27/47] refactoring to ParserContext --- spec/parser.spec.js | 17 +++++++++-------- src/services/search/parser.js | 25 +++++++++++++------------ src/services/search/parsing_context.js | 18 ++++++++++++++++++ src/services/search/search.js | 14 ++++++++------ 4 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 src/services/search/parsing_context.js diff --git a/spec/parser.spec.js b/spec/parser.spec.js index 05fabea01..466d90363 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -1,3 +1,4 @@ +const ParsingContext = require("../src/services/search/parsing_context"); const parser = require('../src/services/search/parser'); describe("Parser", () => { @@ -5,7 +6,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: ["hello", "hi"], expressionTokens: [], - includingNoteContent: false + parsingContext: new ParsingContext(false) }); expect(rootExp.constructor.name).toEqual("NoteCacheFulltextExp"); @@ -16,7 +17,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: ["hello", "hi"], expressionTokens: [], - includingNoteContent: true + parsingContext: new ParsingContext(true) }); expect(rootExp.constructor.name).toEqual("OrExp"); @@ -33,7 +34,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#mylabel", "=", "text"], - includingNoteContent: true + parsingContext: new ParsingContext(true) }); expect(rootExp.constructor.name).toEqual("FieldComparisonExp"); @@ -46,7 +47,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "AND", "#second", "=", "text"], - includingNoteContent: true + parsingContext: new ParsingContext(true) }); expect(rootExp.constructor.name).toEqual("AndExp"); @@ -63,7 +64,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "#second", "=", "text"], - includingNoteContent: true + parsingContext: new ParsingContext(true) }); expect(rootExp.constructor.name).toEqual("AndExp"); @@ -80,7 +81,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "OR", "#second", "=", "text"], - includingNoteContent: true + parsingContext: new ParsingContext(true) }); expect(rootExp.constructor.name).toEqual("OrExp"); @@ -97,7 +98,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: ["hello"], expressionTokens: ["#mylabel", "=", "text"], - includingNoteContent: false + parsingContext: new ParsingContext(false) }); expect(rootExp.constructor.name).toEqual("AndExp"); @@ -114,7 +115,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], - includingNoteContent: false + parsingContext: new ParsingContext(false) }); expect(rootExp.constructor.name).toEqual("OrExp"); diff --git a/src/services/search/parser.js b/src/services/search/parser.js index d63d14374..e92d07129 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -1,3 +1,6 @@ +"use strict"; + +const ParsingContext = require('./parsing_context'); const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); @@ -7,13 +10,13 @@ const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); const comparatorBuilder = require('./comparator_builder'); -function getFulltext(tokens, includingNoteContent, highlightedTokens) { - highlightedTokens.push(...tokens); +function getFulltext(tokens, parsingContext) { + parsingContext.highlightedTokens.push(...tokens); if (tokens.length === 0) { return null; } - else if (includingNoteContent) { + else if (parsingContext.includeNoteContent) { return new OrExp([ new NoteCacheFulltextExp(tokens), new NoteContentFulltextExp(tokens) @@ -28,7 +31,7 @@ function isOperator(str) { return str.match(/^[=<>*]+$/); } -function getExpression(tokens, highlightedTokens) { +function getExpression(tokens, parsingContext) { if (tokens.length === 0) { return null; } @@ -44,18 +47,18 @@ function getExpression(tokens, highlightedTokens) { } if (Array.isArray(token)) { - expressions.push(getExpression(token, highlightedTokens)); + expressions.push(getExpression(token, parsingContext)); } else if (token.startsWith('#') || token.startsWith('@')) { const type = token.startsWith('#') ? 'label' : 'relation'; - highlightedTokens.push(token.substr(1)); + parsingContext.highlightedTokens.push(token.substr(1)); if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { const operator = tokens[i + 1]; const comparedValue = tokens[i + 2]; - highlightedTokens.push(comparedValue); + parsingContext.highlightedTokens.push(comparedValue); const comparator = comparatorBuilder(operator, comparedValue); @@ -99,12 +102,10 @@ function getExpression(tokens, highlightedTokens) { } } -function parse({fulltextTokens, expressionTokens, includingNoteContent, highlightedTokens}) { - highlightedTokens = highlightedTokens || []; - +function parse({fulltextTokens, expressionTokens, parsingContext}) { return AndExp.of([ - getFulltext(fulltextTokens, includingNoteContent, highlightedTokens), - getExpression(expressionTokens, highlightedTokens) + getFulltext(fulltextTokens, parsingContext), + getExpression(expressionTokens, parsingContext) ]); } diff --git a/src/services/search/parsing_context.js b/src/services/search/parsing_context.js new file mode 100644 index 000000000..f76ff3b3d --- /dev/null +++ b/src/services/search/parsing_context.js @@ -0,0 +1,18 @@ +"use strict"; + +class ParsingContext { + constructor(includeNoteContent) { + this.includeNoteContent = includeNoteContent; + this.highlightedTokens = []; + this.error = null; + } + + addError(error) { + // we record only the first error, subsequent ones are usually consequence of the first + if (!this.error) { + this.error = error; + } + } +} + +module.exports = ParsingContext; diff --git a/src/services/search/search.js b/src/services/search/search.js index a4d86802f..1eb6faf77 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -42,15 +42,14 @@ async function findNotesWithExpression(expression) { return searchResults; } -function parseQueryToExpression(query, highlightedTokens) { +function parseQueryToExpression(query, parsingContext) { const {fulltextTokens, expressionTokens} = lexer(query); const structuredExpressionTokens = parens(expressionTokens); const expression = parser({ fulltextTokens, expressionTokens: structuredExpressionTokens, - includingNoteContent: false, - highlightedTokens + parsingContext }); return expression; @@ -61,9 +60,12 @@ async function searchNotesForAutocomplete(query) { return []; } - const highlightedTokens = []; + const parsingContext = { + includeNoteContent: false, + highlightedTokens: [] + }; - const expression = parseQueryToExpression(query, highlightedTokens); + const expression = parseQueryToExpression(query, parsingContext); if (!expression) { return []; @@ -73,7 +75,7 @@ async function searchNotesForAutocomplete(query) { searchResults = searchResults.slice(0, 200); - highlightSearchResults(searchResults, highlightedTokens); + highlightSearchResults(searchResults, parsingContext.highlightedTokens); return searchResults.map(result => { return { From 2e6395ad8875dd26de2d8d82b4bdc17852e206be Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 12:05:12 +0200 Subject: [PATCH 28/47] prefix match for autocomplete attribute search --- src/services/note_cache/note_cache.js | 14 ++++++++++++++ .../search/expressions/attribute_exists.js | 8 ++++++-- src/services/search/parser.js | 12 ++++++------ src/services/search/search.js | 7 +++---- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index 12942ee7f..a39898750 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -27,6 +27,20 @@ class NoteCache { return this.attributeIndex[`${type}-${name}`] || []; } + /** @return {Attribute[]} */ + findAttributesWithPrefix(type, name) { + const resArr = []; + const key = `${type}-${name}`; + + for (const idx in this.attributeIndex) { + if (idx.startsWith(key)) { + resArr.push(this.attributeIndex[idx]); + } + } + + return resArr.flat(); + } + decryptProtectedNotes() { for (const note of Object.values(this.notes)) { note.decrypt(); diff --git a/src/services/search/expressions/attribute_exists.js b/src/services/search/expressions/attribute_exists.js index 196937777..4f117a1fc 100644 --- a/src/services/search/expressions/attribute_exists.js +++ b/src/services/search/expressions/attribute_exists.js @@ -4,13 +4,17 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); class AttributeExistsExp { - constructor(attributeType, attributeName) { + constructor(attributeType, attributeName, prefixMatch) { this.attributeType = attributeType; this.attributeName = attributeName; + this.prefixMatch = prefixMatch; } execute(noteSet) { - const attrs = noteCache.findAttributes(this.attributeType, this.attributeName); + const attrs = this.prefixMatch + ? noteCache.findAttributesWithPrefix(this.attributeType, this.attributeName) + : noteCache.findAttributes(this.attributeType, this.attributeName); + const resultNoteSet = new NoteSet(); for (const attr of attrs) { diff --git a/src/services/search/parser.js b/src/services/search/parser.js index e92d07129..da81e9218 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -1,6 +1,5 @@ "use strict"; -const ParsingContext = require('./parsing_context'); const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); @@ -63,7 +62,8 @@ function getExpression(tokens, parsingContext) { const comparator = comparatorBuilder(operator, comparedValue); if (!comparator) { - throw new Error(`Can't find operator '${operator}'`); + parsingContext.addError(`Can't find operator '${operator}'`); + continue; } expressions.push(new FieldComparisonExp(type, token.substr(1), comparator)); @@ -71,7 +71,7 @@ function getExpression(tokens, parsingContext) { i += 2; } else { - expressions.push(new AttributeExistsExp(type, token.substr(1))); + expressions.push(new AttributeExistsExp(type, token.substr(1), parsingContext.fuzzyAttributeSearch)); } } else if (['and', 'or'].includes(token.toLowerCase())) { @@ -79,14 +79,14 @@ function getExpression(tokens, parsingContext) { op = token.toLowerCase(); } else if (op !== token.toLowerCase()) { - throw new Error('Mixed usage of AND/OR - always use parenthesis to group AND/OR expressions.'); + parsingContext.addError('Mixed usage of AND/OR - always use parenthesis to group AND/OR expressions.'); } } else if (isOperator(token)) { - throw new Error(`Misplaced or incomplete expression "${token}"`); + parsingContext.addError(`Misplaced or incomplete expression "${token}"`); } else { - throw new Error(`Unrecognized expression "${token}"`); + parsingContext.addError(`Unrecognized expression "${token}"`); } if (!op && expressions.length > 1) { diff --git a/src/services/search/search.js b/src/services/search/search.js index 1eb6faf77..c5c2d5236 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -5,6 +5,7 @@ const parens = require('./parens'); const parser = require('./parser'); const NoteSet = require("./note_set"); const SearchResult = require("./search_result"); +const ParsingContext = require("./parsing_context"); const noteCache = require('../note_cache/note_cache'); const noteCacheService = require('../note_cache/note_cache_service'); const hoistedNoteService = require('../hoisted_note'); @@ -60,10 +61,8 @@ async function searchNotesForAutocomplete(query) { return []; } - const parsingContext = { - includeNoteContent: false, - highlightedTokens: [] - }; + const parsingContext = new ParsingContext(false); + parsingContext.fuzzyAttributeSearch = true; const expression = parseQueryToExpression(query, parsingContext); From a06662f4ce6d8065233481dbd89e01e7bf9a12b3 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 13:45:18 +0200 Subject: [PATCH 29/47] fuzzy search for values as well --- src/services/search/parser.js | 6 +++++- src/services/search/parsing_context.js | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/services/search/parser.js b/src/services/search/parser.js index da81e9218..d13e1c93d 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -54,11 +54,15 @@ function getExpression(tokens, parsingContext) { parsingContext.highlightedTokens.push(token.substr(1)); if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { - const operator = tokens[i + 1]; + let operator = tokens[i + 1]; const comparedValue = tokens[i + 2]; parsingContext.highlightedTokens.push(comparedValue); + if (parsingContext.fuzzyAttributeSearch && operator === '=') { + operator = '*=*'; + } + const comparator = comparatorBuilder(operator, comparedValue); if (!comparator) { diff --git a/src/services/search/parsing_context.js b/src/services/search/parsing_context.js index f76ff3b3d..24118050f 100644 --- a/src/services/search/parsing_context.js +++ b/src/services/search/parsing_context.js @@ -4,6 +4,7 @@ class ParsingContext { constructor(includeNoteContent) { this.includeNoteContent = includeNoteContent; this.highlightedTokens = []; + this.fuzzyAttributeSearch = false; this.error = null; } From cd481353941e308128172348cf9e6a83c7e4a3f5 Mon Sep 17 00:00:00 2001 From: zadam Date: Thu, 21 May 2020 14:05:56 +0200 Subject: [PATCH 30/47] refactoring to allow unit tests of the whole search subsystem --- spec/parser.spec.js | 14 +++++++------- spec/search.spec.js | 7 +++++++ src/app.js | 4 +++- src/services/hoisted_note.js | 19 +++---------------- src/services/hoisted_note_loader.js | 14 ++++++++++++++ src/services/note_cache/note_cache_loader.js | 2 +- src/services/note_cache/note_cache_service.js | 2 -- src/services/search/parsing_context.js | 6 +++--- src/services/search/search.js | 6 ++++-- 9 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 spec/search.spec.js create mode 100644 src/services/hoisted_note_loader.js diff --git a/spec/parser.spec.js b/spec/parser.spec.js index 466d90363..e0993e51d 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -6,7 +6,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: ["hello", "hi"], expressionTokens: [], - parsingContext: new ParsingContext(false) + parsingContext: new ParsingContext({includeNoteContent: false}) }); expect(rootExp.constructor.name).toEqual("NoteCacheFulltextExp"); @@ -17,7 +17,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: ["hello", "hi"], expressionTokens: [], - parsingContext: new ParsingContext(true) + parsingContext: new ParsingContext({includeNoteContent: true}) }); expect(rootExp.constructor.name).toEqual("OrExp"); @@ -34,7 +34,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#mylabel", "=", "text"], - parsingContext: new ParsingContext(true) + parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("FieldComparisonExp"); @@ -64,7 +64,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "#second", "=", "text"], - parsingContext: new ParsingContext(true) + parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("AndExp"); @@ -81,7 +81,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "OR", "#second", "=", "text"], - parsingContext: new ParsingContext(true) + parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("OrExp"); @@ -98,7 +98,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: ["hello"], expressionTokens: ["#mylabel", "=", "text"], - parsingContext: new ParsingContext(false) + parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("AndExp"); @@ -115,7 +115,7 @@ describe("Parser", () => { const rootExp = parser({ fulltextTokens: [], expressionTokens: ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], - parsingContext: new ParsingContext(false) + parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("OrExp"); diff --git a/spec/search.spec.js b/spec/search.spec.js new file mode 100644 index 000000000..836043da1 --- /dev/null +++ b/spec/search.spec.js @@ -0,0 +1,7 @@ +const searchService = require('../src/services/search/search'); + +describe("Search", () => { + it("fulltext parser without content", () => { +// searchService. + }); +}); diff --git a/src/app.js b/src/app.js index 3ef1a7c0f..1c6a31d3e 100644 --- a/src/app.js +++ b/src/app.js @@ -12,6 +12,8 @@ const sessionSecret = require('./services/session_secret'); const cls = require('./services/cls'); require('./entities/entity_constructor'); require('./services/handlers'); +require('./services/hoisted_note_loader'); +require('./services/note_cache/note_cache_loader'); const app = express(); @@ -120,4 +122,4 @@ require('./services/scheduler'); module.exports = { app, sessionParser -}; \ No newline at end of file +}; diff --git a/src/services/hoisted_note.js b/src/services/hoisted_note.js index dc9536bc9..2a45cd322 100644 --- a/src/services/hoisted_note.js +++ b/src/services/hoisted_note.js @@ -1,19 +1,6 @@ -const optionService = require('./options'); -const sqlInit = require('./sql_init'); -const eventService = require('./events'); - let hoistedNoteId = 'root'; -eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entity}) => { - if (entityName === 'options' && entity.name === 'hoistedNoteId') { - hoistedNoteId = entity.value; - } -}); - -sqlInit.dbReady.then(async () => { - hoistedNoteId = await optionService.getOption('hoistedNoteId'); -}); - module.exports = { - getHoistedNoteId: () => hoistedNoteId -}; \ No newline at end of file + getHoistedNoteId: () => hoistedNoteId, + setHoistedNoteId(noteId) { hoistedNoteId = noteId; } +}; diff --git a/src/services/hoisted_note_loader.js b/src/services/hoisted_note_loader.js new file mode 100644 index 000000000..fb7b6df12 --- /dev/null +++ b/src/services/hoisted_note_loader.js @@ -0,0 +1,14 @@ +const optionService = require('./options'); +const sqlInit = require('./sql_init'); +const eventService = require('./events'); +const hoistedNote = require('./hoisted_note'); + +eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entity}) => { + if (entityName === 'options' && entity.name === 'hoistedNoteId') { + hoistedNote.setHoistedNoteId(entity.value); + } +}); + +sqlInit.dbReady.then(async () => { + hoistedNote.setHoistedNoteId(await optionService.getOption('hoistedNoteId')); +}); diff --git a/src/services/note_cache/note_cache_loader.js b/src/services/note_cache/note_cache_loader.js index 33d387c3e..9fb160128 100644 --- a/src/services/note_cache/note_cache_loader.js +++ b/src/services/note_cache/note_cache_loader.js @@ -166,4 +166,4 @@ eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, () => { noteCache.loadedPromise.then(() => noteCache.decryptProtectedNotes()); }); -module.exports = load; +load(); diff --git a/src/services/note_cache/note_cache_service.js b/src/services/note_cache/note_cache_service.js index 35b078be1..8007c9cb3 100644 --- a/src/services/note_cache/note_cache_service.js +++ b/src/services/note_cache/note_cache_service.js @@ -4,8 +4,6 @@ const noteCache = require('./note_cache'); const hoistedNoteService = require('../hoisted_note'); const stringSimilarity = require('string-similarity'); -require('./note_cache_loader')(); - function isNotePathArchived(notePath) { const noteId = notePath[notePath.length - 1]; const note = noteCache.notes[noteId]; diff --git a/src/services/search/parsing_context.js b/src/services/search/parsing_context.js index 24118050f..59bc487d8 100644 --- a/src/services/search/parsing_context.js +++ b/src/services/search/parsing_context.js @@ -1,10 +1,10 @@ "use strict"; class ParsingContext { - constructor(includeNoteContent) { - this.includeNoteContent = includeNoteContent; + constructor(params = {}) { + this.includeNoteContent = !!params.includeNoteContent; + this.fuzzyAttributeSearch = !!params.fuzzyAttributeSearch; this.highlightedTokens = []; - this.fuzzyAttributeSearch = false; this.error = null; } diff --git a/src/services/search/search.js b/src/services/search/search.js index c5c2d5236..254c35869 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -61,8 +61,10 @@ async function searchNotesForAutocomplete(query) { return []; } - const parsingContext = new ParsingContext(false); - parsingContext.fuzzyAttributeSearch = true; + const parsingContext = new ParsingContext({ + includeNoteContent: false, + fuzzyAttributeSearch: true + }); const expression = parseQueryToExpression(query, parsingContext); From ee053b9fdfe919e6b749f11d1390d29d655ed584 Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 22 May 2020 09:38:30 +0200 Subject: [PATCH 31/47] basic search tests --- spec/lexer.spec.js | 6 +- spec/search.spec.js | 119 +++++++++++++++++- src/services/note_cache/entities/attribute.js | 1 + src/services/note_cache/entities/branch.js | 1 + src/services/note_cache/entities/note.js | 2 + src/services/note_cache/note_cache.js | 12 +- src/services/note_cache/note_cache_loader.js | 26 +--- src/services/search/expressions/and.js | 5 +- .../search/expressions/attribute_exists.js | 5 +- src/services/search/expressions/expression.js | 11 ++ .../search/expressions/field_comparison.js | 5 +- src/services/search/expressions/not.js | 6 +- .../search/expressions/note_cache_fulltext.js | 5 +- .../expressions/note_content_fulltext.js | 5 +- src/services/search/expressions/or.js | 5 +- src/services/search/lexer.js | 2 + src/services/search/search.js | 30 +++-- 17 files changed, 202 insertions(+), 44 deletions(-) create mode 100644 src/services/search/expressions/expression.js diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js index 14f4314fb..7532faa35 100644 --- a/spec/lexer.spec.js +++ b/spec/lexer.spec.js @@ -18,8 +18,8 @@ describe("Lexer fulltext", () => { }); it("you can use different quotes and other special characters inside quotes", () => { - expect(lexer("'I can use \" or ` or #@=*' without problem").fulltextTokens) - .toEqual(["I can use \" or ` or #@=*", "without", "problem"]); + expect(lexer("'i can use \" or ` or #@=*' without problem").fulltextTokens) + .toEqual(["i can use \" or ` or #@=*", "without", "problem"]); }); it("if quote is not ended then it's just one long token", () => { @@ -56,6 +56,6 @@ describe("Lexer expression", () => { it("complex expressions with and, or and parenthesis", () => { expect(lexer(`# (#label=text OR #second=text) AND @relation`).expressionTokens) - .toEqual(["#", "(", "#label", "=", "text", "OR", "#second", "=", "text", ")", "AND", "@relation"]); + .toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "@relation"]); }); }); diff --git a/spec/search.spec.js b/spec/search.spec.js index 836043da1..907e74e0a 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -1,7 +1,122 @@ const searchService = require('../src/services/search/search'); +const Note = require('../src/services/note_cache/entities/note'); +const Branch = require('../src/services/note_cache/entities/branch'); +const Attribute = require('../src/services/note_cache/entities/attribute'); +const ParsingContext = require('../src/services/search/parsing_context'); +const noteCache = require('../src/services/note_cache/note_cache'); +const randtoken = require('rand-token').generator({source: 'crypto'}); describe("Search", () => { - it("fulltext parser without content", () => { -// searchService. + let rootNote; + + beforeEach(() => { + noteCache.reset(); + + rootNote = new NoteBuilder(new Note(noteCache, {noteId: 'root', title: 'root'})); + new Branch(noteCache, {branchId: 'root', noteId: 'root', parentNoteId: 'none'}); + }); + + it("simple path match", async () => { + rootNote.child( + note("Europe") + .child( + note("Austria") + ) + ); + + const parsingContext = new ParsingContext(); + const searchResults = await searchService.findNotesWithQuery('europe austria', parsingContext); + + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + }); + + it("only end leafs are results", async () => { + rootNote.child( + note("Europe") + .child( + note("Austria") + ) + ); + + const parsingContext = new ParsingContext(); + const searchResults = await searchService.findNotesWithQuery('europe', parsingContext); + + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); + }); + + it("only end leafs are results", async () => { + rootNote.child( + note("Europe") + .child( + note("Austria") + .label('capital', 'Vienna') + ) + ); + + const parsingContext = new ParsingContext(); + + const searchResults = await searchService.findNotesWithQuery('Vienna', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); }); }); + +/** @return {Note} */ +function findNoteByTitle(searchResults, title) { + return searchResults + .map(sr => noteCache.notes[sr.noteId]) + .find(note => note.title === title); +} + +class NoteBuilder { + constructor(note) { + this.note = note; + } + + label(name, value) { + new Attribute(noteCache, { + attributeId: id(), + noteId: this.note.noteId, + type: 'label', + name, + value + }); + + return this; + } + + relation(name, note) { + new Attribute(noteCache, { + attributeId: id(), + noteId: this.note.noteId, + type: 'relation', + name, + value: note.noteId + }); + + return this; + } + + child(childNoteBuilder, prefix = "") { + new Branch(noteCache, { + branchId: id(), + noteId: childNoteBuilder.note.noteId, + parentNoteId: this.note.noteId, + prefix + }); + + return this; + } +} + +function id() { + return randtoken.generate(10); +} + +function note(title) { + const note = new Note(noteCache, {noteId: id(), title}); + + return new NoteBuilder(note); +} diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index 173f80071..0de3c539f 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -17,6 +17,7 @@ class Attribute { /** @param {boolean} */ this.isInheritable = !!row.isInheritable; + this.noteCache.attributes[this.attributeId] = this; this.noteCache.notes[this.noteId].ownedAttributes.push(this); const key = `${this.type}-${this.name}`; diff --git a/src/services/note_cache/entities/branch.js b/src/services/note_cache/entities/branch.js index b3a3af904..dfff5a393 100644 --- a/src/services/note_cache/entities/branch.js +++ b/src/services/note_cache/entities/branch.js @@ -30,6 +30,7 @@ class Branch { parentNote.children.push(childNote); + this.noteCache.branches[this.branchId] = this; this.noteCache.childParentToBranch[`${this.noteId}-${this.parentNoteId}`] = this; } diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index 8d3a7abc2..7556ce250 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -34,6 +34,8 @@ class Note { /** @param {string|null} */ this.flatTextCache = null; + this.noteCache.notes[this.noteId] = this; + if (protectedSessionService.isProtectedSessionAvailable()) { this.decrypt(); } diff --git a/src/services/note_cache/note_cache.js b/src/services/note_cache/note_cache.js index a39898750..c517563d6 100644 --- a/src/services/note_cache/note_cache.js +++ b/src/services/note_cache/note_cache.js @@ -6,16 +6,20 @@ const Attribute = require('./entities/attribute'); class NoteCache { constructor() { + this.reset(); + } + + reset() { /** @type {Object.} */ - this.notes = null; + this.notes = []; /** @type {Object.} */ - this.branches = null; + this.branches = []; /** @type {Object.} */ this.childParentToBranch = {}; /** @type {Object.} */ - this.attributes = null; + this.attributes = []; /** @type {Object.} Points from attribute type-name to list of attributes them */ - this.attributeIndex = null; + this.attributeIndex = {}; this.loaded = false; this.loadedResolve = null; diff --git a/src/services/note_cache/note_cache_loader.js b/src/services/note_cache/note_cache_loader.js index 9fb160128..b3f4893b9 100644 --- a/src/services/note_cache/note_cache_loader.js +++ b/src/services/note_cache/note_cache_loader.js @@ -11,34 +11,20 @@ const Attribute = require('./entities/attribute'); async function load() { await sqlInit.dbReady; - noteCache.notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, - row => new Note(noteCache, row)); + noteCache.reset(); - noteCache.branches = await getMappedRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, - row => new Branch(noteCache, row)); + (await sql.getRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, [])) + .map(row => new Note(noteCache, row)); - noteCache.attributeIndex = []; + (await sql.getRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, [])) + .map(row => new Branch(noteCache, row)); - noteCache.attributes = await getMappedRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, - row => new Attribute(noteCache, row)); + (await sql.getRows(`SELECT attributeId, noteId, type, name, value, isInheritable FROM attributes WHERE isDeleted = 0`, [])).map(row => new Attribute(noteCache, row)); noteCache.loaded = true; noteCache.loadedResolve(); } -async function getMappedRows(query, cb) { - const map = {}; - const results = await sql.getRows(query, []); - - for (const row of results) { - const keys = Object.keys(row); - - map[row[keys[0]]] = cb(row); - } - - return map; -} - eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], async ({entityName, entity}) => { // note that entity can also be just POJO without methods if coming from sync diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index cbdd4e108..56973b11c 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -1,6 +1,8 @@ "use strict"; -class AndExp { +const Expression = require('./expression'); + +class AndExp extends Expression{ static of(subExpressions) { subExpressions = subExpressions.filter(exp => !!exp); @@ -12,6 +14,7 @@ class AndExp { } constructor(subExpressions) { + super(); this.subExpressions = subExpressions; } diff --git a/src/services/search/expressions/attribute_exists.js b/src/services/search/expressions/attribute_exists.js index 4f117a1fc..8b1da1b11 100644 --- a/src/services/search/expressions/attribute_exists.js +++ b/src/services/search/expressions/attribute_exists.js @@ -2,9 +2,12 @@ const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); +const Expression = require('./expression'); -class AttributeExistsExp { +class AttributeExistsExp extends Expression { constructor(attributeType, attributeName, prefixMatch) { + super(); + this.attributeType = attributeType; this.attributeName = attributeName; this.prefixMatch = prefixMatch; diff --git a/src/services/search/expressions/expression.js b/src/services/search/expressions/expression.js new file mode 100644 index 000000000..49a1e64e5 --- /dev/null +++ b/src/services/search/expressions/expression.js @@ -0,0 +1,11 @@ +"use strict"; + +class Expression { + /** + * @param {NoteSet} noteSet + * @param {object} searchContext + */ + execute(noteSet, searchContext) {} +} + +module.exports = Expression; diff --git a/src/services/search/expressions/field_comparison.js b/src/services/search/expressions/field_comparison.js index cf3523d20..c497552d7 100644 --- a/src/services/search/expressions/field_comparison.js +++ b/src/services/search/expressions/field_comparison.js @@ -1,10 +1,13 @@ "use strict"; +const Expression = require('./expression'); const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class FieldComparisonExp { +class FieldComparisonExp extends Expression { constructor(attributeType, attributeName, comparator) { + super(); + this.attributeType = attributeType; this.attributeName = attributeName; this.comparator = comparator; diff --git a/src/services/search/expressions/not.js b/src/services/search/expressions/not.js index 22d8ebee4..434a46dc6 100644 --- a/src/services/search/expressions/not.js +++ b/src/services/search/expressions/not.js @@ -1,7 +1,11 @@ "use strict"; -class NotExp { +const Expression = require('./expression'); + +class NotExp extends Expression { constructor(subExpression) { + super(); + this.subExpression = subExpression; } diff --git a/src/services/search/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js index 7a816f11b..93ec54328 100644 --- a/src/services/search/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -1,10 +1,13 @@ "use strict"; +const Expression = require('./expression'); const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class NoteCacheFulltextExp { +class NoteCacheFulltextExp extends Expression { constructor(tokens) { + super(); + this.tokens = tokens; } diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index 606dc780c..3148ded4d 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -1,10 +1,13 @@ "use strict"; +const Expression = require('./expression'); const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class NoteContentFulltextExp { +class NoteContentFulltextExp extends Expression { constructor(tokens) { + super(); + this.tokens = tokens; } diff --git a/src/services/search/expressions/or.js b/src/services/search/expressions/or.js index 51406bcfc..3b21f24c5 100644 --- a/src/services/search/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -1,8 +1,9 @@ "use strict"; +const Expression = require('./expression'); const NoteSet = require('../note_set'); -class OrExp { +class OrExp extends Expression { static of(subExpressions) { subExpressions = subExpressions.filter(exp => !!exp); @@ -15,6 +16,8 @@ class OrExp { } constructor(subExpressions) { + super(); + this.subExpressions = subExpressions; } diff --git a/src/services/search/lexer.js b/src/services/search/lexer.js index 301355d30..8a5b90f0a 100644 --- a/src/services/search/lexer.js +++ b/src/services/search/lexer.js @@ -1,4 +1,6 @@ function lexer(str) { + str = str.toLowerCase(); + const fulltextTokens = []; const expressionTokens = []; diff --git a/src/services/search/search.js b/src/services/search/search.js index 254c35869..5af43925b 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -11,6 +11,10 @@ const noteCacheService = require('../note_cache/note_cache_service'); const hoistedNoteService = require('../hoisted_note'); const utils = require('../utils'); +/** + * @param {Expression} expression + * @return {Promise} + */ async function findNotesWithExpression(expression) { const hoistedNote = noteCache.notes[hoistedNoteService.getHoistedNoteId()]; const allNotes = (hoistedNote && hoistedNote.noteId !== 'root') @@ -56,6 +60,21 @@ function parseQueryToExpression(query, parsingContext) { return expression; } +/** + * @param {string} query + * @param {ParsingContext} parsingContext + * @return {Promise} + */ +async function findNotesWithQuery(query, parsingContext) { + const expression = parseQueryToExpression(query, parsingContext); + + if (!expression) { + return []; + } + + return await findNotesWithExpression(expression); +} + async function searchNotesForAutocomplete(query) { if (!query.trim().length) { return []; @@ -66,13 +85,7 @@ async function searchNotesForAutocomplete(query) { fuzzyAttributeSearch: true }); - const expression = parseQueryToExpression(query, parsingContext); - - if (!expression) { - return []; - } - - let searchResults = await findNotesWithExpression(expression); + let searchResults = findNotesWithQuery(query, parsingContext); searchResults = searchResults.slice(0, 200); @@ -141,5 +154,6 @@ function formatAttribute(attr) { } module.exports = { - searchNotesForAutocomplete + searchNotesForAutocomplete, + findNotesWithQuery }; From 714881ad99fc82f833d359b9886a166f2df3303e Mon Sep 17 00:00:00 2001 From: zadam Date: Fri, 22 May 2020 19:08:06 +0200 Subject: [PATCH 32/47] more search tests + numeric label comparison --- spec/search.spec.js | 82 ++++++++++++++++++++++- src/services/search/comparator_builder.js | 19 ++++-- src/services/search/note_set.js | 2 +- 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/spec/search.spec.js b/spec/search.spec.js index 907e74e0a..19eb70345 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -61,6 +61,85 @@ describe("Search", () => { expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); }); + + it("numeric label comparison", async () => { + rootNote.child( + note("Europe") + .label('country', '', true) + .child( + note("Austria") + .label('population', '8859000') + ) + .child( + note("Czech Republic") + .label('population', '10650000') + ) + ); + + const parsingContext = new ParsingContext(); + + const searchResults = await searchService.findNotesWithQuery('#country #population >= 10000000', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + }); + + it("logical or", async () => { + rootNote.child( + note("Europe") + .label('country', '', true) + .child( + note("Austria") + .label('languageFamily', 'germanic') + ) + .child( + note("Czech Republic") + .label('languageFamily', 'slavic') + ) + .child( + note("Hungary") + .label('languageFamily', 'finnougric') + ) + ); + + const parsingContext = new ParsingContext(); + + const searchResults = await searchService.findNotesWithQuery('#languageFamily = slavic OR #languageFamily = germanic', parsingContext); + expect(searchResults.length).toEqual(2); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + }); + + it("fuzzy attribute search", async () => { + rootNote.child( + note("Europe") + .label('country', '', true) + .child( + note("Austria") + .label('languageFamily', 'germanic') + ) + .child( + note("Czech Republic") + .label('languageFamily', 'slavic') + ) + ); + + let parsingContext = new ParsingContext({fuzzyAttributeSearch: false}); + + let searchResults = await searchService.findNotesWithQuery('#language', parsingContext); + expect(searchResults.length).toEqual(0); + + searchResults = await searchService.findNotesWithQuery('#languageFamily=ger', parsingContext); + expect(searchResults.length).toEqual(0); + + parsingContext = new ParsingContext({fuzzyAttributeSearch: true}); + + searchResults = await searchService.findNotesWithQuery('#language', parsingContext); + expect(searchResults.length).toEqual(2); + + searchResults = await searchService.findNotesWithQuery('#languageFamily=ger', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + }); }); /** @return {Note} */ @@ -75,11 +154,12 @@ class NoteBuilder { this.note = note; } - label(name, value) { + label(name, value, isInheritable = false) { new Attribute(noteCache, { attributeId: id(), noteId: this.note.noteId, type: 'label', + isInheritable, name, value }); diff --git a/src/services/search/comparator_builder.js b/src/services/search/comparator_builder.js index 72c1103a8..ef589aacf 100644 --- a/src/services/search/comparator_builder.js +++ b/src/services/search/comparator_builder.js @@ -1,6 +1,6 @@ const dayjs = require("dayjs"); -const comparators = { +const stringComparators = { "=": comparedValue => (val => val === comparedValue), "!=": comparedValue => (val => val !== comparedValue), ">": comparedValue => (val => val > comparedValue), @@ -10,7 +10,14 @@ const comparators = { "*=": comparedValue => (val => val.endsWith(comparedValue)), "=*": comparedValue => (val => val.startsWith(comparedValue)), "*=*": comparedValue => (val => val.includes(comparedValue)), -} +}; + +const numericComparators = { + ">": comparedValue => (val => parseFloat(val) > comparedValue), + ">=": comparedValue => (val => parseFloat(val) >= comparedValue), + "<": comparedValue => (val => parseFloat(val) < comparedValue), + "<=": comparedValue => (val => parseFloat(val) <= comparedValue) +}; const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i; @@ -58,8 +65,12 @@ function buildComparator(operator, comparedValue) { comparedValue = calculateSmartValue(comparedValue); - if (operator in comparators) { - return comparators[operator](comparedValue); + if (operator in numericComparators && !isNaN(comparedValue)) { + return numericComparators[operator](parseFloat(comparedValue)); + } + + if (operator in stringComparators) { + return stringComparators[operator](comparedValue); } } diff --git a/src/services/search/note_set.js b/src/services/search/note_set.js index 287f2f6bd..ad22d5487 100644 --- a/src/services/search/note_set.js +++ b/src/services/search/note_set.js @@ -19,7 +19,7 @@ class NoteSet { } mergeIn(anotherNoteSet) { - this.notes = this.notes.concat(anotherNoteSet.arr); + this.notes = this.notes.concat(anotherNoteSet.notes); } minus(anotherNoteSet) { From 4ea934509ebe0fe830fbaef9d85880b134f6606e Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 10:25:22 +0200 Subject: [PATCH 33/47] implemented property based access + parent --- spec/lexer.spec.js | 25 ++++--- spec/parser.spec.js | 26 +++---- spec/search.spec.js | 70 +++++++++++++++++++ src/services/search/expressions/and.js | 2 +- src/services/search/expressions/child_of.js | 36 ++++++++++ src/services/search/expressions/expression.js | 1 + ...ield_comparison.js => label_comparison.js} | 4 +- .../search/expressions/property_comparison.js | 29 ++++++++ src/services/search/lexer.js | 4 +- src/services/search/note_set.js | 12 +++- src/services/search/parser.js | 58 ++++++++++++--- 11 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 src/services/search/expressions/child_of.js rename src/services/search/expressions/{field_comparison.js => label_comparison.js} (92%) create mode 100644 src/services/search/expressions/property_comparison.js diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js index 7532faa35..1482f27ee 100644 --- a/spec/lexer.spec.js +++ b/spec/lexer.spec.js @@ -18,8 +18,8 @@ describe("Lexer fulltext", () => { }); it("you can use different quotes and other special characters inside quotes", () => { - expect(lexer("'i can use \" or ` or #@=*' without problem").fulltextTokens) - .toEqual(["i can use \" or ` or #@=*", "without", "problem"]); + expect(lexer("'i can use \" or ` or #~=*' without problem").fulltextTokens) + .toEqual(["i can use \" or ` or #~=*", "without", "problem"]); }); it("if quote is not ended then it's just one long token", () => { @@ -33,15 +33,15 @@ describe("Lexer fulltext", () => { }); it("escaping special characters", () => { - expect(lexer("hello \\#\\@\\'").fulltextTokens) - .toEqual(["hello", "#@'"]); + expect(lexer("hello \\#\\~\\'").fulltextTokens) + .toEqual(["hello", "#~'"]); }); }); describe("Lexer expression", () => { it("simple attribute existence", () => { - expect(lexer("#label @relation").expressionTokens) - .toEqual(["#label", "@relation"]); + expect(lexer("#label ~relation").expressionTokens) + .toEqual(["#label", "~relation"]); }); it("simple label operators", () => { @@ -50,12 +50,17 @@ describe("Lexer expression", () => { }); it("spaces in attribute names and values", () => { - expect(lexer(`#'long label'="hello o' world" @'long relation'`).expressionTokens) - .toEqual(["#long label", "=", "hello o' world", "@long relation"]); + expect(lexer(`#'long label'="hello o' world" ~'long relation'`).expressionTokens) + .toEqual(["#long label", "=", "hello o' world", "~long relation"]); }); it("complex expressions with and, or and parenthesis", () => { - expect(lexer(`# (#label=text OR #second=text) AND @relation`).expressionTokens) - .toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "@relation"]); + expect(lexer(`# (#label=text OR #second=text) AND ~relation`).expressionTokens) + .toEqual(["#", "(", "#label", "=", "text", "or", "#second", "=", "text", ")", "and", "~relation"]); + }); + + it("dot separated properties", () => { + expect(lexer(`# ~author.title = 'Hugh Howey' AND note.title = 'Silo'`).expressionTokens) + .toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "title", "=", "silo"]); }); }); diff --git a/spec/parser.spec.js b/spec/parser.spec.js index e0993e51d..360e679fa 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -37,7 +37,7 @@ describe("Parser", () => { parsingContext: new ParsingContext() }); - expect(rootExp.constructor.name).toEqual("FieldComparisonExp"); + expect(rootExp.constructor.name).toEqual("LabelComparisonExp"); expect(rootExp.attributeType).toEqual("label"); expect(rootExp.attributeName).toEqual("mylabel"); expect(rootExp.comparator).toBeTruthy(); @@ -53,10 +53,10 @@ describe("Parser", () => { expect(rootExp.constructor.name).toEqual("AndExp"); const [firstSub, secondSub] = rootExp.subExpressions; - expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.constructor.name).toEqual("LabelComparisonExp"); expect(firstSub.attributeName).toEqual("first"); - expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.constructor.name).toEqual("LabelComparisonExp"); expect(secondSub.attributeName).toEqual("second"); }); @@ -70,27 +70,27 @@ describe("Parser", () => { expect(rootExp.constructor.name).toEqual("AndExp"); const [firstSub, secondSub] = rootExp.subExpressions; - expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.constructor.name).toEqual("LabelComparisonExp"); expect(firstSub.attributeName).toEqual("first"); - expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.constructor.name).toEqual("LabelComparisonExp"); expect(secondSub.attributeName).toEqual("second"); }); it("simple label OR", () => { const rootExp = parser({ fulltextTokens: [], - expressionTokens: ["#first", "=", "text", "OR", "#second", "=", "text"], + expressionTokens: ["#first", "=", "text", "or", "#second", "=", "text"], parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("OrExp"); const [firstSub, secondSub] = rootExp.subExpressions; - expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.constructor.name).toEqual("LabelComparisonExp"); expect(firstSub.attributeName).toEqual("first"); - expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.constructor.name).toEqual("LabelComparisonExp"); expect(secondSub.attributeName).toEqual("second"); }); @@ -107,30 +107,30 @@ describe("Parser", () => { expect(firstSub.constructor.name).toEqual("NoteCacheFulltextExp"); expect(firstSub.tokens).toEqual(["hello"]); - expect(secondSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSub.constructor.name).toEqual("LabelComparisonExp"); expect(secondSub.attributeName).toEqual("mylabel"); }); it("label sub-expression", () => { const rootExp = parser({ fulltextTokens: [], - expressionTokens: ["#first", "=", "text", "OR", ["#second", "=", "text", "AND", "#third", "=", "text"]], + expressionTokens: ["#first", "=", "text", "or", ["#second", "=", "text", "and", "#third", "=", "text"]], parsingContext: new ParsingContext() }); expect(rootExp.constructor.name).toEqual("OrExp"); const [firstSub, secondSub] = rootExp.subExpressions; - expect(firstSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSub.constructor.name).toEqual("LabelComparisonExp"); expect(firstSub.attributeName).toEqual("first"); expect(secondSub.constructor.name).toEqual("AndExp"); const [firstSubSub, secondSubSub] = secondSub.subExpressions; - expect(firstSubSub.constructor.name).toEqual("FieldComparisonExp"); + expect(firstSubSub.constructor.name).toEqual("LabelComparisonExp"); expect(firstSubSub.attributeName).toEqual("second"); - expect(secondSubSub.constructor.name).toEqual("FieldComparisonExp"); + expect(secondSubSub.constructor.name).toEqual("LabelComparisonExp"); expect(secondSubSub.attributeName).toEqual("third"); }); }); diff --git a/spec/search.spec.js b/spec/search.spec.js index 19eb70345..e60ab658c 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -83,6 +83,31 @@ describe("Search", () => { expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); }); + it("numeric label comparison fallback to string comparison", async () => { + rootNote.child( + note("Europe") + .label('country', '', true) + .child( + note("Austria") + .label('established', '1955-07-27') + ) + .child( + note("Czech Republic") + .label('established', '1993-01-01') + ) + .child( + note("Hungary") + .label('established', '..wrong..') + ) + ); + + const parsingContext = new ParsingContext(); + + const searchResults = await searchService.findNotesWithQuery('#established < 1990', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + }); + it("logical or", async () => { rootNote.child( note("Europe") @@ -140,6 +165,51 @@ describe("Search", () => { expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); }); + + it("filter by note property", async () => { + rootNote.child( + note("Europe") + .child( + note("Austria") + ) + .child( + note("Czech Republic") + ) + ); + + const parsingContext = new ParsingContext(); + + const searchResults = await searchService.findNotesWithQuery('# note.title =* czech', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + }); + + it("filter by note's parent", async () => { + rootNote.child( + note("Europe") + .child( + note("Austria") + ) + .child( + note("Czech Republic") + ) + ) + .child( + note("Asia") + .child(note('Taiwan')) + ); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# note.parent.title = Europe', parsingContext); + expect(searchResults.length).toEqual(2); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('# note.parent.title = Asia', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Taiwan")).toBeTruthy(); + }); }); /** @return {Note} */ diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index 56973b11c..b5991503f 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -2,7 +2,7 @@ const Expression = require('./expression'); -class AndExp extends Expression{ +class AndExp extends Expression { static of(subExpressions) { subExpressions = subExpressions.filter(exp => !!exp); diff --git a/src/services/search/expressions/child_of.js b/src/services/search/expressions/child_of.js new file mode 100644 index 000000000..a98b0030c --- /dev/null +++ b/src/services/search/expressions/child_of.js @@ -0,0 +1,36 @@ +"use strict"; + +const Expression = require('./expression'); +const NoteSet = require('../note_set'); + +class ChildOfExp extends Expression { + constructor(subExpression) { + super(); + + this.subExpression = subExpression; + } + + execute(inputNoteSet, searchContext) { + const subInputNoteSet = new NoteSet(); + + for (const note of inputNoteSet.notes) { + subInputNoteSet.addAll(note.parents); + } + + const subResNoteSet = this.subExpression.execute(subInputNoteSet, searchContext); + + const resNoteSet = new NoteSet(); + + for (const parentNote of subResNoteSet.notes) { + for (const childNote of parentNote.children) { + if (inputNoteSet.hasNote(childNote)) { + resNoteSet.add(childNote); + } + } + } + + return resNoteSet; + } +} + +module.exports = ChildOfExp; diff --git a/src/services/search/expressions/expression.js b/src/services/search/expressions/expression.js index 49a1e64e5..48c247152 100644 --- a/src/services/search/expressions/expression.js +++ b/src/services/search/expressions/expression.js @@ -4,6 +4,7 @@ class Expression { /** * @param {NoteSet} noteSet * @param {object} searchContext + * @return {NoteSet} */ execute(noteSet, searchContext) {} } diff --git a/src/services/search/expressions/field_comparison.js b/src/services/search/expressions/label_comparison.js similarity index 92% rename from src/services/search/expressions/field_comparison.js rename to src/services/search/expressions/label_comparison.js index c497552d7..b69d07102 100644 --- a/src/services/search/expressions/field_comparison.js +++ b/src/services/search/expressions/label_comparison.js @@ -4,7 +4,7 @@ const Expression = require('./expression'); const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); -class FieldComparisonExp extends Expression { +class LabelComparisonExp extends Expression { constructor(attributeType, attributeName, comparator) { super(); @@ -37,4 +37,4 @@ class FieldComparisonExp extends Expression { } } -module.exports = FieldComparisonExp; +module.exports = LabelComparisonExp; diff --git a/src/services/search/expressions/property_comparison.js b/src/services/search/expressions/property_comparison.js new file mode 100644 index 000000000..1ca544aa0 --- /dev/null +++ b/src/services/search/expressions/property_comparison.js @@ -0,0 +1,29 @@ +"use strict"; + +const Expression = require('./expression'); +const NoteSet = require('../note_set'); + +class PropertyComparisonExp extends Expression { + constructor(propertyName, comparator) { + super(); + + this.propertyName = propertyName; + this.comparator = comparator; + } + + execute(noteSet, searchContext) { + const resNoteSet = new NoteSet(); + + for (const note of noteSet.notes) { + const value = note[this.propertyName].toLowerCase(); + + if (this.comparator(value)) { + resNoteSet.add(note); + } + } + + return resNoteSet; + } +} + +module.exports = PropertyComparisonExp; diff --git a/src/services/search/lexer.js b/src/services/search/lexer.js index 8a5b90f0a..635d0d031 100644 --- a/src/services/search/lexer.js +++ b/src/services/search/lexer.js @@ -77,7 +77,7 @@ function lexer(str) { continue; } else if (!quotes) { - if (currentWord.length === 0 && (chr === '#' || chr === '@')) { + if (currentWord.length === 0 && (chr === '#' || chr === '~')) { fulltextEnded = true; currentWord = chr; @@ -87,7 +87,7 @@ function lexer(str) { finishWord(); continue; } - else if (fulltextEnded && ['(', ')'].includes(chr)) { + else if (fulltextEnded && ['(', ')', '.'].includes(chr)) { finishWord(); currentWord += chr; finishWord(); diff --git a/src/services/search/note_set.js b/src/services/search/note_set.js index ad22d5487..40b38de35 100644 --- a/src/services/search/note_set.js +++ b/src/services/search/note_set.js @@ -6,11 +6,19 @@ class NoteSet { } add(note) { - this.notes.push(note); + if (!this.hasNote(note)) { + this.notes.push(note); + } } addAll(notes) { - this.notes.push(...notes); + for (const note of notes) { + this.add(note); + } + } + + hasNote(note) { + return this.hasNoteId(note.noteId); } hasNoteId(noteId) { diff --git a/src/services/search/parser.js b/src/services/search/parser.js index d13e1c93d..427b8e456 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -3,8 +3,10 @@ const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); +const ChildOfExp = require('./expressions/child_of'); +const PropertyComparisonExp = require('./expressions/property_comparison'); const AttributeExistsExp = require('./expressions/attribute_exists'); -const FieldComparisonExp = require('./expressions/field_comparison'); +const LabelComparisonExp = require('./expressions/label_comparison'); const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); const comparatorBuilder = require('./comparator_builder'); @@ -38,17 +40,50 @@ function getExpression(tokens, parsingContext) { const expressions = []; let op = null; - for (let i = 0; i < tokens.length; i++) { + let i; + + function parseNoteProperty() { + if (tokens[i] !== '.') { + parsingContext.addError('Expected "." to separate field path'); + return; + } + + i++; + + if (tokens[i] === 'parent') { + i += 1; + + return new ChildOfExp(parseNoteProperty()); + } + + if (tokens[i] === 'title') { + const propertyName = tokens[i]; + const operator = tokens[i + 1]; + const comparedValue = tokens[i + 2]; + const comparator = comparatorBuilder(operator, comparedValue); + + if (!comparator) { + parsingContext.addError(`Can't find operator '${operator}'`); + return; + } + + i += 3; + + return new PropertyComparisonExp(propertyName, comparator); + } + } + + for (i = 0; i < tokens.length; i++) { const token = tokens[i]; - if (token === '#' || token === '@') { + if (token === '#' || token === '~') { continue; } if (Array.isArray(token)) { expressions.push(getExpression(token, parsingContext)); } - else if (token.startsWith('#') || token.startsWith('@')) { + else if (token.startsWith('#') || token.startsWith('~')) { const type = token.startsWith('#') ? 'label' : 'relation'; parsingContext.highlightedTokens.push(token.substr(1)); @@ -70,7 +105,7 @@ function getExpression(tokens, parsingContext) { continue; } - expressions.push(new FieldComparisonExp(type, token.substr(1), comparator)); + expressions.push(new LabelComparisonExp(type, token.substr(1), comparator)); i += 2; } @@ -78,11 +113,18 @@ function getExpression(tokens, parsingContext) { expressions.push(new AttributeExistsExp(type, token.substr(1), parsingContext.fuzzyAttributeSearch)); } } - else if (['and', 'or'].includes(token.toLowerCase())) { + else if (token === 'note') { + i++; + + expressions.push(parseNoteProperty(tokens)); + + continue; + } + else if (['and', 'or'].includes(token)) { if (!op) { - op = token.toLowerCase(); + op = token; } - else if (op !== token.toLowerCase()) { + else if (op !== token) { parsingContext.addError('Mixed usage of AND/OR - always use parenthesis to group AND/OR expressions.'); } } From 3d12341ff15502c18317e30941163d7054aaedcf Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 10:36:49 +0200 Subject: [PATCH 34/47] added querying by children --- spec/search.spec.js | 157 ++++++++----------- src/services/search/expressions/parent_of.js | 36 +++++ src/services/search/parser.js | 7 + 3 files changed, 112 insertions(+), 88 deletions(-) create mode 100644 src/services/search/expressions/parent_of.js diff --git a/spec/search.spec.js b/spec/search.spec.js index e60ab658c..053c17c44 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -17,11 +17,9 @@ describe("Search", () => { }); it("simple path match", async () => { - rootNote.child( - note("Europe") - .child( - note("Austria") - ) + rootNote + .child(note("Europe") + .child(note("Austria")) ); const parsingContext = new ParsingContext(); @@ -32,11 +30,9 @@ describe("Search", () => { }); it("only end leafs are results", async () => { - rootNote.child( - note("Europe") - .child( - note("Austria") - ) + rootNote + .child(note("Europe") + .child(note("Austria")) ); const parsingContext = new ParsingContext(); @@ -47,12 +43,10 @@ describe("Search", () => { }); it("only end leafs are results", async () => { - rootNote.child( - note("Europe") - .child( - note("Austria") - .label('capital', 'Vienna') - ) + rootNote + .child(note("Europe") + .child(note("Austria") + .label('capital', 'Vienna')) ); const parsingContext = new ParsingContext(); @@ -63,17 +57,13 @@ describe("Search", () => { }); it("numeric label comparison", async () => { - rootNote.child( - note("Europe") + rootNote + .child(note("Europe") .label('country', '', true) - .child( - note("Austria") - .label('population', '8859000') - ) - .child( - note("Czech Republic") - .label('population', '10650000') - ) + .child(note("Austria") + .label('population', '8859000')) + .child(note("Czech Republic") + .label('population', '10650000')) ); const parsingContext = new ParsingContext(); @@ -84,21 +74,15 @@ describe("Search", () => { }); it("numeric label comparison fallback to string comparison", async () => { - rootNote.child( - note("Europe") + rootNote + .child(note("Europe") .label('country', '', true) - .child( - note("Austria") - .label('established', '1955-07-27') - ) - .child( - note("Czech Republic") - .label('established', '1993-01-01') - ) - .child( - note("Hungary") - .label('established', '..wrong..') - ) + .child(note("Austria") + .label('established', '1955-07-27')) + .child(note("Czech Republic") + .label('established', '1993-01-01')) + .child(note("Hungary") + .label('established', '..wrong..')) ); const parsingContext = new ParsingContext(); @@ -109,22 +93,16 @@ describe("Search", () => { }); it("logical or", async () => { - rootNote.child( - note("Europe") + rootNote + .child(note("Europe") .label('country', '', true) - .child( - note("Austria") - .label('languageFamily', 'germanic') - ) - .child( - note("Czech Republic") - .label('languageFamily', 'slavic') - ) - .child( - note("Hungary") - .label('languageFamily', 'finnougric') - ) - ); + .child(note("Austria") + .label('languageFamily', 'germanic')) + .child(note("Czech Republic") + .label('languageFamily', 'slavic')) + .child(note("Hungary") + .label('languageFamily', 'finnougric')) + ); const parsingContext = new ParsingContext(); @@ -135,18 +113,13 @@ describe("Search", () => { }); it("fuzzy attribute search", async () => { - rootNote.child( - note("Europe") + rootNote + .child(note("Europe") .label('country', '', true) - .child( - note("Austria") - .label('languageFamily', 'germanic') - ) - .child( - note("Czech Republic") - .label('languageFamily', 'slavic') - ) - ); + .child(note("Austria") + .label('languageFamily', 'germanic')) + .child(note("Czech Republic") + .label('languageFamily', 'slavic'))); let parsingContext = new ParsingContext({fuzzyAttributeSearch: false}); @@ -167,15 +140,10 @@ describe("Search", () => { }); it("filter by note property", async () => { - rootNote.child( - note("Europe") - .child( - note("Austria") - ) - .child( - note("Czech Republic") - ) - ); + rootNote + .child(note("Europe") + .child(note("Austria")) + .child(note("Czech Republic"))); const parsingContext = new ParsingContext(); @@ -185,19 +153,12 @@ describe("Search", () => { }); it("filter by note's parent", async () => { - rootNote.child( - note("Europe") - .child( - note("Austria") - ) - .child( - note("Czech Republic") - ) - ) - .child( - note("Asia") - .child(note('Taiwan')) - ); + rootNote + .child(note("Europe") + .child(note("Austria")) + .child(note("Czech Republic"))) + .child(note("Asia") + .child(note('Taiwan'))); const parsingContext = new ParsingContext(); @@ -210,6 +171,26 @@ describe("Search", () => { expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Taiwan")).toBeTruthy(); }); + + it("filter by note's child", async () => { + rootNote + .child(note("Europe") + .child(note("Austria")) + .child(note("Czech Republic"))) + .child(note("Oceania") + .child(note('Australia'))); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# note.child.title =* Aust', parsingContext); + expect(searchResults.length).toEqual(2); + expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); + expect(findNoteByTitle(searchResults, "Oceania")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('# note.child.title =* Aust AND note.child.title *= republic', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); + }); }); /** @return {Note} */ diff --git a/src/services/search/expressions/parent_of.js b/src/services/search/expressions/parent_of.js new file mode 100644 index 000000000..e7924feb7 --- /dev/null +++ b/src/services/search/expressions/parent_of.js @@ -0,0 +1,36 @@ +"use strict"; + +const Expression = require('./expression'); +const NoteSet = require('../note_set'); + +class ParentOfExp extends Expression { + constructor(subExpression) { + super(); + + this.subExpression = subExpression; + } + + execute(inputNoteSet, searchContext) { + const subInputNoteSet = new NoteSet(); + + for (const note of inputNoteSet.notes) { + subInputNoteSet.addAll(note.children); + } + + const subResNoteSet = this.subExpression.execute(subInputNoteSet, searchContext); + + const resNoteSet = new NoteSet(); + + for (const childNote of subResNoteSet.notes) { + for (const parentNote of childNote.parents) { + if (inputNoteSet.hasNote(parentNote)) { + resNoteSet.add(parentNote); + } + } + } + + return resNoteSet; + } +} + +module.exports = ParentOfExp; diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 427b8e456..9ca4b149c 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -4,6 +4,7 @@ const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); const ChildOfExp = require('./expressions/child_of'); +const ParentOfExp = require('./expressions/parent_of'); const PropertyComparisonExp = require('./expressions/property_comparison'); const AttributeExistsExp = require('./expressions/attribute_exists'); const LabelComparisonExp = require('./expressions/label_comparison'); @@ -56,6 +57,12 @@ function getExpression(tokens, parsingContext) { return new ChildOfExp(parseNoteProperty()); } + if (tokens[i] === 'child') { + i += 1; + + return new ParentOfExp(parseNoteProperty()); + } + if (tokens[i] === 'title') { const propertyName = tokens[i]; const operator = tokens[i + 1]; From 355ffd3d029e6a2fc609c8e8cfe3e35afcdf1c5c Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 12:27:44 +0200 Subject: [PATCH 35/47] added querying by relation's properties --- spec/search.spec.js | 29 ++++++++++++- src/services/note_cache/entities/attribute.js | 2 +- src/services/search/expressions/and.js | 6 +-- .../search/expressions/attribute_exists.js | 4 +- src/services/search/expressions/expression.js | 4 +- .../search/expressions/label_comparison.js | 4 +- src/services/search/expressions/not.js | 6 +-- .../search/expressions/note_cache_fulltext.js | 4 +- .../expressions/note_content_fulltext.js | 4 +- src/services/search/expressions/or.js | 4 +- .../search/expressions/property_comparison.js | 4 +- .../search/expressions/relation_where.js | 41 +++++++++++++++++++ src/services/search/note_set.js | 12 ++++++ src/services/search/parser.js | 25 ++++++++--- 14 files changed, 120 insertions(+), 29 deletions(-) create mode 100644 src/services/search/expressions/relation_where.js diff --git a/spec/search.spec.js b/spec/search.spec.js index 053c17c44..8a4589499 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -191,6 +191,31 @@ describe("Search", () => { expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); }); + + it("filter by relation's note properties", async () => { + const austria = note("Austria"); + const portugal = note("Portugal"); + + rootNote + .child(note("Europe") + .child(austria) + .child(note("Czech Republic") + .relation('neighbor', austria.note)) + .child(portugal) + .child(note("Spain") + .relation('neighbor', portugal.note)) + ); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# ~neighbor.title = Austria', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('# ~neighbor.title = Portugal', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Spain")).toBeTruthy(); + }); }); /** @return {Note} */ @@ -218,13 +243,13 @@ class NoteBuilder { return this; } - relation(name, note) { + relation(name, targetNote) { new Attribute(noteCache, { attributeId: id(), noteId: this.note.noteId, type: 'relation', name, - value: note.noteId + value: targetNote.noteId }); return this; diff --git a/src/services/note_cache/entities/attribute.js b/src/services/note_cache/entities/attribute.js index 0de3c539f..73ae80e86 100644 --- a/src/services/note_cache/entities/attribute.js +++ b/src/services/note_cache/entities/attribute.js @@ -13,7 +13,7 @@ class Attribute { /** @param {string} */ this.name = row.name.toLowerCase(); /** @param {string} */ - this.value = row.value.toLowerCase(); + this.value = row.type === 'label'? row.value.toLowerCase() : row.value; /** @param {boolean} */ this.isInheritable = !!row.isInheritable; diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index b5991503f..ee22f6b13 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -18,12 +18,12 @@ class AndExp extends Expression { this.subExpressions = subExpressions; } - execute(noteSet, searchContext) { + execute(inputNoteSet, searchContext) { for (const subExpression of this.subExpressions) { - noteSet = subExpression.execute(noteSet, searchContext); + inputNoteSet = subExpression.execute(inputNoteSet, searchContext); } - return noteSet; + return inputNoteSet; } } diff --git a/src/services/search/expressions/attribute_exists.js b/src/services/search/expressions/attribute_exists.js index 8b1da1b11..b368ab92a 100644 --- a/src/services/search/expressions/attribute_exists.js +++ b/src/services/search/expressions/attribute_exists.js @@ -13,7 +13,7 @@ class AttributeExistsExp extends Expression { this.prefixMatch = prefixMatch; } - execute(noteSet) { + execute(inputNoteSet) { const attrs = this.prefixMatch ? noteCache.findAttributesWithPrefix(this.attributeType, this.attributeName) : noteCache.findAttributes(this.attributeType, this.attributeName); @@ -23,7 +23,7 @@ class AttributeExistsExp extends Expression { for (const attr of attrs) { const note = attr.note; - if (noteSet.hasNoteId(note.noteId)) { + if (inputNoteSet.hasNoteId(note.noteId)) { if (attr.isInheritable) { resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); } diff --git a/src/services/search/expressions/expression.js b/src/services/search/expressions/expression.js index 48c247152..41192cbb4 100644 --- a/src/services/search/expressions/expression.js +++ b/src/services/search/expressions/expression.js @@ -2,11 +2,11 @@ class Expression { /** - * @param {NoteSet} noteSet + * @param {NoteSet} inputNoteSet * @param {object} searchContext * @return {NoteSet} */ - execute(noteSet, searchContext) {} + execute(inputNoteSet, searchContext) {} } module.exports = Expression; diff --git a/src/services/search/expressions/label_comparison.js b/src/services/search/expressions/label_comparison.js index b69d07102..143c41b6e 100644 --- a/src/services/search/expressions/label_comparison.js +++ b/src/services/search/expressions/label_comparison.js @@ -13,14 +13,14 @@ class LabelComparisonExp extends Expression { this.comparator = comparator; } - execute(noteSet) { + execute(inputNoteSet) { const attrs = noteCache.findAttributes(this.attributeType, this.attributeName); const resultNoteSet = new NoteSet(); for (const attr of attrs) { const note = attr.note; - if (noteSet.hasNoteId(note.noteId) && this.comparator(attr.value)) { + if (inputNoteSet.hasNoteId(note.noteId) && this.comparator(attr.value)) { if (attr.isInheritable) { resultNoteSet.addAll(note.subtreeNotesIncludingTemplated); } diff --git a/src/services/search/expressions/not.js b/src/services/search/expressions/not.js index 434a46dc6..a24d3c2c1 100644 --- a/src/services/search/expressions/not.js +++ b/src/services/search/expressions/not.js @@ -9,10 +9,10 @@ class NotExp extends Expression { this.subExpression = subExpression; } - execute(noteSet, searchContext) { - const subNoteSet = this.subExpression.execute(noteSet, searchContext); + execute(inputNoteSet, searchContext) { + const subNoteSet = this.subExpression.execute(inputNoteSet, searchContext); - return noteSet.minus(subNoteSet); + return inputNoteSet.minus(subNoteSet); } } diff --git a/src/services/search/expressions/note_cache_fulltext.js b/src/services/search/expressions/note_cache_fulltext.js index 93ec54328..eedc3e279 100644 --- a/src/services/search/expressions/note_cache_fulltext.js +++ b/src/services/search/expressions/note_cache_fulltext.js @@ -11,7 +11,7 @@ class NoteCacheFulltextExp extends Expression { this.tokens = tokens; } - execute(noteSet, searchContext) { + execute(inputNoteSet, searchContext) { // has deps on SQL which breaks unit test so needs to be dynamically required const noteCacheService = require('../../note_cache/note_cache_service'); const resultNoteSet = new NoteSet(); @@ -66,7 +66,7 @@ class NoteCacheFulltextExp extends Expression { } } - const candidateNotes = this.getCandidateNotes(noteSet); + const candidateNotes = this.getCandidateNotes(inputNoteSet); for (const note of candidateNotes) { // autocomplete should be able to find notes by their noteIds as well (only leafs) diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index 3148ded4d..94932cde7 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -11,7 +11,7 @@ class NoteContentFulltextExp extends Expression { this.tokens = tokens; } - async execute(noteSet) { + async execute(inputNoteSet) { const resultNoteSet = new NoteSet(); const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); @@ -24,7 +24,7 @@ class NoteContentFulltextExp extends Expression { WHERE isDeleted = 0 AND isProtected = 0 AND ${wheres.join(' AND ')}`); for (const noteId of noteIds) { - if (noteSet.hasNoteId(noteId) && noteId in noteCache.notes) { + if (inputNoteSet.hasNoteId(noteId) && noteId in noteCache.notes) { resultNoteSet.add(noteCache.notes[noteId]); } } diff --git a/src/services/search/expressions/or.js b/src/services/search/expressions/or.js index 3b21f24c5..62c16f5cf 100644 --- a/src/services/search/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -21,11 +21,11 @@ class OrExp extends Expression { this.subExpressions = subExpressions; } - execute(noteSet, searchContext) { + execute(inputNoteSet, searchContext) { const resultNoteSet = new NoteSet(); for (const subExpression of this.subExpressions) { - resultNoteSet.mergeIn(subExpression.execute(noteSet, searchContext)); + resultNoteSet.mergeIn(subExpression.execute(inputNoteSet, searchContext)); } return resultNoteSet; diff --git a/src/services/search/expressions/property_comparison.js b/src/services/search/expressions/property_comparison.js index 1ca544aa0..730bf5597 100644 --- a/src/services/search/expressions/property_comparison.js +++ b/src/services/search/expressions/property_comparison.js @@ -11,10 +11,10 @@ class PropertyComparisonExp extends Expression { this.comparator = comparator; } - execute(noteSet, searchContext) { + execute(inputNoteSet, searchContext) { const resNoteSet = new NoteSet(); - for (const note of noteSet.notes) { + for (const note of inputNoteSet.notes) { const value = note[this.propertyName].toLowerCase(); if (this.comparator(value)) { diff --git a/src/services/search/expressions/relation_where.js b/src/services/search/expressions/relation_where.js new file mode 100644 index 000000000..f873762e0 --- /dev/null +++ b/src/services/search/expressions/relation_where.js @@ -0,0 +1,41 @@ +"use strict"; + +const Expression = require('./expression'); +const NoteSet = require('../note_set'); +const noteCache = require('../../note_cache/note_cache'); + +class RelationWhereExp extends Expression { + constructor(relationName, subExpression) { + super(); + + this.relationName = relationName; + this.subExpression = subExpression; + } + + execute(inputNoteSet, searchContext) { + const candidateNoteSet = new NoteSet(); + + for (const attr of noteCache.findAttributes('relation', this.relationName)) { + const note = attr.note; + + if (inputNoteSet.hasNoteId(note.noteId)) { + const subInputNoteSet = new NoteSet([attr.targetNote]); + const subResNoteSet = this.subExpression.execute(subInputNoteSet, searchContext); + + if (subResNoteSet.hasNote(attr.targetNote)) { + if (attr.isInheritable) { + candidateNoteSet.addAll(note.subtreeNotesIncludingTemplated); + } else if (note.isTemplate) { + candidateNoteSet.addAll(note.templatedNotes); + } else { + candidateNoteSet.add(note); + } + } + } + } + + return candidateNoteSet.intersection(inputNoteSet); + } +} + +module.exports = RelationWhereExp; diff --git a/src/services/search/note_set.js b/src/services/search/note_set.js index 40b38de35..85c402a29 100644 --- a/src/services/search/note_set.js +++ b/src/services/search/note_set.js @@ -41,6 +41,18 @@ class NoteSet { return newNoteSet; } + + intersection(anotherNoteSet) { + const newNoteSet = new NoteSet(); + + for (const note of this.notes) { + if (anotherNoteSet.hasNote(note)) { + newNoteSet.add(note); + } + } + + return newNoteSet; + } } module.exports = NoteSet; diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 9ca4b149c..bb03b2d73 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -5,6 +5,7 @@ const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); const ChildOfExp = require('./expressions/child_of'); const ParentOfExp = require('./expressions/parent_of'); +const RelationWhereExp = require('./expressions/relation_where'); const PropertyComparisonExp = require('./expressions/property_comparison'); const AttributeExistsExp = require('./expressions/attribute_exists'); const LabelComparisonExp = require('./expressions/label_comparison'); @@ -90,10 +91,9 @@ function getExpression(tokens, parsingContext) { if (Array.isArray(token)) { expressions.push(getExpression(token, parsingContext)); } - else if (token.startsWith('#') || token.startsWith('~')) { - const type = token.startsWith('#') ? 'label' : 'relation'; - - parsingContext.highlightedTokens.push(token.substr(1)); + else if (token.startsWith('#')) { + const labelName = token.substr(1); + parsingContext.highlightedTokens.push(labelName); if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { let operator = tokens[i + 1]; @@ -112,12 +112,25 @@ function getExpression(tokens, parsingContext) { continue; } - expressions.push(new LabelComparisonExp(type, token.substr(1), comparator)); + expressions.push(new LabelComparisonExp('label', labelName, comparator)); i += 2; } else { - expressions.push(new AttributeExistsExp(type, token.substr(1), parsingContext.fuzzyAttributeSearch)); + expressions.push(new AttributeExistsExp('label', labelName, parsingContext.fuzzyAttributeSearch)); + } + } + else if (token.startsWith('~')) { + const relationName = token.substr(1); + parsingContext.highlightedTokens.push(relationName); + + if (i < tokens.length - 2 && tokens[i + 1] === '.') { + i += 1; + + expressions.push(new RelationWhereExp(relationName, parseNoteProperty())); + } + else { + expressions.push(new AttributeExistsExp('relation', relationName, parsingContext.fuzzyAttributeSearch)); } } else if (token === 'note') { From ae772288e2e219810a6a6f6a136869117e2d216e Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 18:13:35 +0200 Subject: [PATCH 36/47] support for long syntax of labels and relations --- spec/search.spec.js | 63 +++++++++++++++++++-- src/services/search/parser.js | 101 ++++++++++++++++++++++------------ 2 files changed, 123 insertions(+), 41 deletions(-) diff --git a/spec/search.spec.js b/spec/search.spec.js index 8a4589499..f0b1fcbc4 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -56,6 +56,38 @@ describe("Search", () => { expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); }); + it("label comparison with short syntax", async () => { + rootNote + .child(note("Europe") + .child(note("Austria") + .label('capital', 'Vienna')) + .child(note("Czech Republic") + .label('capital', 'Prague')) + ); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('#capital=Vienna', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + }); + + it("label comparison with full syntax", async () => { + rootNote + .child(note("Europe") + .child(note("Austria") + .label('capital', 'Vienna')) + .child(note("Czech Republic") + .label('capital', 'Prague')) + ); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# note.labels.capital=Prague', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + }); + it("numeric label comparison", async () => { rootNote .child(note("Europe") @@ -162,12 +194,12 @@ describe("Search", () => { const parsingContext = new ParsingContext(); - let searchResults = await searchService.findNotesWithQuery('# note.parent.title = Europe', parsingContext); + let searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe', parsingContext); expect(searchResults.length).toEqual(2); expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); - searchResults = await searchService.findNotesWithQuery('# note.parent.title = Asia', parsingContext); + searchResults = await searchService.findNotesWithQuery('# note.parents.title = Asia', parsingContext); expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Taiwan")).toBeTruthy(); }); @@ -182,17 +214,17 @@ describe("Search", () => { const parsingContext = new ParsingContext(); - let searchResults = await searchService.findNotesWithQuery('# note.child.title =* Aust', parsingContext); + let searchResults = await searchService.findNotesWithQuery('# note.children.title =* Aust', parsingContext); expect(searchResults.length).toEqual(2); expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); expect(findNoteByTitle(searchResults, "Oceania")).toBeTruthy(); - searchResults = await searchService.findNotesWithQuery('# note.child.title =* Aust AND note.child.title *= republic', parsingContext); + searchResults = await searchService.findNotesWithQuery('# note.children.title =* Aust AND note.children.title *= republic', parsingContext); expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); }); - it("filter by relation's note properties", async () => { + it("filter by relation's note properties using short syntax", async () => { const austria = note("Austria"); const portugal = note("Portugal"); @@ -216,6 +248,27 @@ describe("Search", () => { expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Spain")).toBeTruthy(); }); + + it("filter by relation's note properties using long syntax", async () => { + const austria = note("Austria"); + const portugal = note("Portugal"); + + rootNote + .child(note("Europe") + .child(austria) + .child(note("Czech Republic") + .relation('neighbor', austria.note)) + .child(portugal) + .child(note("Spain") + .relation('neighbor', portugal.note)) + ); + + const parsingContext = new ParsingContext(); + + const searchResults = await searchService.findNotesWithQuery('# note.relations.neighbor.title = Austria', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + }); }); /** @return {Note} */ diff --git a/src/services/search/parser.js b/src/services/search/parser.js index bb03b2d73..db6c3b21c 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -52,18 +52,40 @@ function getExpression(tokens, parsingContext) { i++; - if (tokens[i] === 'parent') { + if (tokens[i] === 'parents') { i += 1; return new ChildOfExp(parseNoteProperty()); } - if (tokens[i] === 'child') { + if (tokens[i] === 'children') { i += 1; return new ParentOfExp(parseNoteProperty()); } + if (tokens[i] === 'labels') { + if (tokens[i + 1] !== '.') { + parsingContext.addError(`Expected "." to separate field path, god "${tokens[i + 1]}"`); + return; + } + + i += 2; + + return parseLabel(tokens[i]); + } + + if (tokens[i] === 'relations') { + if (tokens[i + 1] !== '.') { + parsingContext.addError(`Expected "." to separate field path, god "${tokens[i + 1]}"`); + return; + } + + i += 2; + + return parseRelation(tokens[i]); + } + if (tokens[i] === 'title') { const propertyName = tokens[i]; const operator = tokens[i + 1]; @@ -81,6 +103,45 @@ function getExpression(tokens, parsingContext) { } } + function parseLabel(labelName) { + parsingContext.highlightedTokens.push(labelName); + + if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { + let operator = tokens[i + 1]; + const comparedValue = tokens[i + 2]; + + parsingContext.highlightedTokens.push(comparedValue); + + if (parsingContext.fuzzyAttributeSearch && operator === '=') { + operator = '*=*'; + } + + const comparator = comparatorBuilder(operator, comparedValue); + + if (!comparator) { + parsingContext.addError(`Can't find operator '${operator}'`); + } else { + i += 2; + + return new LabelComparisonExp('label', labelName, comparator); + } + } else { + return new AttributeExistsExp('label', labelName, parsingContext.fuzzyAttributeSearch); + } + } + + function parseRelation(relationName) { + parsingContext.highlightedTokens.push(relationName); + + if (i < tokens.length - 2 && tokens[i + 1] === '.') { + i += 1; + + return new RelationWhereExp(relationName, parseNoteProperty()); + } else { + return new AttributeExistsExp('relation', relationName, parsingContext.fuzzyAttributeSearch); + } + } + for (i = 0; i < tokens.length; i++) { const token = tokens[i]; @@ -93,45 +154,13 @@ function getExpression(tokens, parsingContext) { } else if (token.startsWith('#')) { const labelName = token.substr(1); - parsingContext.highlightedTokens.push(labelName); - if (i < tokens.length - 2 && isOperator(tokens[i + 1])) { - let operator = tokens[i + 1]; - const comparedValue = tokens[i + 2]; - - parsingContext.highlightedTokens.push(comparedValue); - - if (parsingContext.fuzzyAttributeSearch && operator === '=') { - operator = '*=*'; - } - - const comparator = comparatorBuilder(operator, comparedValue); - - if (!comparator) { - parsingContext.addError(`Can't find operator '${operator}'`); - continue; - } - - expressions.push(new LabelComparisonExp('label', labelName, comparator)); - - i += 2; - } - else { - expressions.push(new AttributeExistsExp('label', labelName, parsingContext.fuzzyAttributeSearch)); - } + expressions.push(parseLabel(labelName)); } else if (token.startsWith('~')) { const relationName = token.substr(1); - parsingContext.highlightedTokens.push(relationName); - if (i < tokens.length - 2 && tokens[i + 1] === '.') { - i += 1; - - expressions.push(new RelationWhereExp(relationName, parseNoteProperty())); - } - else { - expressions.push(new AttributeExistsExp('relation', relationName, parsingContext.fuzzyAttributeSearch)); - } + expressions.push(parseRelation(relationName)); } else if (token === 'note') { i++; From bb03a8714a55a08d83fa37b77ff5c27e5d73c1d9 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 18:42:32 +0200 Subject: [PATCH 37/47] more tests --- spec/search.spec.js | 49 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/spec/search.spec.js b/spec/search.spec.js index f0b1fcbc4..32108f8e3 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -188,7 +188,9 @@ describe("Search", () => { rootNote .child(note("Europe") .child(note("Austria")) - .child(note("Czech Republic"))) + .child(note("Czech Republic") + .child(note("Prague"))) + ) .child(note("Asia") .child(note('Taiwan'))); @@ -202,13 +204,19 @@ describe("Search", () => { searchResults = await searchService.findNotesWithQuery('# note.parents.title = Asia', parsingContext); expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Taiwan")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('# note.parents.parents.title = Europe', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Prague")).toBeTruthy(); }); it("filter by note's child", async () => { rootNote .child(note("Europe") - .child(note("Austria")) - .child(note("Czech Republic"))) + .child(note("Austria") + .child(note("Vienna"))) + .child(note("Czech Republic") + .child(note("Prague")))) .child(note("Oceania") .child(note('Australia'))); @@ -222,6 +230,10 @@ describe("Search", () => { searchResults = await searchService.findNotesWithQuery('# note.children.title =* Aust AND note.children.title *= republic', parsingContext); expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('# note.children.children.title = Prague', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Europe")).toBeTruthy(); }); it("filter by relation's note properties using short syntax", async () => { @@ -269,6 +281,37 @@ describe("Search", () => { expect(searchResults.length).toEqual(1); expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); }); + + it("filter by multiple level relation", async () => { + const austria = note("Austria"); + const slovakia = note("Slovakia"); + const italy = note("Italy"); + const ukraine = note("Ukraine"); + + rootNote + .child(note("Europe") + .child(austria + .relation('neighbor', italy.note) + .relation('neighbor', slovakia.note)) + .child(note("Czech Republic") + .relation('neighbor', austria.note) + .relation('neighbor', slovakia.note)) + .child(slovakia + .relation('neighbor', ukraine.note)) + .child(ukraine) + ); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# note.relations.neighbor.relations.neighbor.title = Italy', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('# note.relations.neighbor.relations.neighbor.title = Ukraine', parsingContext); + expect(searchResults.length).toEqual(2); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + }); }); /** @return {Note} */ From a2e1fb35b864523497d3397542a0a93389afbf50 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 20:52:55 +0200 Subject: [PATCH 38/47] tests for note properties --- spec/search.spec.js | 78 +++++++++++++++++++ src/services/note_cache/entities/note.js | 34 ++++++++ src/services/note_cache/note_cache_loader.js | 2 +- .../search/expressions/property_comparison.js | 37 ++++++++- src/services/search/parser.js | 4 +- 5 files changed, 151 insertions(+), 4 deletions(-) diff --git a/spec/search.spec.js b/spec/search.spec.js index 32108f8e3..46d368559 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -312,6 +312,84 @@ describe("Search", () => { expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); }); + + it("test note properties", async () => { + const austria = note("Austria"); + + austria.relation('myself', austria.note); + austria.label('capital', 'Vienna'); + austria.label('population', '8859000'); + + rootNote + .child(note("Asia")) + .child(note("Europe") + .child(austria + .child(note("Vienna")) + .child(note("Sebastian Kurz")) + ) + ) + .child(note("Mozart") + .child(austria)); + + austria.note.type = 'text'; + austria.note.mime = 'text/html'; + austria.note.isProtected = false; + austria.note.dateCreated = '2020-05-14 12:11:42.001+0200'; + austria.note.dateModified = '2020-05-14 13:11:42.001+0200'; + austria.note.utcDateCreated = '2020-05-14 10:11:42.001Z'; + austria.note.utcDateModified = '2020-05-14 11:11:42.001Z'; + austria.note.contentLength = 1001; + + const parsingContext = new ParsingContext(); + + async function test(propertyName, value, expectedResultCount) { + const searchResults = await searchService.findNotesWithQuery(`# note.${propertyName} = ${value}`, parsingContext); + expect(searchResults.length).toEqual(expectedResultCount); + + if (expectedResultCount === 1) { + expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + } + } + + await test("type", "text", 1); + await test("type", "code", 0); + + await test("mime", "text/html", 1); + await test("mime", "application/json", 0); + + await test("isProtected", "false", 7); + await test("isProtected", "true", 0); + + await test("dateCreated", "'2020-05-14 12:11:42.001+0200'", 1); + await test("dateCreated", "wrong", 0); + + await test("dateModified", "'2020-05-14 13:11:42.001+0200'", 1); + await test("dateModified", "wrong", 0); + + await test("utcDateCreated", "'2020-05-14 10:11:42.001Z'", 1); + await test("utcDateCreated", "wrong", 0); + + await test("utcDateModified", "'2020-05-14 11:11:42.001Z'", 1); + await test("utcDateModified", "wrong", 0); + + await test("contentLength", "1001", 1); + await test("contentLength", "10010", 0); + + await test("parentCount", "2", 1); + await test("parentCount", "3", 0); + + await test("childrenCount", "2", 1); + await test("childrenCount", "10", 0); + + await test("attributeCount", "3", 1); + await test("attributeCount", "4", 0); + + await test("labelCount", "2", 1); + await test("labelCount", "3", 0); + + await test("relationCount", "1", 1); + await test("relationCount", "2", 0); + }) }); /** @return {Note} */ diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index 7556ce250..b5a47c576 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -10,6 +10,20 @@ class Note { this.noteId = row.noteId; /** @param {string} */ this.title = row.title; + /** @param {string} */ + this.type = row.type; + /** @param {string} */ + this.mime = row.mime; + /** @param {number} */ + this.contentLength = row.contentLength; + /** @param {string} */ + this.dateCreated = row.dateCreated; + /** @param {string} */ + this.dateModified = row.dateModified; + /** @param {string} */ + this.utcDateCreated = row.utcDateCreated; + /** @param {string} */ + this.utcDateModified = row.utcDateModified; /** @param {boolean} */ this.isProtected = !!row.isProtected; /** @param {boolean} */ @@ -224,6 +238,26 @@ class Note { return arr.flat(); } + get parentCount() { + return this.parents.length; + } + + get childrenCount() { + return this.children.length; + } + + get labelCount() { + return this.attributes.filter(attr => attr.type === 'label').length; + } + + get relationCount() { + return this.attributes.filter(attr => attr.type === 'relation').length; + } + + get attributeCount() { + return this.attributes.length; + } + /** @return {Note[]} - returns only notes which are templated, does not include their subtrees * in effect returns notes which are influenced by note's non-inheritable attributes */ get templatedNotes() { diff --git a/src/services/note_cache/note_cache_loader.js b/src/services/note_cache/note_cache_loader.js index b3f4893b9..e2104d984 100644 --- a/src/services/note_cache/note_cache_loader.js +++ b/src/services/note_cache/note_cache_loader.js @@ -13,7 +13,7 @@ async function load() { noteCache.reset(); - (await sql.getRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`, [])) + (await sql.getRows(`SELECT noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified, contentLength FROM notes WHERE isDeleted = 0`, [])) .map(row => new Note(noteCache, row)); (await sql.getRows(`SELECT branchId, noteId, parentNoteId, prefix FROM branches WHERE isDeleted = 0`, [])) diff --git a/src/services/search/expressions/property_comparison.js b/src/services/search/expressions/property_comparison.js index 730bf5597..3a52f02df 100644 --- a/src/services/search/expressions/property_comparison.js +++ b/src/services/search/expressions/property_comparison.js @@ -3,11 +3,37 @@ const Expression = require('./expression'); const NoteSet = require('../note_set'); +/** + * Search string is lower cased for case insensitive comparison. But when retrieving properties + * we need case sensitive form so we have this translation object. + */ +const PROP_MAPPING = { + "noteid": "noteId", + "title": "title", + "type": "type", + "mime": "mime", + "isprotected": "isProtected", + "datecreated": "dateCreated", + "datemodified": "dateModified", + "utcdatecreated": "utcDateCreated", + "utcdatemodified": "utcDateModified", + "contentlength": "contentLength", + "parentcount": "parentCount", + "childrencount": "childrenCount", + "attributecount": "attributeCount", + "labelcount": "labelCount", + "relationcount": "relationCount" +}; + class PropertyComparisonExp extends Expression { + static isProperty(name) { + return name in PROP_MAPPING; + } + constructor(propertyName, comparator) { super(); - this.propertyName = propertyName; + this.propertyName = PROP_MAPPING[propertyName]; this.comparator = comparator; } @@ -15,8 +41,15 @@ class PropertyComparisonExp extends Expression { const resNoteSet = new NoteSet(); for (const note of inputNoteSet.notes) { - const value = note[this.propertyName].toLowerCase(); + let value = note[this.propertyName]; + if (value !== undefined && value !== null && typeof value !== 'string') { + value = value.toString(); + } + + if (value) { + value = value.toLowerCase(); + } if (this.comparator(value)) { resNoteSet.add(note); } diff --git a/src/services/search/parser.js b/src/services/search/parser.js index db6c3b21c..552707c82 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -86,7 +86,7 @@ function getExpression(tokens, parsingContext) { return parseRelation(tokens[i]); } - if (tokens[i] === 'title') { + if (PropertyComparisonExp.isProperty(tokens[i])) { const propertyName = tokens[i]; const operator = tokens[i + 1]; const comparedValue = tokens[i + 2]; @@ -101,6 +101,8 @@ function getExpression(tokens, parsingContext) { return new PropertyComparisonExp(propertyName, comparator); } + + parsingContext.addError(`Unrecognized note property "${tokens[i]}"`); } function parseLabel(labelName) { From 8ce2afff8a3e840091b309b734b8447530eb5516 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 22:18:06 +0200 Subject: [PATCH 39/47] more tests --- spec/lexer.spec.js | 4 ++-- spec/search.spec.js | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js index 1482f27ee..4c65cecf3 100644 --- a/spec/lexer.spec.js +++ b/spec/lexer.spec.js @@ -60,7 +60,7 @@ describe("Lexer expression", () => { }); it("dot separated properties", () => { - expect(lexer(`# ~author.title = 'Hugh Howey' AND note.title = 'Silo'`).expressionTokens) - .toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "title", "=", "silo"]); + expect(lexer(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens) + .toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "book title", "=", "silo"]); }); }); diff --git a/spec/search.spec.js b/spec/search.spec.js index 46d368559..b194c53d7 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -106,6 +106,8 @@ describe("Search", () => { }); it("numeric label comparison fallback to string comparison", async () => { + // dates should not be coerced into numbers which would then give wrong numbers + rootNote .child(note("Europe") .label('country', '', true) @@ -114,14 +116,19 @@ describe("Search", () => { .child(note("Czech Republic") .label('established', '1993-01-01')) .child(note("Hungary") - .label('established', '..wrong..')) + .label('established', '1920-06-04')) ); const parsingContext = new ParsingContext(); - const searchResults = await searchService.findNotesWithQuery('#established < 1990', parsingContext); + let searchResults = await searchService.findNotesWithQuery('#established <= 1955-01-01', parsingContext); expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Hungary")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('#established > 1955-01-01', parsingContext); + expect(searchResults.length).toEqual(2); expect(findNoteByTitle(searchResults, "Austria")).toBeTruthy(); + expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); }); it("logical or", async () => { @@ -389,7 +396,7 @@ describe("Search", () => { await test("relationCount", "1", 1); await test("relationCount", "2", 0); - }) + }); }); /** @return {Note} */ From 9ede77aead40db516480ae2c0145521a1178ca27 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 23:44:55 +0200 Subject: [PATCH 40/47] added ancestor --- spec/search.spec.js | 68 ++++++++++++++++++- src/services/note_cache/entities/note.js | 27 ++++++++ .../search/expressions/descendant_of.js | 30 ++++++++ src/services/search/note_set.js | 1 + src/services/search/parser.js | 7 ++ 5 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 src/services/search/expressions/descendant_of.js diff --git a/spec/search.spec.js b/spec/search.spec.js index b194c53d7..25a812216 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -3,6 +3,7 @@ const Note = require('../src/services/note_cache/entities/note'); const Branch = require('../src/services/note_cache/entities/branch'); const Attribute = require('../src/services/note_cache/entities/attribute'); const ParsingContext = require('../src/services/search/parsing_context'); +const dateUtils = require('../src/services/date_utils'); const noteCache = require('../src/services/note_cache/note_cache'); const randtoken = require('rand-token').generator({source: 'crypto'}); @@ -131,6 +132,48 @@ describe("Search", () => { expect(findNoteByTitle(searchResults, "Czech Republic")).toBeTruthy(); }); + it("smart date comparisons", async () => { + // dates should not be coerced into numbers which would then give wrong numbers + + rootNote + .child(note("My note") + .label('year', new Date().getFullYear().toString()) + .label('month', dateUtils.localNowDate().substr(0, 7)) + .label('date', dateUtils.localNowDate()) + .label('dateTime', dateUtils.localNowDateTime()) + ); + + const parsingContext = new ParsingContext(); + + async function test(query, expectedResultCount) { + const searchResults = await searchService.findNotesWithQuery(query, parsingContext); + expect(searchResults.length).toEqual(expectedResultCount); + + if (expectedResultCount === 1) { + expect(findNoteByTitle(searchResults, "My note")).toBeTruthy(); + } + } + + await test("#year = YEAR", 1); + await test("#year >= YEAR", 1); + await test("#year <= YEAR", 1); + await test("#year < YEAR+1", 1); + await test("#year > YEAR+1", 0); + + await test("#month = MONTH", 1); + + await test("#date = TODAY", 1); + await test("#date > TODAY", 0); + await test("#date > TODAY-1", 1); + await test("#date < TODAY+1", 1); + await test("#date < 'TODAY + 1'", 1); + + await test("#dateTime <= NOW+10", 1); + await test("#dateTime < NOW-10", 0); + await test("#dateTime >= NOW-10", 1); + await test("#dateTime < NOW-10", 0); + }); + it("logical or", async () => { rootNote .child(note("Europe") @@ -217,6 +260,29 @@ describe("Search", () => { expect(findNoteByTitle(searchResults, "Prague")).toBeTruthy(); }); + it("filter by note's ancestor", async () => { + rootNote + .child(note("Europe") + .child(note("Austria")) + .child(note("Czech Republic") + .child(note("Prague").label('city'))) + ) + .child(note("Asia") + .child(note('Taiwan') + .child(note('Taipei').label('city'))) + ); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('#city AND note.ancestors.title = Europe', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Prague")).toBeTruthy(); + + searchResults = await searchService.findNotesWithQuery('#city AND note.ancestors.title = Asia', parsingContext); + expect(searchResults.length).toEqual(1); + expect(findNoteByTitle(searchResults, "Taipei")).toBeTruthy(); + }); + it("filter by note's child", async () => { rootNote .child(note("Europe") @@ -411,7 +477,7 @@ class NoteBuilder { this.note = note; } - label(name, value, isInheritable = false) { + label(name, value = '', isInheritable = false) { new Attribute(noteCache, { attributeId: id(), noteId: this.note.noteId, diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index b5a47c576..ec869d0d1 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -53,6 +53,9 @@ class Note { if (protectedSessionService.isProtectedSessionAvailable()) { this.decrypt(); } + + /** @param {Note[]|null} */ + this.ancestorCache = null; } /** @return {Attribute[]} */ @@ -164,6 +167,7 @@ class Note { this.attributeCache = null; this.inheritableAttributeCache = null; + this.ancestorCache = null; } invalidateSubtreeCaches() { @@ -258,6 +262,29 @@ class Note { return this.attributes.length; } + get ancestors() { + if (!this.ancestorCache) { + const noteIds = new Set(); + this.ancestorCache = []; + + for (const parent of this.parents) { + if (!noteIds.has(parent.noteId)) { + this.ancestorCache.push(parent); + noteIds.add(parent.noteId); + } + + for (const ancestorNote of parent.ancestors) { + if (!noteIds.has(ancestorNote.noteId)) { + this.ancestorCache.push(ancestorNote); + noteIds.add(ancestorNote.noteId); + } + } + } + } + + return this.ancestorCache; + } + /** @return {Note[]} - returns only notes which are templated, does not include their subtrees * in effect returns notes which are influenced by note's non-inheritable attributes */ get templatedNotes() { diff --git a/src/services/search/expressions/descendant_of.js b/src/services/search/expressions/descendant_of.js new file mode 100644 index 000000000..0d7298077 --- /dev/null +++ b/src/services/search/expressions/descendant_of.js @@ -0,0 +1,30 @@ +"use strict"; + +const Expression = require('./expression'); +const NoteSet = require('../note_set'); + +class DescendantOfExp extends Expression { + constructor(subExpression) { + super(); + + this.subExpression = subExpression; + } + + execute(inputNoteSet, searchContext) { + const resNoteSet = new NoteSet(); + + for (const note of inputNoteSet.notes) { + const subInputNoteSet = new NoteSet(note.ancestors); + + const subResNoteSet = this.subExpression.execute(subInputNoteSet, searchContext); + + if (subResNoteSet.notes.length > 0) { + resNoteSet.add(note); + } + } + + return resNoteSet; + } +} + +module.exports = DescendantOfExp; diff --git a/src/services/search/note_set.js b/src/services/search/note_set.js index 85c402a29..8488692c2 100644 --- a/src/services/search/note_set.js +++ b/src/services/search/note_set.js @@ -2,6 +2,7 @@ class NoteSet { constructor(notes = []) { + /** @type {Note[]} */ this.notes = notes; } diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 552707c82..adad9cece 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -4,6 +4,7 @@ const AndExp = require('./expressions/and'); const OrExp = require('./expressions/or'); const NotExp = require('./expressions/not'); const ChildOfExp = require('./expressions/child_of'); +const DescendantOfExp = require('./expressions/descendant_of'); const ParentOfExp = require('./expressions/parent_of'); const RelationWhereExp = require('./expressions/relation_where'); const PropertyComparisonExp = require('./expressions/property_comparison'); @@ -64,6 +65,12 @@ function getExpression(tokens, parsingContext) { return new ParentOfExp(parseNoteProperty()); } + if (tokens[i] === 'ancestors') { + i += 1; + + return new DescendantOfExp(parseNoteProperty()); + } + if (tokens[i] === 'labels') { if (tokens[i + 1] !== '.') { parsingContext.addError(`Expected "." to separate field path, god "${tokens[i + 1]}"`); From e2490f99754d9d6aec3fedadd0e23886467f0348 Mon Sep 17 00:00:00 2001 From: zadam Date: Sat, 23 May 2020 23:57:59 +0200 Subject: [PATCH 41/47] faster implementation of ancestors --- src/services/search/expressions/descendant_of.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/services/search/expressions/descendant_of.js b/src/services/search/expressions/descendant_of.js index 0d7298077..51e01748a 100644 --- a/src/services/search/expressions/descendant_of.js +++ b/src/services/search/expressions/descendant_of.js @@ -2,6 +2,7 @@ const Expression = require('./expression'); const NoteSet = require('../note_set'); +const noteCache = require('../../note_cache/note_cache'); class DescendantOfExp extends Expression { constructor(subExpression) { @@ -11,19 +12,16 @@ class DescendantOfExp extends Expression { } execute(inputNoteSet, searchContext) { - const resNoteSet = new NoteSet(); + const subInputNoteSet = new NoteSet(Object.values(noteCache.notes)); + const subResNoteSet = this.subExpression.execute(subInputNoteSet, searchContext); - for (const note of inputNoteSet.notes) { - const subInputNoteSet = new NoteSet(note.ancestors); + const subTreeNoteSet = new NoteSet(); - const subResNoteSet = this.subExpression.execute(subInputNoteSet, searchContext); - - if (subResNoteSet.notes.length > 0) { - resNoteSet.add(note); - } + for (const note of subResNoteSet.notes) { + subTreeNoteSet.addAll(note.subtreeNotes); } - return resNoteSet; + return inputNoteSet.intersection(subTreeNoteSet); } } From cb4d0624b53ed163c29c3857ca1836e29ab8a59d Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 24 May 2020 00:21:20 +0200 Subject: [PATCH 42/47] put session directory into data dir to avoid conflict of multiple instances on a single server, fixes #1033 --- src/app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app.js b/src/app.js index 1c6a31d3e..5fa590378 100644 --- a/src/app.js +++ b/src/app.js @@ -10,6 +10,7 @@ const FileStore = require('session-file-store')(session); const os = require('os'); const sessionSecret = require('./services/session_secret'); const cls = require('./services/cls'); +const dataDir = require('./services/data_dir'); require('./entities/entity_constructor'); require('./services/handlers'); require('./services/hoisted_note_loader'); @@ -58,7 +59,7 @@ const sessionParser = session({ }, store: new FileStore({ ttl: 30 * 24 * 3600, - path: os.tmpdir() + '/trilium-sessions' + path: dataDir.TRILIUM_DATA_DIR + '/sessions' }) }); app.use(sessionParser); From b5627b138a5fc4bf4ed20496812e89eb85335740 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 24 May 2020 09:30:08 +0200 Subject: [PATCH 43/47] integrate new search into the main global search --- package-lock.json | 44 +++++++------------ package.json | 4 +- src/public/app/widgets/search_results.js | 6 +-- src/routes/api/search.js | 6 +-- .../search/expressions/property_comparison.js | 1 + src/services/search/search.js | 20 ++++++++- 6 files changed, 45 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4e7f1cb65..dd61de609 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3385,9 +3385,9 @@ } }, "electron-debug": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/electron-debug/-/electron-debug-3.0.1.tgz", - "integrity": "sha512-fo3mtDM4Bxxm3DW1I+XcJKfQlUlns4QGWyWGs8OrXK1bBZ2X9HeqYMntYBx78MYRcGY5S/ualuG4GhCnPlaZEA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/electron-debug/-/electron-debug-3.1.0.tgz", + "integrity": "sha512-SWEqLj4MgfV3tGuO5eBLQ5/Nr6M+KPxsnE0bUJZvQebGJus6RAcdmvd7L+l0Ji31h2mmrN23l2tHFtCa2FvurA==", "requires": { "electron-is-dev": "^1.1.0", "electron-localshortcut": "^3.1.0" @@ -3568,29 +3568,19 @@ "integrity": "sha1-UJ5RDCala1Xhf4Y6SwThEYRqsns=" }, "electron-is-dev": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/electron-is-dev/-/electron-is-dev-1.1.0.tgz", - "integrity": "sha512-Z1qA/1oHNowGtSBIcWk0pcLEqYT/j+13xUw/MYOrBUOL4X7VN0i0KCTf5SqyvMPmW5pSPKbo28wkxMxzZ20YnQ==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/electron-is-dev/-/electron-is-dev-1.2.0.tgz", + "integrity": "sha512-R1oD5gMBPS7PVU8gJwH6CtT0e6VSoD0+SzSnYpNm+dBkcijgA+K7VAMHDfnRq/lkKPZArpzplTW6jfiMYosdzw==" }, "electron-localshortcut": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/electron-localshortcut/-/electron-localshortcut-3.1.0.tgz", - "integrity": "sha512-MgL/j5jdjW7iA0R6cI7S045B0GlKXWM1FjjujVPjlrmyXRa6yH0bGSaIAfxXAF9tpJm3pLEiQzerYHkRh9JG/A==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/electron-localshortcut/-/electron-localshortcut-3.2.1.tgz", + "integrity": "sha512-DWvhKv36GsdXKnaFFhEiK8kZZA+24/yFLgtTwJJHc7AFgDjNRIBJZ/jq62Y/dWv9E4ypYwrVWN2bVrCYw1uv7Q==", "requires": { - "debug": "^2.6.8", + "debug": "^4.0.1", "electron-is-accelerator": "^0.1.0", - "keyboardevent-from-electron-accelerator": "^1.1.0", + "keyboardevent-from-electron-accelerator": "^2.0.0", "keyboardevents-areequal": "^0.2.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - } } }, "electron-notarize": { @@ -6507,9 +6497,9 @@ "integrity": "sha512-pBxcB3LFc8QVgdggvZWyeys+hnrNWg4OcZIU/1X59k5jQdLBlCsYGRQaz234SqoRLTCgMH00fY0xRJH+F9METQ==" }, "keyboardevent-from-electron-accelerator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/keyboardevent-from-electron-accelerator/-/keyboardevent-from-electron-accelerator-1.1.0.tgz", - "integrity": "sha512-VDC4vKWGrR3VgIKCE4CsXnvObGgP8C2idnTKEMUkuEuvDGE1GEBX9FtNdJzrD00iQlhI3xFxRaeItsUmlERVng==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/keyboardevent-from-electron-accelerator/-/keyboardevent-from-electron-accelerator-2.0.0.tgz", + "integrity": "sha512-iQcmNA0M4ETMNi0kG/q0h/43wZk7rMeKYrXP7sqKIJbHkTU8Koowgzv+ieR/vWJbOwxx5nDC3UnudZ0aLSu4VA==" }, "keyboardevents-areequal": { "version": "0.2.2", @@ -10338,9 +10328,9 @@ "dev": true }, "sqlite": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.7.tgz", - "integrity": "sha512-1bBO+me3gXRfqwRR3K9aNDoSbTkQ87o6fSjj/BE2gSHHsK3qIDR+LoFZHgZ6kSPdFBoLTsy5/w/+8PBBaK+lvg==" + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.8.tgz", + "integrity": "sha512-MOy63kITfjJnZimrwgQ50+L83J3IBPjuyTZ98YooAmSXdLtfGHDTMgH5csWturZ/mzm4TafLvtjkIbhmQVNgcw==" }, "sqlite3": { "version": "4.1.1", diff --git a/package.json b/package.json index 892152425..3ba80e1be 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "dayjs": "1.8.27", "debug": "4.1.1", "ejs": "3.1.3", - "electron-debug": "3.0.1", + "electron-debug": "3.1.0", "electron-dl": "3.0.0", "electron-find": "1.0.6", "electron-window-state": "5.0.3", @@ -67,7 +67,7 @@ "serve-favicon": "2.5.0", "session-file-store": "1.4.0", "simple-node-logger": "18.12.24", - "sqlite": "4.0.7", + "sqlite": "4.0.8", "sqlite3": "4.1.1", "string-similarity": "4.0.1", "tar-stream": "2.1.2", diff --git a/src/public/app/widgets/search_results.js b/src/public/app/widgets/search_results.js index 88aa434a9..0a5f1031a 100644 --- a/src/public/app/widgets/search_results.js +++ b/src/public/app/widgets/search_results.js @@ -48,8 +48,8 @@ export default class SearchResultsWidget extends BasicWidget { for (const result of results) { const link = $('', { href: 'javascript:', - text: result.title - }).attr('data-action', 'note').attr('data-note-path', result.path); + text: result.notePathTitle + }).attr('data-action', 'note').attr('data-note-path', result.notePath); const $result = $('
  • ').append(link); @@ -60,4 +60,4 @@ export default class SearchResultsWidget extends BasicWidget { searchFlowEndedEvent() { this.$searchResults.hide(); } -} \ No newline at end of file +} diff --git a/src/routes/api/search.js b/src/routes/api/search.js index a5596830d..1bbbfbc40 100644 --- a/src/routes/api/search.js +++ b/src/routes/api/search.js @@ -4,15 +4,15 @@ const repository = require('../../services/repository'); const noteCacheService = require('../../services/note_cache/note_cache.js'); const log = require('../../services/log'); const scriptService = require('../../services/script'); -const searchService = require('../../services/search'); +const searchService = require('../../services/search/search'); async function searchNotes(req) { - const noteIds = await searchService.searchForNoteIds(req.params.searchString); + const notePaths = await searchService.searchNotes(req.params.searchString); try { return { success: true, - results: noteIds.map(noteCacheService.getNotePath).filter(res => !!res) + results: notePaths } } catch { diff --git a/src/services/search/expressions/property_comparison.js b/src/services/search/expressions/property_comparison.js index 3a52f02df..7bd681bd8 100644 --- a/src/services/search/expressions/property_comparison.js +++ b/src/services/search/expressions/property_comparison.js @@ -13,6 +13,7 @@ const PROP_MAPPING = { "type": "type", "mime": "mime", "isprotected": "isProtected", + "isarhived": "isArchived", "datecreated": "dateCreated", "datemodified": "dateModified", "utcdatecreated": "utcDateCreated", diff --git a/src/services/search/search.js b/src/services/search/search.js index 5af43925b..79517ae49 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -75,6 +75,23 @@ async function findNotesWithQuery(query, parsingContext) { return await findNotesWithExpression(expression); } +async function searchNotes(query) { + if (!query.trim().length) { + return []; + } + + const parsingContext = new ParsingContext({ + includeNoteContent: true, + fuzzyAttributeSearch: false + }); + + let searchResults = await findNotesWithQuery(query, parsingContext); + + searchResults = searchResults.slice(0, 200); + + return searchResults; +} + async function searchNotesForAutocomplete(query) { if (!query.trim().length) { return []; @@ -85,7 +102,7 @@ async function searchNotesForAutocomplete(query) { fuzzyAttributeSearch: true }); - let searchResults = findNotesWithQuery(query, parsingContext); + let searchResults = await findNotesWithQuery(query, parsingContext); searchResults = searchResults.slice(0, 200); @@ -154,6 +171,7 @@ function formatAttribute(attr) { } module.exports = { + searchNotes, searchNotesForAutocomplete, findNotesWithQuery }; From a1a744bb00b7efe14d3432863cec3dd348fffce4 Mon Sep 17 00:00:00 2001 From: zadam Date: Mon, 25 May 2020 00:25:47 +0200 Subject: [PATCH 44/47] order & limit implementation WIP --- spec/note_cache_mocking.js | 70 +++++++++++ spec/search.spec.js | 93 ++++++--------- spec/value_extractor.spec.js | 86 ++++++++++++++ src/services/note_cache/entities/note.js | 12 ++ src/services/search.js | 15 +++ .../search/expressions/order_by_and_limit.js | 58 +++++++++ src/services/search/note_set.js | 2 + src/services/search/parser.js | 82 +++++++++++-- src/services/search/parsing_context.js | 1 + src/services/search/search.js | 18 +-- src/services/search/value_extractor.js | 110 ++++++++++++++++++ 11 files changed, 470 insertions(+), 77 deletions(-) create mode 100644 spec/note_cache_mocking.js create mode 100644 spec/value_extractor.spec.js create mode 100644 src/services/search/expressions/order_by_and_limit.js create mode 100644 src/services/search/value_extractor.js diff --git a/spec/note_cache_mocking.js b/spec/note_cache_mocking.js new file mode 100644 index 000000000..e96252dfd --- /dev/null +++ b/spec/note_cache_mocking.js @@ -0,0 +1,70 @@ +const Note = require('../src/services/note_cache/entities/note'); +const Branch = require('../src/services/note_cache/entities/branch'); +const Attribute = require('../src/services/note_cache/entities/attribute'); +const noteCache = require('../src/services/note_cache/note_cache'); +const randtoken = require('rand-token').generator({source: 'crypto'}); + +/** @return {Note} */ +function findNoteByTitle(searchResults, title) { + return searchResults + .map(sr => noteCache.notes[sr.noteId]) + .find(note => note.title === title); +} + +class NoteBuilder { + constructor(note) { + this.note = note; + } + + label(name, value = '', isInheritable = false) { + new Attribute(noteCache, { + attributeId: id(), + noteId: this.note.noteId, + type: 'label', + isInheritable, + name, + value + }); + + return this; + } + + relation(name, targetNote) { + new Attribute(noteCache, { + attributeId: id(), + noteId: this.note.noteId, + type: 'relation', + name, + value: targetNote.noteId + }); + + return this; + } + + child(childNoteBuilder, prefix = "") { + new Branch(noteCache, { + branchId: id(), + noteId: childNoteBuilder.note.noteId, + parentNoteId: this.note.noteId, + prefix + }); + + return this; + } +} + +function id() { + return randtoken.generate(10); +} + +function note(title) { + const note = new Note(noteCache, {noteId: id(), title}); + + return new NoteBuilder(note); +} + +module.exports = { + NoteBuilder, + findNoteByTitle, + note +}; diff --git a/spec/search.spec.js b/spec/search.spec.js index 25a812216..2a20a4ba9 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -5,7 +5,7 @@ const Attribute = require('../src/services/note_cache/entities/attribute'); const ParsingContext = require('../src/services/search/parsing_context'); const dateUtils = require('../src/services/date_utils'); const noteCache = require('../src/services/note_cache/note_cache'); -const randtoken = require('rand-token').generator({source: 'crypto'}); +const {NoteBuilder, findNoteByTitle, note} = require('./note_cache_mocking'); describe("Search", () => { let rootNote; @@ -463,63 +463,36 @@ describe("Search", () => { await test("relationCount", "1", 1); await test("relationCount", "2", 0); }); + + it("test order by", async () => { + const italy = note("Italy").label("capital", "Rome"); + const slovakia = note("Slovakia").label("capital", "Bratislava"); + const austria = note("Austria").label("capital", "Vienna"); + const ukraine = note("Ukraine").label("capital", "Kiev"); + + rootNote + .child(note("Europe") + .child(ukraine) + .child(slovakia) + .child(austria) + .child(italy)); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe orderBy note.title', parsingContext); + expect(searchResults.length).toEqual(4); + expect(noteCache.notes[searchResults[0].noteId].title).toEqual("Austria"); + expect(noteCache.notes[searchResults[1].noteId].title).toEqual("Italy"); + expect(noteCache.notes[searchResults[2].noteId].title).toEqual("Slovakia"); + expect(noteCache.notes[searchResults[3].noteId].title).toEqual("Ukraine"); + + searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe orderBy note.labels.capital', parsingContext); + expect(searchResults.length).toEqual(4); + expect(noteCache.notes[searchResults[0].noteId].title).toEqual("Slovakia"); + expect(noteCache.notes[searchResults[1].noteId].title).toEqual("Ukraine"); + expect(noteCache.notes[searchResults[2].noteId].title).toEqual("Italy"); + expect(noteCache.notes[searchResults[3].noteId].title).toEqual("Austria"); + }); + + // FIXME: test what happens when we order without any filter criteria }); - -/** @return {Note} */ -function findNoteByTitle(searchResults, title) { - return searchResults - .map(sr => noteCache.notes[sr.noteId]) - .find(note => note.title === title); -} - -class NoteBuilder { - constructor(note) { - this.note = note; - } - - label(name, value = '', isInheritable = false) { - new Attribute(noteCache, { - attributeId: id(), - noteId: this.note.noteId, - type: 'label', - isInheritable, - name, - value - }); - - return this; - } - - relation(name, targetNote) { - new Attribute(noteCache, { - attributeId: id(), - noteId: this.note.noteId, - type: 'relation', - name, - value: targetNote.noteId - }); - - return this; - } - - child(childNoteBuilder, prefix = "") { - new Branch(noteCache, { - branchId: id(), - noteId: childNoteBuilder.note.noteId, - parentNoteId: this.note.noteId, - prefix - }); - - return this; - } -} - -function id() { - return randtoken.generate(10); -} - -function note(title) { - const note = new Note(noteCache, {noteId: id(), title}); - - return new NoteBuilder(note); -} diff --git a/spec/value_extractor.spec.js b/spec/value_extractor.spec.js new file mode 100644 index 000000000..40a9e193e --- /dev/null +++ b/spec/value_extractor.spec.js @@ -0,0 +1,86 @@ +const {NoteBuilder, findNoteByTitle, note} = require('./note_cache_mocking'); +const ValueExtractor = require('../src/services/search/value_extractor'); +const noteCache = require('../src/services/note_cache/note_cache'); + +describe("Value extractor", () => { + beforeEach(() => { + noteCache.reset(); + }); + + it("simple title extraction", async () => { + const europe = note("Europe").note; + + const valueExtractor = new ValueExtractor(["note", "title"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(europe)).toEqual("Europe"); + }); + + it("label extraction", async () => { + const austria = note("Austria") + .label("Capital", "Vienna") + .note; + + let valueExtractor = new ValueExtractor(["note", "labels", "capital"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(austria)).toEqual("vienna"); + + valueExtractor = new ValueExtractor(["#capital"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(austria)).toEqual("vienna"); + }); + + it("parent/child property extraction", async () => { + const vienna = note("Vienna"); + const europe = note("Europe") + .child(note("Austria") + .child(vienna)); + + let valueExtractor = new ValueExtractor(["note", "children", "children", "title"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(europe.note)).toEqual("Vienna"); + + valueExtractor = new ValueExtractor(["note", "parents", "parents", "title"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(vienna.note)).toEqual("Europe"); + }); + + it("extract through relation", async () => { + const czechRepublic = note("Czech Republic").label("capital", "Prague"); + const slovakia = note("Slovakia").label("capital", "Bratislava"); + const austria = note("Austria") + .relation('neighbor', czechRepublic.note) + .relation('neighbor', slovakia.note); + + let valueExtractor = new ValueExtractor(["note", "relations", "neighbor", "labels", "capital"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(austria.note)).toEqual("prague"); + + valueExtractor = new ValueExtractor(["~neighbor", "labels", "capital"]); + + expect(valueExtractor.validate()).toBeFalsy(); + expect(valueExtractor.extract(austria.note)).toEqual("prague"); + }); +}); + +describe("Invalid value extractor property path", () => { + it('each path must start with "note" (or label/relation)', + () => expect(new ValueExtractor(["neighbor"]).validate()).toBeTruthy()); + + it("extra path element after terminal label", + () => expect(new ValueExtractor(["~neighbor", "labels", "capital", "noteId"]).validate()).toBeTruthy()); + + it("extra path element after terminal title", + () => expect(new ValueExtractor(["note", "title", "isProtected"]).validate()).toBeTruthy()); + + it("relation name and note property is missing", + () => expect(new ValueExtractor(["note", "relations"]).validate()).toBeTruthy()); + + it("relation is specified but target note property is not specified", + () => expect(new ValueExtractor(["note", "relations", "myrel"]).validate()).toBeTruthy()); +}); diff --git a/src/services/note_cache/entities/note.js b/src/services/note_cache/entities/note.js index ec869d0d1..409589263 100644 --- a/src/services/note_cache/entities/note.js +++ b/src/services/note_cache/entities/note.js @@ -107,6 +107,18 @@ class Note { return this.attributes.find(attr => attr.type === type && attr.name === name); } + getLabelValue(name) { + const label = this.attributes.find(attr => attr.type === 'label' && attr.name === name); + + return label ? label.value : null; + } + + getRelationTarget(name) { + const relation = this.attributes.find(attr => attr.type === 'relation' && attr.name === name); + + return relation ? relation.targetNote : null; + } + get isArchived() { return this.hasAttribute('label', 'archived'); } diff --git a/src/services/search.js b/src/services/search.js index 774614ca8..2efa62984 100644 --- a/src/services/search.js +++ b/src/services/search.js @@ -1,3 +1,18 @@ +"use strict"; + +/** + * Missing things from the OLD search: + * - orderBy + * - limit + * - in - replaced with note.ancestors + * - content in attribute search + * - not - pherhaps not necessary + * + * other potential additions: + * - targetRelations - either named or not + * - any relation without name + */ + const repository = require('./repository'); const sql = require('./sql'); const log = require('./log'); diff --git a/src/services/search/expressions/order_by_and_limit.js b/src/services/search/expressions/order_by_and_limit.js new file mode 100644 index 000000000..6d1ad8ebc --- /dev/null +++ b/src/services/search/expressions/order_by_and_limit.js @@ -0,0 +1,58 @@ +"use strict"; + +const Expression = require('./expression'); +const NoteSet = require('../note_set'); + +class OrderByAndLimitExp extends Expression { + constructor(orderDefinitions, limit) { + super(); + + this.orderDefinitions = orderDefinitions; + + for (const od of this.orderDefinitions) { + od.smaller = od.direction === "asc" ? -1 : 1; + od.larger = od.direction === "asc" ? 1 : -1; + } + + this.limit = limit; + + /** @type {Expression} */ + this.subExpression = null; // it's expected to be set after construction + } + + execute(inputNoteSet, searchContext) { + let {notes} = this.subExpression.execute(inputNoteSet, searchContext); + + notes.sort((a, b) => { + for (const {valueExtractor, smaller, larger} of this.orderDefinitions) { + let valA = valueExtractor.extract(a); + let valB = valueExtractor.extract(b); + + if (!isNaN(valA) && !isNaN(valB)) { + valA = parseFloat(valA); + valB = parseFloat(valB); + } + + if (valA < valB) { + return smaller; + } else if (valA > valB) { + return larger; + } + // else go to next order definition + } + + return 0; + }); + + if (this.limit) { + notes = notes.slice(0, this.limit); + } + + const noteSet = new NoteSet(notes); + noteSet.sorted = true; + + return noteSet; + } +} + +module.exports = OrderByAndLimitExp; diff --git a/src/services/search/note_set.js b/src/services/search/note_set.js index 8488692c2..3f8dc9dcf 100644 --- a/src/services/search/note_set.js +++ b/src/services/search/note_set.js @@ -4,6 +4,8 @@ class NoteSet { constructor(notes = []) { /** @type {Note[]} */ this.notes = notes; + /** @type {boolean} */ + this.sorted = false; } add(note) { diff --git a/src/services/search/parser.js b/src/services/search/parser.js index adad9cece..73dd5ea55 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -12,7 +12,9 @@ const AttributeExistsExp = require('./expressions/attribute_exists'); const LabelComparisonExp = require('./expressions/label_comparison'); const NoteCacheFulltextExp = require('./expressions/note_cache_fulltext'); const NoteContentFulltextExp = require('./expressions/note_content_fulltext'); +const OrderByAndLimitExp = require('./expressions/order_by_and_limit'); const comparatorBuilder = require('./comparator_builder'); +const ValueExtractor = require('./value_extractor'); function getFulltext(tokens, parsingContext) { parsingContext.highlightedTokens.push(...tokens); @@ -35,7 +37,7 @@ function isOperator(str) { return str.match(/^[=<>*]+$/); } -function getExpression(tokens, parsingContext) { +function getExpression(tokens, parsingContext, level = 0) { if (tokens.length === 0) { return null; } @@ -104,7 +106,7 @@ function getExpression(tokens, parsingContext) { return; } - i += 3; + i += 2; return new PropertyComparisonExp(propertyName, comparator); } @@ -151,6 +153,57 @@ function getExpression(tokens, parsingContext) { } } + function parseOrderByAndLimit() { + const orderDefinitions = []; + let limit; + + if (tokens[i] === 'orderby') { + do { + const propertyPath = []; + let direction = "asc"; + + do { + i++; + + propertyPath.push(tokens[i]); + + i++; + } while (tokens[i] === '.'); + + if (["asc", "desc"].includes(tokens[i + 1])) { + direction = tokens[i + 1]; + i++; + } + + const valueExtractor = new ValueExtractor(propertyPath); + + if (valueExtractor.validate()) { + parsingContext.addError(valueExtractor.validate()); + } + + orderDefinitions.push({ + valueExtractor, + direction + }); + } while (tokens[i] === ','); + } + + if (tokens[i] === 'limit') { + limit = parseInt(tokens[i + 1]); + } + + return new OrderByAndLimitExp(orderDefinitions, limit); + } + + function getAggregateExpression() { + if (op === null || op === 'and') { + return AndExp.of(expressions); + } + else if (op === 'or') { + return OrExp.of(expressions); + } + } + for (i = 0; i < tokens.length; i++) { const token = tokens[i]; @@ -159,7 +212,7 @@ function getExpression(tokens, parsingContext) { } if (Array.isArray(token)) { - expressions.push(getExpression(token, parsingContext)); + expressions.push(getExpression(token, parsingContext, level++)); } else if (token.startsWith('#')) { const labelName = token.substr(1); @@ -171,6 +224,22 @@ function getExpression(tokens, parsingContext) { expressions.push(parseRelation(relationName)); } + else if (['orderby', 'limit'].includes(token)) { + if (level !== 0) { + parsingContext.addError('orderBy can appear only on the top expression level'); + continue; + } + + const exp = parseOrderByAndLimit(); + + if (!exp) { + continue; + } + + exp.subExpression = getAggregateExpression(); + + return exp; + } else if (token === 'note') { i++; @@ -198,12 +267,7 @@ function getExpression(tokens, parsingContext) { } } - if (op === null || op === 'and') { - return AndExp.of(expressions); - } - else if (op === 'or') { - return OrExp.of(expressions); - } + return getAggregateExpression(); } function parse({fulltextTokens, expressionTokens, parsingContext}) { diff --git a/src/services/search/parsing_context.js b/src/services/search/parsing_context.js index 59bc487d8..58ab77d39 100644 --- a/src/services/search/parsing_context.js +++ b/src/services/search/parsing_context.js @@ -12,6 +12,7 @@ class ParsingContext { // we record only the first error, subsequent ones are usually consequence of the first if (!this.error) { this.error = error; + console.log(this.error); } } } diff --git a/src/services/search/search.js b/src/services/search/search.js index 79517ae49..1e7437b58 100644 --- a/src/services/search/search.js +++ b/src/services/search/search.js @@ -34,15 +34,17 @@ async function findNotesWithExpression(expression) { .filter(notePathArray => notePathArray.includes(hoistedNoteService.getHoistedNoteId())) .map(notePathArray => new SearchResult(notePathArray)); - // sort results by depth of the note. This is based on the assumption that more important results - // are closer to the note root. - searchResults.sort((a, b) => { - if (a.notePathArray.length === b.notePathArray.length) { - return a.notePathTitle < b.notePathTitle ? -1 : 1; - } + if (!noteSet.sorted) { + // sort results by depth of the note. This is based on the assumption that more important results + // are closer to the note root. + searchResults.sort((a, b) => { + if (a.notePathArray.length === b.notePathArray.length) { + return a.notePathTitle < b.notePathTitle ? -1 : 1; + } - return a.notePathArray.length < b.notePathArray.length ? -1 : 1; - }); + return a.notePathArray.length < b.notePathArray.length ? -1 : 1; + }); + } return searchResults; } diff --git a/src/services/search/value_extractor.js b/src/services/search/value_extractor.js new file mode 100644 index 000000000..ea1bb38d8 --- /dev/null +++ b/src/services/search/value_extractor.js @@ -0,0 +1,110 @@ +"use strict"; + +/** + * Search string is lower cased for case insensitive comparison. But when retrieving properties + * we need case sensitive form so we have this translation object. + */ +const PROP_MAPPING = { + "noteid": "noteId", + "title": "title", + "type": "type", + "mime": "mime", + "isprotected": "isProtected", + "isarhived": "isArchived", + "datecreated": "dateCreated", + "datemodified": "dateModified", + "utcdatecreated": "utcDateCreated", + "utcdatemodified": "utcDateModified", + "contentlength": "contentLength", + "parentcount": "parentCount", + "childrencount": "childrenCount", + "attributecount": "attributeCount", + "labelcount": "labelCount", + "relationcount": "relationCount" +}; + +class ValueExtractor { + constructor(propertyPath) { + this.propertyPath = propertyPath.map(pathEl => pathEl.toLowerCase()); + + if (this.propertyPath[0].startsWith('#')) { + this.propertyPath = ['note', 'labels', this.propertyPath[0].substr(1), ...this.propertyPath.slice( 1, this.propertyPath.length)]; + } + else if (this.propertyPath[0].startsWith('~')) { + this.propertyPath = ['note', 'relations', this.propertyPath[0].substr(1), ...this.propertyPath.slice( 1, this.propertyPath.length)]; + } + } + + validate() { + if (this.propertyPath[0] !== 'note') { + return `property specifier must start with 'note', but starts with '${this.propertyPath[0]}'`; + } + + for (let i = 1; i < this.propertyPath.length; i++) { + const pathEl = this.propertyPath[i]; + + if (pathEl === 'labels') { + if (i !== this.propertyPath.length - 2) { + return `label is a terminal property specifier and must be at the end`; + } + + i++; + } + else if (pathEl === 'relations') { + if (i >= this.propertyPath.length - 2) { + return `relation name or property name is missing`; + } + + i++; + } + else if (pathEl in PROP_MAPPING) { + if (i !== this.propertyPath.length - 1) { + return `${pathEl} is a terminal property specifier and must be at the end`; + } + } + else if (!["parents", "children"].includes(pathEl)) { + return `Unrecognized property specifier ${pathEl}`; + } + } + } + + extract(note) { + let cursor = note; + + let i; + + const cur = () => this.propertyPath[i]; + + for (i = 0; i < this.propertyPath.length; i++) { + if (!cursor) { + return cursor; + } + + if (cur() === 'labels') { + i++; + + return cursor.getLabelValue(cur()); + } + + if (cur() === 'relations') { + i++; + + cursor = cursor.getRelationTarget(cur()); + } + else if (cur() === 'parents') { + cursor = cursor.parents[0]; + } + else if (cur() === 'children') { + cursor = cursor.children[0]; + } + else if (cur() in PROP_MAPPING) { + return cursor[PROP_MAPPING[cur()]]; + } + else { + // FIXME + } + } + } +} + +module.exports = ValueExtractor; From c753f228ac89cb3d6e374acdf0ddd1002f92a356 Mon Sep 17 00:00:00 2001 From: zadam Date: Mon, 25 May 2020 23:39:34 +0200 Subject: [PATCH 45/47] more tests & fixes --- spec/search.spec.js | 18 ++++++++++++++++++ .../search/expressions/order_by_and_limit.js | 2 +- src/services/search/parser.js | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/spec/search.spec.js b/spec/search.spec.js index 2a20a4ba9..f193857bc 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -492,6 +492,24 @@ describe("Search", () => { expect(noteCache.notes[searchResults[1].noteId].title).toEqual("Ukraine"); expect(noteCache.notes[searchResults[2].noteId].title).toEqual("Italy"); expect(noteCache.notes[searchResults[3].noteId].title).toEqual("Austria"); + + searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe orderBy note.labels.capital DESC', parsingContext); + expect(searchResults.length).toEqual(4); + expect(noteCache.notes[searchResults[0].noteId].title).toEqual("Austria"); + expect(noteCache.notes[searchResults[1].noteId].title).toEqual("Italy"); + expect(noteCache.notes[searchResults[2].noteId].title).toEqual("Ukraine"); + expect(noteCache.notes[searchResults[3].noteId].title).toEqual("Slovakia"); + + searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe orderBy note.labels.capital DESC limit 2', parsingContext); + expect(searchResults.length).toEqual(2); + expect(noteCache.notes[searchResults[0].noteId].title).toEqual("Austria"); + expect(noteCache.notes[searchResults[1].noteId].title).toEqual("Italy"); + + searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe orderBy #capital DESC limit 0', parsingContext); + expect(searchResults.length).toEqual(0); + + searchResults = await searchService.findNotesWithQuery('# note.parents.title = Europe orderBy #capital DESC limit 1000', parsingContext); + expect(searchResults.length).toEqual(4); }); // FIXME: test what happens when we order without any filter criteria diff --git a/src/services/search/expressions/order_by_and_limit.js b/src/services/search/expressions/order_by_and_limit.js index 6d1ad8ebc..3a91484c5 100644 --- a/src/services/search/expressions/order_by_and_limit.js +++ b/src/services/search/expressions/order_by_and_limit.js @@ -44,7 +44,7 @@ class OrderByAndLimitExp extends Expression { return 0; }); - if (this.limit) { + if (this.limit >= 0) { notes = notes.slice(0, this.limit); } diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 73dd5ea55..3ff60278e 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -170,8 +170,8 @@ function getExpression(tokens, parsingContext, level = 0) { i++; } while (tokens[i] === '.'); - if (["asc", "desc"].includes(tokens[i + 1])) { - direction = tokens[i + 1]; + if (["asc", "desc"].includes(tokens[i])) { + direction = tokens[i]; i++; } From dc2d5a0a79eddf2e59191a53693f15532313bd9c Mon Sep 17 00:00:00 2001 From: zadam Date: Tue, 26 May 2020 23:25:13 +0200 Subject: [PATCH 46/47] fix fulltext content search --- package-lock.json | 6 +++--- package.json | 2 +- src/services/search/expressions/and.js | 4 ++-- .../search/expressions/note_content_fulltext.js | 8 ++++++-- src/services/search/expressions/or.js | 4 ++-- src/services/search/parser.js | 17 ++++++++++++++++- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index dd61de609..1f7ab50ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10328,9 +10328,9 @@ "dev": true }, "sqlite": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.8.tgz", - "integrity": "sha512-MOy63kITfjJnZimrwgQ50+L83J3IBPjuyTZ98YooAmSXdLtfGHDTMgH5csWturZ/mzm4TafLvtjkIbhmQVNgcw==" + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-4.0.9.tgz", + "integrity": "sha512-vB6Xzn5S5XxMfmyO0ErKjuP5jEQ0z+oFXFC4zXC0s12NMULLETUTb6+PST8sZ7/2HR4KLk4Jsj5yeXkCvogYxg==" }, "sqlite3": { "version": "4.1.1", diff --git a/package.json b/package.json index 3ba80e1be..67115fee0 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "serve-favicon": "2.5.0", "session-file-store": "1.4.0", "simple-node-logger": "18.12.24", - "sqlite": "4.0.8", + "sqlite": "4.0.9", "sqlite3": "4.1.1", "string-similarity": "4.0.1", "tar-stream": "2.1.2", diff --git a/src/services/search/expressions/and.js b/src/services/search/expressions/and.js index ee22f6b13..9d0237c2e 100644 --- a/src/services/search/expressions/and.js +++ b/src/services/search/expressions/and.js @@ -18,9 +18,9 @@ class AndExp extends Expression { this.subExpressions = subExpressions; } - execute(inputNoteSet, searchContext) { + async execute(inputNoteSet, searchContext) { for (const subExpression of this.subExpressions) { - inputNoteSet = subExpression.execute(inputNoteSet, searchContext); + inputNoteSet = await subExpression.execute(inputNoteSet, searchContext); } return inputNoteSet; diff --git a/src/services/search/expressions/note_content_fulltext.js b/src/services/search/expressions/note_content_fulltext.js index 94932cde7..6c8b9dc34 100644 --- a/src/services/search/expressions/note_content_fulltext.js +++ b/src/services/search/expressions/note_content_fulltext.js @@ -3,17 +3,21 @@ const Expression = require('./expression'); const NoteSet = require('../note_set'); const noteCache = require('../../note_cache/note_cache'); +const utils = require('../../utils'); class NoteContentFulltextExp extends Expression { - constructor(tokens) { + constructor(operator, tokens) { super(); + this.likePrefix = ["*=*", "*="].includes(operator) ? "%" : ""; + this.likeSuffix = ["*=*", "=*"].includes(operator) ? "%" : ""; + this.tokens = tokens; } async execute(inputNoteSet) { const resultNoteSet = new NoteSet(); - const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike('%', token, '%')); + const wheres = this.tokens.map(token => "note_contents.content LIKE " + utils.prepareSqlForLike(this.likePrefix, token, this.likeSuffix)); const sql = require('../../sql'); diff --git a/src/services/search/expressions/or.js b/src/services/search/expressions/or.js index 62c16f5cf..63586e0cc 100644 --- a/src/services/search/expressions/or.js +++ b/src/services/search/expressions/or.js @@ -21,11 +21,11 @@ class OrExp extends Expression { this.subExpressions = subExpressions; } - execute(inputNoteSet, searchContext) { + async execute(inputNoteSet, searchContext) { const resultNoteSet = new NoteSet(); for (const subExpression of this.subExpressions) { - resultNoteSet.mergeIn(subExpression.execute(inputNoteSet, searchContext)); + resultNoteSet.mergeIn(await subExpression.execute(inputNoteSet, searchContext)); } return resultNoteSet; diff --git a/src/services/search/parser.js b/src/services/search/parser.js index 3ff60278e..e7b6bac15 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -25,7 +25,7 @@ function getFulltext(tokens, parsingContext) { else if (parsingContext.includeNoteContent) { return new OrExp([ new NoteCacheFulltextExp(tokens), - new NoteContentFulltextExp(tokens) + new NoteContentFulltextExp('*=*', tokens) ]); } else { @@ -55,6 +55,21 @@ function getExpression(tokens, parsingContext, level = 0) { i++; + if (tokens[i] === 'content') { + i += 1; + + const operator = tokens[i]; + + if (!isOperator(operator)) { + parsingContext.addError(`After content expected operator, but got "${tokens[i]}"`); + return; + } + + i++; + + return new NoteContentFulltextExp(operator, [tokens[i]]); + } + if (tokens[i] === 'parents') { i += 1; From 55b210d7c5a0a03567ae1039501c3c756036ef86 Mon Sep 17 00:00:00 2001 From: zadam Date: Wed, 27 May 2020 00:09:19 +0200 Subject: [PATCH 47/47] added not() expression --- spec/lexer.spec.js | 5 +++++ spec/parser.spec.js | 2 +- spec/search.spec.js | 16 ++++++++++++++++ src/services/search/parser.js | 10 ++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/spec/lexer.spec.js b/spec/lexer.spec.js index 4c65cecf3..2d366e763 100644 --- a/spec/lexer.spec.js +++ b/spec/lexer.spec.js @@ -63,4 +63,9 @@ describe("Lexer expression", () => { expect(lexer(`# ~author.title = 'Hugh Howey' AND note.'book title' = 'Silo'`).expressionTokens) .toEqual(["#", "~author", ".", "title", "=", "hugh howey", "and", "note", ".", "book title", "=", "silo"]); }); + + it("negation of sub-expression", () => { + expect(lexer(`# not(#capital) and note.noteId != "root"`).expressionTokens) + .toEqual(["#", "not", "(", "#capital", ")", "and", "note", ".", "noteid", "!=", "root"]); + }); }); diff --git a/spec/parser.spec.js b/spec/parser.spec.js index 360e679fa..b5ac8d8ee 100644 --- a/spec/parser.spec.js +++ b/spec/parser.spec.js @@ -46,7 +46,7 @@ describe("Parser", () => { it("simple label AND", () => { const rootExp = parser({ fulltextTokens: [], - expressionTokens: ["#first", "=", "text", "AND", "#second", "=", "text"], + expressionTokens: ["#first", "=", "text", "and", "#second", "=", "text"], parsingContext: new ParsingContext(true) }); diff --git a/spec/search.spec.js b/spec/search.spec.js index f193857bc..35ec670d8 100644 --- a/spec/search.spec.js +++ b/spec/search.spec.js @@ -512,5 +512,21 @@ describe("Search", () => { expect(searchResults.length).toEqual(4); }); + it("test not(...)", async () => { + const italy = note("Italy").label("capital", "Rome"); + const slovakia = note("Slovakia").label("capital", "Bratislava"); + + rootNote + .child(note("Europe") + .child(slovakia) + .child(italy)); + + const parsingContext = new ParsingContext(); + + let searchResults = await searchService.findNotesWithQuery('# not(#capital) and note.noteId != root', parsingContext); + expect(searchResults.length).toEqual(1); + expect(noteCache.notes[searchResults[0].noteId].title).toEqual("Europe"); + }); + // FIXME: test what happens when we order without any filter criteria }); diff --git a/src/services/search/parser.js b/src/services/search/parser.js index e7b6bac15..64102b6b5 100644 --- a/src/services/search/parser.js +++ b/src/services/search/parser.js @@ -255,6 +255,16 @@ function getExpression(tokens, parsingContext, level = 0) { return exp; } + else if (token === 'not') { + i += 1; + + if (!Array.isArray(tokens[i])) { + parsingContext.addError(`not keyword should be followed by sub-expression in parenthesis, got ${tokens[i]} instead`); + continue; + } + + expressions.push(new NotExp(getExpression(tokens[i], parsingContext, level++))); + } else if (token === 'note') { i++;