script to generate large documents, closes #55

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "trilium",
-  "version": "0.9.1-beta",
+  "version": "0.9.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -7121,6 +7121,32 @@
         "js-tokens": "3.0.2"
       }
     },
+    "lorem-ipsum": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/lorem-ipsum/-/lorem-ipsum-1.0.4.tgz",
+      "integrity": "sha1-MLcqOx4ZH1UGKvjH36spGuT72RI=",
+      "dev": true,
+      "requires": {
+        "optimist": "0.3.7"
+      },
+      "dependencies": {
+        "optimist": {
+          "version": "0.3.7",
+          "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.7.tgz",
+          "integrity": "sha1-yQlBrVnkJzMokjB00s8ufLxuwNk=",
+          "dev": true,
+          "requires": {
+            "wordwrap": "0.0.3"
+          }
+        },
+        "wordwrap": {
+          "version": "0.0.3",
+          "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
+          "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
+          "dev": true
+        }
+      }
+    },
     "loud-rejection": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz",

package.json

@@ -66,6 +66,7 @@
     "electron-compile": "^6.4.2",
     "electron-packager": "^11.1.0",
     "electron-prebuilt-compile": "2.0.0-beta.5",
+    "lorem-ipsum": "^1.0.4",
     "tape": "^4.9.0",
     "xo": "^0.18.0"
   },

src/entities/branch.js

@@ -3,6 +3,7 @@
 const Entity = require('./entity');
 const dateUtils = require('../services/date_utils');
 const repository = require('../services/repository');
+const sql = require('../services/sql');
 
 class Branch extends Entity {
     static get tableName() { return "branches"; }
@@ -12,9 +13,14 @@ class Branch extends Entity {
         return await repository.getEntity("SELECT * FROM notes WHERE noteId = ?", [this.noteId]);
     }
 
-    beforeSaving() {
+    async beforeSaving() {
         super.beforeSaving();
 
+        if (this.notePosition === undefined) {
+            const maxNotePos = await sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [this.parentNoteId]);
+            this.notePosition = maxNotePos === null ? 0 : maxNotePos + 1;
+        }
+
         if (!this.isDeleted) {
             this.isDeleted = false;
         }

src/public/javascripts/dialogs/jump_to_note.js

@@ -19,7 +19,7 @@ async function showDialog() {
 
     await $autoComplete.autocomplete({
         source: await utils.stopWatch("building autocomplete", autocompleteService.getAutocompleteItems),
-        minLength: 0
+        minLength: 1
     });
 }
 

src/public/javascripts/services/autocomplete.js

@@ -46,6 +46,10 @@ async function getAutocompleteItems(parentNoteId, notePath, titlePath) {
         }
     }
 
+    if (parentNoteId === 'root') {
+        console.log(`Generated ${autocompleteItems.length} autocomplete items`);
+    }
+
     return autocompleteItems;
 }
 

src/routes/api/branches.js

@@ -8,8 +8,8 @@ const notes = require('../../services/notes');
 const repository = require('../../services/repository');
 
 /**
- * Code in this file deals with moving and cloning note tree rows. Relationship between note and parent note is unique
- * for not deleted note trees. There may be multiple deleted note-parent note relationships.
+ * Code in this file deals with moving and cloning branches. Relationship between note and parent note is unique
+ * for not deleted branches. There may be multiple deleted note-parent note relationships.
  */
 
 async function moveBranchToParent(req) {
@@ -49,7 +49,7 @@ async function moveBranchBeforeNote(req) {
     }
 
     // we don't change dateModified so other changes are prioritized in case of conflict
-    // also we would have to sync all those modified note trees otherwise hash checks would fail
+    // also we would have to sync all those modified branches otherwise hash checks would fail
     await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition >= ? AND isDeleted = 0",
         [beforeNote.parentNoteId, beforeNote.notePosition]);
 
@@ -77,7 +77,7 @@ async function moveBranchAfterNote(req) {
     }
 
     // we don't change dateModified so other changes are prioritized in case of conflict
-    // also we would have to sync all those modified note trees otherwise hash checks would fail
+    // also we would have to sync all those modified branches otherwise hash checks would fail
     await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0",
         [afterNote.parentNoteId, afterNote.notePosition]);
 

src/routes/api/cloning.js

@@ -1,64 +1,20 @@
 "use strict";
 
-const sql = require('../../services/sql');
-const syncTable = require('../../services/sync_table');
-const tree = require('../../services/tree');
-const Branch = require('../../entities/branch');
+const cloningService = require('../../services/cloning');
 
 async function cloneNoteToParent(req) {
     const noteId = req.params.noteId;
     const parentNoteId = req.params.parentNoteId;
     const prefix = req.body.prefix;
 
-    const validationResult = await tree.validateParentChild(parentNoteId, noteId);
-
-    if (!validationResult.success) {
-        return validationResult;
-    }
-
-    const maxNotePos = await sql.getValue('SELECT MAX(notePosition) FROM branches WHERE parentNoteId = ? AND isDeleted = 0', [parentNoteId]);
-    const newNotePos = maxNotePos === null ? 0 : maxNotePos + 1;
-
-    const branch = await new Branch({
-        noteId: noteId,
-        parentNoteId: parentNoteId,
-        prefix: prefix,
-        notePosition: newNotePos,
-        isExpanded: 0
-    }).save();
-
-    await sql.execute("UPDATE branches SET isExpanded = 1 WHERE noteId = ?", [parentNoteId]);
-
-    return { success: true };
+    return await cloningService.cloneNoteToParent(noteId, parentNoteId, prefix);
 }
 
 async function cloneNoteAfter(req) {
     const noteId = req.params.noteId;
     const afterBranchId = req.params.afterBranchId;
 
-    const afterNote = await tree.getBranch(afterBranchId);
-
-    const validationResult = await tree.validateParentChild(afterNote.parentNoteId, noteId);
-
-    if (!validationResult.result) {
-        return validationResult;
-    }
-
-    // we don't change dateModified so other changes are prioritized in case of conflict
-    // also we would have to sync all those modified note trees otherwise hash checks would fail
-    await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0",
-        [afterNote.parentNoteId, afterNote.notePosition]);
-
-    await syncTable.addNoteReorderingSync(afterNote.parentNoteId);
-
-    const branch = await new Branch({
-        noteId: noteId,
-        parentNoteId: afterNote.parentNoteId,
-        notePosition: afterNote.notePosition + 1,
-        isExpanded: 0
-    }).save();
-
-    return { success: true };
+    return await cloningService.cloneNoteAfter(noteId, afterBranchId);
 }
 
 module.exports = {

src/routes/api/import.js

@@ -110,17 +110,17 @@ async function importNotes(files, parentNoteId) {
             file.data = file.data.toString("UTF-8");
         }
 
-        const noteId = await noteService.createNote(parentNoteId, file.meta.title, file.data, {
+        const {note} = await noteService.createNote(parentNoteId, file.meta.title, file.data, {
             type: file.meta.type,
             mime: file.meta.mime
         });
 
         for (const label of file.meta.labels) {
-            await labelService.createLabel(noteId, label.name, label.value);
+            await labelService.createLabel(note.noteId, label.name, label.value);
         }
 
         if (file.children.length > 0) {
-            await importNotes(file.children, noteId);
+            await importNotes(file.children, note.noteId);
         }
     }
 }

src/routes/api/search.js

@@ -20,13 +20,13 @@ async function saveSearchToNote(req) {
         searchString: req.params.searchString
     };
 
-    const noteId = await noteService.createNote('root', 'Search note', noteContent, {
+    const {note} = await noteService.createNote('root', 'Search note', noteContent, {
         json: true,
         type: 'search',
         mime: "application/json"
     });
 
-    return { noteId };
+    return { noteId: note.noteId };
 }
 
 module.exports = {

src/services/cloning.js

@@ -0,0 +1,56 @@
+"use strict";
+
+const sql = require('./sql');
+const syncTable = require('./sync_table');
+const tree = require('./tree');
+const Branch = require('../entities/branch');
+
+async function cloneNoteToParent(noteId, parentNoteId, prefix) {
+    const validationResult = await tree.validateParentChild(parentNoteId, noteId);
+
+    if (!validationResult.success) {
+        return validationResult;
+    }
+
+    await new Branch({
+        noteId: noteId,
+        parentNoteId: parentNoteId,
+        prefix: prefix,
+        isExpanded: 0
+    }).save();
+
+    await sql.execute("UPDATE branches SET isExpanded = 1 WHERE noteId = ?", [parentNoteId]);
+
+    return { success: true };
+}
+
+async function cloneNoteAfter(noteId, afterBranchId) {
+    const afterNote = await tree.getBranch(afterBranchId);
+
+    const validationResult = await tree.validateParentChild(afterNote.parentNoteId, noteId);
+
+    if (!validationResult.result) {
+        return validationResult;
+    }
+
+    // we don't change dateModified so other changes are prioritized in case of conflict
+    // also we would have to sync all those modified branches otherwise hash checks would fail
+    await sql.execute("UPDATE branches SET notePosition = notePosition + 1 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0",
+        [afterNote.parentNoteId, afterNote.notePosition]);
+
+    await syncTable.addNoteReorderingSync(afterNote.parentNoteId);
+
+    await new Branch({
+        noteId: noteId,
+        parentNoteId: afterNote.parentNoteId,
+        notePosition: afterNote.notePosition + 1,
+        isExpanded: 0
+    }).save();
+
+    return { success: true };
+}
+
+module.exports = {
+    cloneNoteToParent,
+    cloneNoteAfter
+};

src/services/consistency_checks.js

@@ -105,7 +105,7 @@ async function runAllChecks() {
             LEFT JOIN notes USING(noteId) 
           WHERE 
             notes.noteId IS NULL`,
-        "Missing notes records for following note tree ID > note ID", errorList);
+        "Missing notes records for following branch ID > note ID", errorList);
 
     await runCheck(`
           SELECT 
@@ -116,7 +116,7 @@ async function runAllChecks() {
           WHERE 
             notes.isDeleted = 1 
             AND branches.isDeleted = 0`,
-        "Note tree is not deleted even though main note is deleted for following note tree IDs", errorList);
+        "Note tree is not deleted even though main note is deleted for following branch IDs", errorList);
 
     await runCheck(`
           SELECT 
@@ -128,7 +128,7 @@ async function runAllChecks() {
             AND child.parentNoteId != 'root'
             AND (SELECT COUNT(*) FROM branches AS parent WHERE parent.noteId = child.parentNoteId 
                                                                  AND parent.isDeleted = 0) = 0`,
-        "All parent note trees are deleted but child note tree is not for these child note tree IDs", errorList);
+        "All parent branches are deleted but child note tree is not for these child note tree IDs", errorList);
 
     // we do extra JOIN to eliminate orphan notes without note tree (which are reported separately)
     await runCheck(`
@@ -140,7 +140,7 @@ async function runAllChecks() {
           WHERE
             (SELECT COUNT(*) FROM branches WHERE notes.noteId = branches.noteId AND branches.isDeleted = 0) = 0
             AND notes.isDeleted = 0
-    `, 'No undeleted note trees for note IDs', errorList);
+    `, 'No undeleted branches for note IDs', errorList);
 
     await runCheck(`
           SELECT 

src/services/notes.js

@@ -83,7 +83,7 @@ async function createNote(parentNoteId, title, content = "", extraOptions = {})
         noteData.mime = "application/json";
     }
 
-    const {note} = await createNewNote(parentNoteId, noteData);
+    const {note, branch} = await createNewNote(parentNoteId, noteData);
 
     if (extraOptions.labels) {
         for (const labelName in extraOptions.labels) {
@@ -91,7 +91,7 @@ async function createNote(parentNoteId, title, content = "", extraOptions = {})
         }
     }
 
-    return note.noteId;
+    return {note, branch};
 }
 
 async function protectNoteRecursively(note, protect) {

src/services/sql_init.js

@@ -11,6 +11,9 @@ async function createConnection() {
     return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise});
 }
 
+let schemaReadyResolve = null;
+const schemaReady = new Promise((resolve, reject) => schemaReadyResolve = resolve);
+
 let dbReadyResolve = null;
 const dbReady = new Promise((resolve, reject) => {
     cls.init(async () => {
@@ -29,7 +32,9 @@ const dbReady = new Promise((resolve, reject) => {
         if (tableResults.length !== 1) {
             await createInitialDatabase();
         }
-        else {
+
+        schemaReadyResolve();
+
         if (!await isUserInitialized()) {
             log.info("Login/password not initialized. DB not ready.");
 
@@ -41,7 +46,6 @@ const dbReady = new Promise((resolve, reject) => {
         }
 
         resolve(db);
-        }
     });
 });
 
@@ -97,6 +101,7 @@ async function isUserInitialized() {
 
 module.exports = {
     dbReady,
+    schemaReady,
     isUserInitialized,
     setDbReadyAsResolved,
     isDbUpToDate

src/services/sync_update.js

@@ -37,7 +37,7 @@ async function updateBranch(entity, sourceId) {
 
             await syncTableService.addBranchSync(entity.branchId, sourceId);
 
-            log.info("Update/sync note tree " + entity.branchId);
+            log.info("Update/sync branch " + entity.branchId);
         }
     });
 }

src/tools/generate_document.js

@@ -0,0 +1,74 @@
+const fs = require('fs');
+const dataDir = require('../services/data_dir');
+
+fs.unlinkSync(dataDir.DOCUMENT_PATH);
+
+require('../entities/entity_constructor');
+const optionService = require('../services/options');
+const sqlInit = require('../services/sql_init');
+const myScryptService = require('../services/my_scrypt');
+const passwordEncryptionService = require('../services/password_encryption');
+const utils = require('../services/utils');
+const noteService = require('../services/notes');
+const cls = require('../services/cls');
+const cloningService = require('../services/cloning');
+const loremIpsum = require('lorem-ipsum');
+
+async function setUserNamePassword() {
+    const username = "test";
+    const password = "test";
+
+    await optionService.setOption('username', username);
+
+    await optionService.setOption('passwordVerificationSalt', utils.randomSecureToken(32));
+    await optionService.setOption('passwordDerivedKeySalt', utils.randomSecureToken(32));
+
+    const passwordVerificationKey = utils.toBase64(await myScryptService.getVerificationHash(password));
+    await optionService.setOption('passwordVerificationHash', passwordVerificationKey);
+
+    await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16));
+
+    sqlInit.setDbReadyAsResolved();
+}
+
+const noteCount = parseInt(process.argv[2]);
+const notes = ['root'];
+
+function getRandomParentNoteId() {
+    const index = Math.floor(Math.random() * notes.length);
+
+    return notes[index];
+}
+
+async function start() {
+    await setUserNamePassword();
+
+    for (let i = 0; i < noteCount; i++) {
+        const title = loremIpsum({ count: 1, units: 'sentences', sentenceLowerBound: 1, sentenceUpperBound: 10 });
+
+        const paragraphCount = Math.floor(Math.random() * Math.random() * 100);
+        const content = loremIpsum({ count: paragraphCount, units: 'paragraphs', sentenceLowerBound: 1, sentenceUpperBound: 15,
+            paragraphLowerBound: 3, paragraphUpperBound: 10, format: 'html' });
+
+        const {note} = await noteService.createNote(getRandomParentNoteId(), title, content);
+
+        console.log(`Created note ${i}: ${title}`);
+
+        notes.push(note.noteId);
+    }
+
+    // we'll create clones for 20% of notes
+    for (let i = 0; i < (noteCount / 5); i++) {
+        const noteIdToClone = getRandomParentNoteId();
+        const parentNoteId = getRandomParentNoteId();
+        const prefix = Math.random() > 0.8 ? "prefix" : null;
+
+        const result = await cloningService.cloneNoteToParent(noteIdToClone, parentNoteId, prefix);
+
+        console.log(`Cloning ${i}:`, result.success ? "succeeded" : "FAILED");
+    }
+
+    process.exit(0);
+}
+
+sqlInit.schemaReady.then(cls.wrap(start));