changed export model to single metadata file per exported .tar

src/entities/note.js

@@ -4,6 +4,7 @@ const Entity = require('./entity');
 const Attribute = require('./attribute');
 const protectedSessionService = require('../services/protected_session');
 const repository = require('../services/repository');
+const sql = require('../services/sql');
 const dateUtils = require('../services/date_utils');
 
 const LABEL = 'label';
@@ -433,14 +434,32 @@ class Note extends Entity {
     }
 
     /**
-     * Finds child notes with given attribute name and value. Only own attributes are considered, not inherited ones
+     * @return {Promise<string[]>} return list of all descendant noteIds of this note. Returning just noteIds because number of notes can be huge. Includes also this note's noteId
+     */
+    async getDescendantNoteIds() {
+        return await sql.getColumn(`
+            WITH RECURSIVE
+            tree(noteId) AS (
+                SELECT ?
+                UNION
+                SELECT branches.noteId FROM branches
+                    JOIN tree ON branches.parentNoteId = tree.noteId
+                    JOIN notes ON notes.noteId = branches.noteId
+                WHERE notes.isDeleted = 0
+                  AND branches.isDeleted = 0
+            )
+            SELECT noteId FROM tree`, [this.noteId]);
+    }
+
+    /**
+     * Finds descendant notes with given attribute name and value. Only own attributes are considered, not inherited ones
      *
      * @param {string} type - attribute type (label, relation, etc.)
      * @param {string} name - attribute name
      * @param {string} [value] - attribute value
      * @returns {Promise<Note[]>}
      */
-    async findChildNotesWithAttribute(type, name, value) {
+    async getDescendantNotesWithAttribute(type, name, value) {
         const params = [this.noteId, name];
         let valueCondition = "";
 
@@ -472,22 +491,22 @@ class Note extends Entity {
     }
 
     /**
-     * Finds notes with given label name and value. Only own labels are considered, not inherited ones
+     * Finds descendant notes with given label name and value. Only own labels are considered, not inherited ones
      *
      * @param {string} name - label name
      * @param {string} [value] - label value
      * @returns {Promise<Note[]>}
      */
-    async findChildNotesWithLabel(name, value) { return await this.findChildNotesWithAttribute(LABEL, name, value); }
+    async getDescendantNotesWithLabel(name, value) { return await this.getDescendantNotesWithAttribute(LABEL, name, value); }
 
     /**
-     * Finds notes with given relation name and value. Only own relations are considered, not inherited ones
+     * Finds descendant notes with given relation name and value. Only own relations are considered, not inherited ones
      *
      * @param {string} name - relation name
      * @param {string} [value] - relation value
      * @returns {Promise<Note[]>}
      */
-    async findChildNotesWithRelation(name, value) { return await this.findChildNotesWithAttribute(RELATION, name, value); }
+    async getDescendantNotesWithRelation(name, value) { return await this.getDescendantNotesWithAttribute(RELATION, name, value); }
 
     /**
      * Returns note revisions of this note.

src/services/export/tar.js

@@ -1,6 +1,7 @@
 "use strict";
 
 const html = require('html');
+const repository = require('../repository');
 const tar = require('tar-stream');
 const path = require('path');
 const sanitize = require("sanitize-filename");
@@ -11,56 +12,79 @@ const TurndownService = require('turndown');
  * @param format - 'html' or 'markdown'
  */
 async function exportToTar(branch, format, res) {
-    const turndownService = new TurndownService();
+    let turndownService = format === 'markdown' ? new TurndownService() : null;
-
-    // path -> number of occurences
-    const existingPaths = {};
 
     const pack = tar.pack();
 
     const exportedNoteIds = [];
-    const name = await exportNoteInner(branch, '');
 
-    function getUniqueFilename(fileName) {
+    function getUniqueFilename(existingFileNames, fileName) {
         const lcFileName = fileName.toLowerCase();
 
-        if (lcFileName in existingPaths) {
+        if (lcFileName in existingFileNames) {
             let index;
             let newName;
 
             do {
-                index = existingPaths[lcFileName]++;
+                index = existingFileNames[lcFileName]++;
 
                 newName = lcFileName + "_" + index;
             }
-            while (newName in existingPaths);
+            while (newName in existingFileNames);
 
             return fileName + "_" + index;
         }
         else {
-            existingPaths[lcFileName] = 1;
+            existingFileNames[lcFileName] = 1;
 
             return fileName;
         }
     }
 
-    async function exportNoteInner(branch, directory, existingNames) {
+    function getDataFileName(note, baseFileName, existingFileNames) {
+        let extension;
+
+        if (note.type === 'text' && format === 'markdown') {
+            extension = 'md';
+        }
+        else if (note.mime === 'application/x-javascript') {
+            extension = 'js';
+        }
+        else {
+            extension = mimeTypes.extension(note.mime) || "dat";
+        }
+
+        let fileName = baseFileName;
+
+        if (!fileName.toLowerCase().endsWith(extension)) {
+            fileName += "." + extension;
+        }
+
+        return getUniqueFilename(existingFileNames, fileName);
+    }
+
+    async function getNote(branch, existingFileNames) {
         const note = await branch.getNote();
-        const baseFileName = getUniqueFilename(directory + sanitize(note.title));
-
-        if (exportedNoteIds.includes(note.noteId)) {
-            saveMetadataFile(baseFileName, {
-                version: 1,
-                clone: true,
-                noteId: note.noteId,
-                prefix: branch.prefix
-            });
 
+        if (await note.hasLabel('excludeFromExport')) {
             return;
         }
 
-        const metadata = {
+        const baseFileName = branch.prefix ? (branch.prefix + ' - ' + note.title) : note.title;
-            version: 1,
+
+        if (exportedNoteIds.includes(note.noteId)) {
+            const sanitizedFileName = sanitize(baseFileName + ".clone");
+            const fileName = getUniqueFilename(existingFileNames, sanitizedFileName);
+
+            return {
+                clone: true,
+                noteId: note.noteId,
+                prefix: branch.prefix,
+                dataFileName: fileName
+            };
+        }
+
+        const meta = {
             clone: false,
             noteId: note.noteId,
             title: note.title,
@@ -87,89 +111,105 @@ async function exportToTar(branch, format, res) {
         };
 
         if (note.type === 'text') {
-            metadata.format = format;
+            meta.format = format;
         }
 
-        if (await note.hasLabel('excludeFromExport')) {
-            return;
-        }
-
-        metadata.dataFilename = saveDataFile(baseFileName, note);
-
-        saveMetadataFile(baseFileName, metadata);
-
         exportedNoteIds.push(note.noteId);
 
         const childBranches = await note.getChildBranches();
 
+        // if it's a leaf then we'll export it even if it's empty
+        if (note.content.length > 0 || childBranches.length === 0) {
+            meta.dataFileName = getDataFileName(note, baseFileName, existingFileNames);
+        }
+
         if (childBranches.length > 0) {
-            saveDirectory(baseFileName);
+            meta.dirFileName = getUniqueFilename(existingFileNames, baseFileName);
-        }
+            meta.children = [];
 
-        for (const childBranch of childBranches) {
+            const childExistingNames = {};
-            await exportNoteInner(childBranch, baseFileName + "/");
-        }
 
-        return baseFileName;
+            for (const childBranch of childBranches) {
-    }
+                const note = await getNote(childBranch, existingFileNames);
 
-    function saveDataFile(baseFilename, note) {
+                // can be undefined if export is disabled for this note
-        let content = note.content;
+                if (note) {
-        let extension;
+                    meta.children.push(note);
-
+                }
-        if (note.type === 'text') {
-            if (format === 'html') {
-                content = html.prettyPrint(note.content, {indent_size: 2});
-            }
-            else if (format === 'markdown') {
-                content = turndownService.turndown(note.content);
-                extension = 'md';
-            }
-            else {
-                throw new Error("Unknown format: " + format);
             }
         }
 
-        if (!extension) {
+        return meta;
-            extension = mimeTypes.extension(note.mime)
-                || getExceptionalExtension(note.mime)
-                || "dat";
-        }
-
-        let filename = baseFilename;
-
-        if (!filename.toLowerCase().endsWith(extension)) {
-            filename += "." + extension;
-        }
-
-        filename = getUniqueFilename(filename);
-
-        pack.entry({name: filename, size: content.length}, content);
-
-        return path.basename(filename);
     }
 
-    function getExceptionalExtension(mime) {
+    function prepareContent(note, format) {
-        if (mime === 'application/x-javascript') {
+        if (format === 'html') {
-            return 'js';
+            return html.prettyPrint(note.content, {indent_size: 2});
+        }
+        else if (format === 'markdown') {
+            return turndownService.turndown(note.content);
+        }
+        else {
+            return note.content;
         }
     }
 
-    function saveMetadataFile(baseFileName, metadata) {
+    // noteId => file path
-        const metadataJson = JSON.stringify(metadata, null, '\t');
+    const notePaths = {};
 
-        const fileName = getUniqueFilename(baseFileName + ".meta");
+    async function saveNote(noteMeta, path) {
+        if (noteMeta.clone) {
+            const content = "Note is present at " + notePaths[noteMeta.noteId];
 
-        pack.entry({name: fileName, size: metadataJson.length}, metadataJson);
+            pack.entry({name: path + '/' + noteMeta.dataFileName, size: content.length}, content);
+            return;
+        }
+
+        const note = await repository.getNote(noteMeta.noteId);
+
+        notePaths[note.noteId] = path + '/' + (noteMeta.dataFileName || noteMeta.dirFileName);
+
+        if (noteMeta.dataFileName) {
+            const content = prepareContent(note, noteMeta.format);
+
+            pack.entry({name: path + '/' + noteMeta.dataFileName, size: content.length}, content);
+        }
+
+        if (noteMeta.children && noteMeta.children.length > 0) {
+            const directoryPath = path + '/' + noteMeta.dirFileName;
+
+            pack.entry({name: directoryPath, type: 'directory'});
+
+            for (const childMeta of noteMeta.children) {
+                await saveNote(childMeta, directoryPath);
+            }
+        }
     }
 
-    function saveDirectory(baseFileName) {
+    const metaFile = {
-        pack.entry({name: baseFileName, type: 'directory'});
+        version: 1,
+        files: [
+            await getNote(branch, [])
+        ]
+    };
+
+    if (!metaFile.files[0]) { // corner case of disabled export for exported note
+        res.sendStatus(400);
+        return;
     }
 
+    const metaFileJson = JSON.stringify(metaFile, null, '\t');
+
+    pack.entry({name: "!!!meta.json", size: metaFileJson.length}, metaFileJson);
+
+    await saveNote(metaFile.files[0], '');
+
     pack.finalize();
 
-    res.setHeader('Content-Disposition', 'file; filename="' + name + '.tar"');
+    const note = await branch.getNote();
+    const tarFileName = sanitize((branch.prefix ? (branch.prefix + " - ") : "") + note.title);
+
+    res.setHeader('Content-Disposition', `file; filename="${tarFileName}.tar"`);
     res.setHeader('Content-Type', 'application/tar');
 
     pack.pipe(res);