server-ts: Port services/export/zip

src/services/export/zip.ts

@@ -0,0 +1,588 @@
+"use strict";
+
+import html = require('html');
+import dateUtils = require('../date_utils');
+import path = require('path');
+import mimeTypes = require('mime-types');
+import mdService = require('./md');
+import packageInfo = require('../../../package.json');
+import utils = require('../utils');
+import protectedSessionService = require('../protected_session');
+import sanitize = require("sanitize-filename");
+import fs = require("fs");
+import becca = require('../../becca/becca');
+const RESOURCE_DIR = require('../../services/resource_dir').RESOURCE_DIR;
+import archiver = require('archiver');
+import log = require('../log');
+import TaskContext = require('../task_context');
+import ValidationError = require('../../errors/validation_error');
+import NoteMeta = require('../meta/note_meta');
+import AttachmentMeta = require('../meta/attachment_meta');
+import AttributeMeta = require('../meta/attribute_meta');
+import BBranch = require('../../becca/entities/bbranch');
+import { Response } from 'express';
+
+async function exportToZip(taskContext: TaskContext, branch: BBranch, format: "html" | "markdown", res: Response | fs.WriteStream, setHeaders = true) {
+    if (!['html', 'markdown'].includes(format)) {
+        throw new ValidationError(`Only 'html' and 'markdown' allowed as export format, '${format}' given`);
+    }
+
+    const archive = archiver('zip', {
+        zlib: { level: 9 } // Sets the compression level.
+    });
+
+    const noteIdToMeta: Record<string, NoteMeta> = {};
+
+    function getUniqueFilename(existingFileNames: Record<string, number>, fileName: string) {
+        const lcFileName = fileName.toLowerCase();
+
+        if (lcFileName in existingFileNames) {
+            let index;
+            let newName;
+
+            do {
+                index = existingFileNames[lcFileName]++;
+
+                newName = `${index}_${lcFileName}`;
+            }
+            while (newName in existingFileNames);
+
+            return `${index}_${fileName}`;
+        }
+        else {
+            existingFileNames[lcFileName] = 1;
+
+            return fileName;
+        }
+    }
+
+    function getDataFileName(type: string | null, mime: string, baseFileName: string, existingFileNames: Record<string, number>): string {
+        let fileName = baseFileName.trim();
+        if (fileName.length > 30) {
+            fileName = fileName.substr(0, 30).trim();
+        }
+
+        let existingExtension = path.extname(fileName).toLowerCase();
+        let newExtension;
+
+        // the following two are handled specifically since we always want to have these extensions no matter the automatic detection
+        // and/or existing detected extensions in the note name
+        if (type === 'text' && format === 'markdown') {
+            newExtension = 'md';
+        }
+        else if (type === 'text' && format === 'html') {
+            newExtension = 'html';
+        }
+        else if (mime === 'application/x-javascript' || mime === 'text/javascript') {
+            newExtension = 'js';
+        }
+        else if (existingExtension.length > 0) { // if the page already has an extension, then we'll just keep it
+            newExtension = null;
+        }
+        else {
+            if (mime?.toLowerCase()?.trim() === "image/jpg") {
+                newExtension = 'jpg';
+            } else if (mime?.toLowerCase()?.trim() === "text/mermaid") {
+                newExtension = 'txt';
+            } else {
+                newExtension = mimeTypes.extension(mime) || "dat";
+            }
+        }
+
+        // if the note is already named with the extension (e.g. "image.jpg"), then it's silly to append the exact same extension again
+        if (newExtension && existingExtension !== `.${newExtension.toLowerCase()}`) {
+            fileName += `.${newExtension}`;
+        }
+
+        return getUniqueFilename(existingFileNames, fileName);
+    }
+
+    function createNoteMeta(branch: BBranch, parentMeta: Partial<NoteMeta>, existingFileNames: Record<string, number>): NoteMeta | null {
+        const note = branch.getNote();
+
+        if (note.hasOwnedLabel('excludeFromExport')) {
+            return null;
+        }
+
+        const title = note.getTitleOrProtected();
+        const completeTitle = branch.prefix ? (`${branch.prefix} - ${title}`) : title;
+        let baseFileName = sanitize(completeTitle);
+
+        if (baseFileName.length > 200) { // the actual limit is 256 bytes(!) but let's be conservative
+            baseFileName = baseFileName.substr(0, 200);
+        }
+
+        if (!parentMeta.notePath) { throw new Error("Missing parent note path."); }
+        const notePath = parentMeta.notePath.concat([note.noteId]);
+
+        if (note.noteId in noteIdToMeta) {
+            const fileName = getUniqueFilename(existingFileNames, `${baseFileName}.clone.${format === 'html' ? 'html' : 'md'}`);
+
+            const meta: NoteMeta = {
+                isClone: true,
+                noteId: note.noteId,
+                notePath: notePath,
+                title: note.getTitleOrProtected(),
+                prefix: branch.prefix,
+                dataFileName: fileName,
+                type: 'text', // export will have text description
+                format: format
+            };
+            return meta;
+        }
+
+        const meta: Partial<NoteMeta> = {};
+        meta.isClone = false;
+        meta.noteId = note.noteId;
+        meta.notePath = notePath;
+        meta.title = note.getTitleOrProtected();
+        meta.notePosition = branch.notePosition;
+        meta.prefix = branch.prefix;
+        meta.isExpanded = branch.isExpanded;
+        meta.type = note.type;
+        meta.mime = note.mime;
+        meta.attributes = note.getOwnedAttributes().map(attribute => {
+            const attrMeta = {
+                type: attribute.type,
+                name: attribute.name,
+                value: attribute.value,
+                isInheritable: attribute.isInheritable,
+                position: attribute.position
+            };
+
+            return attrMeta;
+        });
+
+        taskContext.increaseProgressCount();
+
+        if (note.type === 'text') {
+            meta.format = format;
+        }
+
+        noteIdToMeta[note.noteId] = meta as NoteMeta;
+
+        // sort children for having a stable / reproducible export format
+        note.sortChildren();
+        const childBranches = note.getChildBranches()
+            .filter(branch => branch?.noteId !== '_hidden');
+
+        const available = !note.isProtected || protectedSessionService.isProtectedSessionAvailable();
+
+        // if it's a leaf, then we'll export it even if it's empty
+        if (available && (note.getContent().length > 0 || childBranches.length === 0)) {
+            meta.dataFileName = getDataFileName(note.type, note.mime, baseFileName, existingFileNames);
+        }
+
+        const attachments = note.getAttachments();
+        meta.attachments = attachments
+            .map(attachment => {
+                const attMeta: AttachmentMeta = {
+                    attachmentId: attachment.attachmentId,
+                    title: attachment.title,
+                    role: attachment.role,
+                    mime: attachment.mime,
+                    position: attachment.position,
+                    dataFileName: getDataFileName(
+                        null,
+                        attachment.mime,
+                        baseFileName + "_" + attachment.title,
+                        existingFileNames
+                    )
+                };
+                return attMeta;
+            });
+
+        if (childBranches.length > 0) {
+            meta.dirFileName = getUniqueFilename(existingFileNames, baseFileName);
+            meta.children = [];
+
+            // namespace is shared by children in the same note
+            const childExistingNames = {};
+
+            for (const childBranch of childBranches) {
+                if (!childBranch) { continue; }
+
+                const note = createNoteMeta(childBranch, meta as NoteMeta, childExistingNames);
+
+                // can be undefined if export is disabled for this note
+                if (note) {
+                    meta.children.push(note);
+                }
+            }
+        }
+
+        return meta as NoteMeta;
+    }
+
+    function getNoteTargetUrl(targetNoteId: string, sourceMeta: NoteMeta): string | null {
+        const targetMeta = noteIdToMeta[targetNoteId];
+
+        if (!targetMeta || !targetMeta.notePath || !sourceMeta.notePath) {
+            return null;
+        }
+
+        const targetPath = targetMeta.notePath.slice();
+        const sourcePath = sourceMeta.notePath.slice();
+
+        // > 1 for the edge case that targetPath and sourcePath are exact same (a link to itself)
+        while (targetPath.length > 1 && sourcePath.length > 1 && targetPath[0] === sourcePath[0]) {
+            targetPath.shift();
+            sourcePath.shift();
+        }
+
+        let url = "../".repeat(sourcePath.length - 1);
+
+        for (let i = 0; i < targetPath.length - 1; i++) {
+            const meta = noteIdToMeta[targetPath[i]];
+            if (meta.dirFileName) {
+                url += `${encodeURIComponent(meta.dirFileName)}/`;
+            }
+        }
+
+        const meta = noteIdToMeta[targetPath[targetPath.length - 1]];
+
+        // link can target note which is only "folder-note" and as such, will not have a file in an export
+        url += encodeURIComponent(meta.dataFileName || meta.dirFileName || "");
+
+        return url;
+    }
+
+    function rewriteLinks(content: string, noteMeta: NoteMeta): string {
+        content = content.replace(/src="[^"]*api\/images\/([a-zA-Z0-9_]+)\/[^"]*"/g, (match, targetNoteId) => {
+            const url = getNoteTargetUrl(targetNoteId, noteMeta);
+
+            return url ? `src="${url}"` : match;
+        });
+
+        content = content.replace(/src="[^"]*api\/attachments\/([a-zA-Z0-9_]+)\/image\/[^"]*"/g, (match, targetAttachmentId) => {
+            const url = findAttachment(targetAttachmentId);
+
+            return url ? `src="${url}"` : match;
+        });
+
+        content = content.replace(/href="[^"]*#root[^"]*attachmentId=([a-zA-Z0-9_]+)\/?"/g, (match, targetAttachmentId) => {
+            const url = findAttachment(targetAttachmentId);
+
+            return url ? `href="${url}"` : match;
+        });
+
+        content = content.replace(/href="[^"]*#root[a-zA-Z0-9_\/]*\/([a-zA-Z0-9_]+)[^"]*"/g, (match, targetNoteId) => {
+            const url = getNoteTargetUrl(targetNoteId, noteMeta);
+
+            return url ? `href="${url}"` : match;
+        });
+
+        return content;
+
+        function findAttachment(targetAttachmentId: string) {
+            let url;
+
+            const attachmentMeta = (noteMeta.attachments || []).find(attMeta => attMeta.attachmentId === targetAttachmentId);
+            if (attachmentMeta) {
+                // easy job here, because attachment will be in the same directory as the note's data file.
+                url = attachmentMeta.dataFileName;
+            } else {
+                log.info(`Could not find attachment meta object for attachmentId '${targetAttachmentId}'`);
+            }
+            return url;
+        }
+    }
+
+    function prepareContent(title: string, content: string | Buffer, noteMeta: NoteMeta): string | Buffer {
+        if (['html', 'markdown'].includes(noteMeta?.format || "")) {
+            content = content.toString();
+
+            content = rewriteLinks(content, noteMeta);
+        }
+
+        if (noteMeta.format === 'html' && typeof content === "string") {
+            if (!content.substr(0, 100).toLowerCase().includes("<html")) {
+                if (!noteMeta?.notePath?.length) { throw new Error("Missing note path."); }
+
+                const cssUrl = `${"../".repeat(noteMeta.notePath.length - 1)}style.css`;
+                const htmlTitle = utils.escapeHtml(title);
+
+                // <base> element will make sure external links are openable - https://github.com/zadam/trilium/issues/1289#issuecomment-704066809
+                content = `<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="${cssUrl}">
+    <base target="_parent">
+    <title data-trilium-title>${htmlTitle}</title>
+</head>
+<body>
+  <div class="content">
+      <h1 data-trilium-h1>${htmlTitle}</h1>
+      
+      <div class="ck-content">${content}</div>
+  </div>
+</body>
+</html>`;
+            }
+
+            return content.length < 100_000
+                ? html.prettyPrint(content, {indent_size: 2})
+                : content;
+        } else if (noteMeta.format === 'markdown' && typeof content === "string") {
+            let markdownContent = mdService.toMarkdown(content);
+
+            if (markdownContent.trim().length > 0 && !markdownContent.startsWith("# ")) {
+                markdownContent = `# ${title}\r
+${markdownContent}`;
+            }
+
+            return markdownContent;
+        } else {
+            return content;
+        }
+    }
+
+    function saveNote(noteMeta: NoteMeta, filePathPrefix: string) {
+        log.info(`Exporting note '${noteMeta.noteId}'`);
+
+        if (!noteMeta.noteId || !noteMeta.title) {
+            throw new Error("Missing note meta.");
+        }
+
+        if (noteMeta.isClone) {
+            const targetUrl = getNoteTargetUrl(noteMeta.noteId, noteMeta);
+
+            let content: string | Buffer = `<p>This is a clone of a note. Go to its <a href="${targetUrl}">primary location</a>.</p>`;
+
+            content = prepareContent(noteMeta.title, content, noteMeta);
+
+            archive.append(content, { name: filePathPrefix + noteMeta.dataFileName });
+
+            return;
+        }
+
+        const note = becca.getNote(noteMeta.noteId);
+        if (!note) { throw new Error("Unable to find note."); }
+        if (!note.utcDateModified) { throw new Error("Unable to find modification date."); }
+
+        if (noteMeta.dataFileName) {
+            const content = prepareContent(noteMeta.title, note.getContent(), noteMeta);
+
+            archive.append(content, {
+                name: filePathPrefix + noteMeta.dataFileName,
+                date: dateUtils.parseDateTime(note.utcDateModified)
+            });
+        }
+
+        taskContext.increaseProgressCount();
+
+        for (const attachmentMeta of noteMeta.attachments || []) {
+            if (!attachmentMeta.attachmentId) { continue; }
+
+            const attachment = note.getAttachmentById(attachmentMeta.attachmentId);
+            const content = attachment.getContent();
+
+            archive.append(content, {
+                name: filePathPrefix + attachmentMeta.dataFileName,
+                date: dateUtils.parseDateTime(note.utcDateModified)
+            });
+        }
+
+        if (noteMeta.children?.length || 0 > 0) {
+            const directoryPath = filePathPrefix + noteMeta.dirFileName;
+
+            // create directory
+            archive.append('', { name: `${directoryPath}/`, date: dateUtils.parseDateTime(note.utcDateModified) });
+
+            for (const childMeta of noteMeta.children || []) {
+                saveNote(childMeta, `${directoryPath}/`);
+            }
+        }
+    }
+
+    function saveNavigation(rootMeta: NoteMeta, navigationMeta: NoteMeta) {
+        function saveNavigationInner(meta: NoteMeta) {
+            let html = '<li>';
+
+            const escapedTitle = utils.escapeHtml(`${meta.prefix ? `${meta.prefix} - ` : ''}${meta.title}`);
+
+            if (meta.dataFileName && meta.noteId) {
+                const targetUrl = getNoteTargetUrl(meta.noteId, rootMeta);
+
+                html += `<a href="${targetUrl}" target="detail">${escapedTitle}</a>`;
+            }
+            else {
+                html += escapedTitle;
+            }
+
+            if (meta.children && meta.children.length > 0) {
+                html += '<ul>';
+
+                for (const child of meta.children) {
+                    html += saveNavigationInner(child);
+                }
+
+                html += '</ul>'
+            }
+
+            return `${html}</li>`;
+        }
+
+        const fullHtml = `<html>
+<head>
+    <meta charset="utf-8">
+    <link rel="stylesheet" href="style.css">
+</head>
+<body>
+    <ul>${saveNavigationInner(rootMeta)}</ul>
+</body>
+</html>`;
+        const prettyHtml = fullHtml.length < 100_000
+            ? html.prettyPrint(fullHtml, {indent_size: 2})
+            : fullHtml;
+
+        archive.append(prettyHtml, { name: navigationMeta.dataFileName });
+    }
+
+    function saveIndex(rootMeta: NoteMeta, indexMeta: NoteMeta) {
+        let firstNonEmptyNote;
+        let curMeta = rootMeta;
+
+        while (!firstNonEmptyNote) {
+            if (curMeta.dataFileName && curMeta.noteId) {
+                firstNonEmptyNote = getNoteTargetUrl(curMeta.noteId, rootMeta);
+            }
+
+            if (curMeta.children && curMeta.children.length > 0) {
+                curMeta = curMeta.children[0];
+            }
+            else {
+                break;
+            }
+        }
+
+        const fullHtml = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+</head>
+<frameset cols="25%,75%">
+    <frame name="navigation" src="navigation.html">
+    <frame name="detail" src="${firstNonEmptyNote}">
+</frameset>
+</html>`;
+
+        archive.append(fullHtml, { name: indexMeta.dataFileName });
+    }
+
+    function saveCss(rootMeta: NoteMeta, cssMeta: NoteMeta) {
+        const cssContent = fs.readFileSync(`${RESOURCE_DIR}/libraries/ckeditor/ckeditor-content.css`);
+
+        archive.append(cssContent, { name: cssMeta.dataFileName });
+    }
+
+    const existingFileNames: Record<string, number> = format === 'html' ? {'navigation': 0, 'index': 1} : {};
+    const rootMeta = createNoteMeta(branch, { notePath: [] }, existingFileNames);
+
+    const metaFile = {
+        formatVersion: 2,
+        appVersion: packageInfo.version,
+        files: [ rootMeta ]
+    };
+
+    let navigationMeta: NoteMeta | null = null;
+    let indexMeta: NoteMeta | null = null;
+    let cssMeta: NoteMeta | null = null;
+
+    if (format === 'html') {
+        navigationMeta = {
+            noImport: true,
+            dataFileName: "navigation.html"
+        };
+
+        metaFile.files.push(navigationMeta);
+
+        indexMeta = {
+            noImport: true,
+            dataFileName: "index.html"
+        };
+
+        metaFile.files.push(indexMeta);
+
+        cssMeta = {
+            noImport: true,
+            dataFileName: "style.css"
+        };
+
+        metaFile.files.push(cssMeta);
+    }
+
+    for (const noteMeta of Object.values(noteIdToMeta)) {
+        // filter out relations which are not inside this export
+        noteMeta.attributes = (noteMeta.attributes || []).filter(attr => {
+            if (attr.type !== 'relation') {
+                return true;
+            } else if (attr.value in noteIdToMeta) {
+                return true;
+            } else if (attr.value === 'root' || attr.value?.startsWith("_")) {
+                // relations to "named" noteIds can be preserved
+                return true;
+            } else {
+                return false;
+            }
+        });
+    }
+
+    if (!rootMeta) { // corner case of disabled export for exported note
+        if ("sendStatus" in res) {
+            res.sendStatus(400);
+        }
+        return;
+    }
+
+    const metaFileJson = JSON.stringify(metaFile, null, '\t');
+
+    archive.append(metaFileJson, { name: "!!!meta.json" });
+
+    saveNote(rootMeta, '');
+
+    if (format === 'html') {
+        if (!navigationMeta || !indexMeta || !cssMeta) {
+            throw new Error("Missing meta.");
+        }
+
+        saveNavigation(rootMeta, navigationMeta);
+        saveIndex(rootMeta, indexMeta);
+        saveCss(rootMeta, cssMeta);
+    }
+
+    const note = branch.getNote();
+    const zipFileName = `${branch.prefix ? `${branch.prefix} - ` : ""}${note.getTitleOrProtected()}.zip`;
+
+    if (setHeaders && "setHeader" in res) {
+        res.setHeader('Content-Disposition', utils.getContentDisposition(zipFileName));
+        res.setHeader('Content-Type', 'application/zip');
+    }
+
+    archive.pipe(res);
+    await archive.finalize();
+
+    taskContext.taskSucceeded();
+}
+
+async function exportToZipFile(noteId: string, format: "markdown" | "html", zipFilePath: string) {
+    const fileOutputStream = fs.createWriteStream(zipFilePath);
+    const taskContext = new TaskContext('no-progress-reporting');
+
+    const note = becca.getNote(noteId);
+
+    if (!note) {
+        throw new ValidationError(`Note ${noteId} not found.`);
+    }
+
+    await exportToZip(taskContext, note.getParentBranches()[0], format, fileOutputStream, false);
+
+    log.info(`Exported '${noteId}' with format '${format}' to '${zipFilePath}'`);
+}
+
+export = {
+    exportToZip,
+    exportToZipFile
+};