From 4ffaadd481abb88fc36d3e765705d37c59ff10f6 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 11:40:06 +0200 Subject: [PATCH 01/18] chore(import): move all services to core (with errors) --- .../trilium-core}/src/services/import/common.ts | 0 .../trilium-core}/src/services/import/enex.ts | 0 .../src/services/import/markdown.spec.ts | 0 .../trilium-core}/src/services/import/markdown.ts | 0 .../import/markdown/wikilink_internal_link.ts | 0 .../import/markdown/wikilink_transclusion.ts | 0 .../trilium-core}/src/services/import/mime.spec.ts | 0 .../trilium-core}/src/services/import/mime.ts | 0 .../trilium-core}/src/services/import/opml.ts | 0 .../import/samples/IREN Reports Q2 FY25 Results.htm | Bin .../samples/IREN.Reports.Q2.FY25.Results_files.zip | Bin .../src/services/import/samples/New note.excalidraw | 0 .../src/services/import/samples/New note.mermaid | 0 .../src/services/import/samples/New note.mmd | 0 .../src/services/import/samples/Text Note.mdx | 0 .../services/import/samples/UTF-16LE Code Note.json | Bin .../services/import/samples/UTF-16LE Text Note.md | Bin .../services/import/samples/UTF-16LE Text Note.txt | Bin .../src/services/import/samples/geomap.zip | Bin .../src/services/import/samples/mdx.zip | Bin .../src/services/import/samples/silverbullet.zip | Bin .../src/services/import/single.spec.ts | 0 .../trilium-core}/src/services/import/single.ts | 0 .../trilium-core}/src/services/import/utils.spec.ts | 0 .../trilium-core}/src/services/import/utils.ts | 0 .../trilium-core}/src/services/import/zip.spec.ts | 0 .../trilium-core}/src/services/import/zip.ts | 0 27 files changed, 0 insertions(+), 0 deletions(-) rename {apps/server => packages/trilium-core}/src/services/import/common.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/enex.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/markdown.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/markdown.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/markdown/wikilink_internal_link.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/markdown/wikilink_transclusion.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/mime.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/mime.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/opml.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/IREN Reports Q2 FY25 Results.htm (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/IREN.Reports.Q2.FY25.Results_files.zip (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/New note.excalidraw (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/New note.mermaid (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/New note.mmd (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/Text Note.mdx (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/UTF-16LE Code Note.json (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/UTF-16LE Text Note.md (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/UTF-16LE Text Note.txt (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/geomap.zip (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/mdx.zip (100%) rename {apps/server => packages/trilium-core}/src/services/import/samples/silverbullet.zip (100%) rename {apps/server => packages/trilium-core}/src/services/import/single.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/single.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/utils.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/utils.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/zip.spec.ts (100%) rename {apps/server => packages/trilium-core}/src/services/import/zip.ts (100%) diff --git a/apps/server/src/services/import/common.ts b/packages/trilium-core/src/services/import/common.ts similarity index 100% rename from apps/server/src/services/import/common.ts rename to packages/trilium-core/src/services/import/common.ts diff --git a/apps/server/src/services/import/enex.ts b/packages/trilium-core/src/services/import/enex.ts similarity index 100% rename from apps/server/src/services/import/enex.ts rename to packages/trilium-core/src/services/import/enex.ts diff --git a/apps/server/src/services/import/markdown.spec.ts b/packages/trilium-core/src/services/import/markdown.spec.ts similarity index 100% rename from apps/server/src/services/import/markdown.spec.ts rename to packages/trilium-core/src/services/import/markdown.spec.ts diff --git a/apps/server/src/services/import/markdown.ts b/packages/trilium-core/src/services/import/markdown.ts similarity index 100% rename from apps/server/src/services/import/markdown.ts rename to packages/trilium-core/src/services/import/markdown.ts diff --git a/apps/server/src/services/import/markdown/wikilink_internal_link.ts b/packages/trilium-core/src/services/import/markdown/wikilink_internal_link.ts similarity index 100% rename from apps/server/src/services/import/markdown/wikilink_internal_link.ts rename to packages/trilium-core/src/services/import/markdown/wikilink_internal_link.ts diff --git a/apps/server/src/services/import/markdown/wikilink_transclusion.ts b/packages/trilium-core/src/services/import/markdown/wikilink_transclusion.ts similarity index 100% rename from apps/server/src/services/import/markdown/wikilink_transclusion.ts rename to packages/trilium-core/src/services/import/markdown/wikilink_transclusion.ts diff --git a/apps/server/src/services/import/mime.spec.ts b/packages/trilium-core/src/services/import/mime.spec.ts similarity index 100% rename from apps/server/src/services/import/mime.spec.ts rename to packages/trilium-core/src/services/import/mime.spec.ts diff --git a/apps/server/src/services/import/mime.ts b/packages/trilium-core/src/services/import/mime.ts similarity index 100% rename from apps/server/src/services/import/mime.ts rename to packages/trilium-core/src/services/import/mime.ts diff --git a/apps/server/src/services/import/opml.ts b/packages/trilium-core/src/services/import/opml.ts similarity index 100% rename from apps/server/src/services/import/opml.ts rename to packages/trilium-core/src/services/import/opml.ts diff --git a/apps/server/src/services/import/samples/IREN Reports Q2 FY25 Results.htm b/packages/trilium-core/src/services/import/samples/IREN Reports Q2 FY25 Results.htm similarity index 100% rename from apps/server/src/services/import/samples/IREN Reports Q2 FY25 Results.htm rename to packages/trilium-core/src/services/import/samples/IREN Reports Q2 FY25 Results.htm diff --git a/apps/server/src/services/import/samples/IREN.Reports.Q2.FY25.Results_files.zip b/packages/trilium-core/src/services/import/samples/IREN.Reports.Q2.FY25.Results_files.zip similarity index 100% rename from apps/server/src/services/import/samples/IREN.Reports.Q2.FY25.Results_files.zip rename to packages/trilium-core/src/services/import/samples/IREN.Reports.Q2.FY25.Results_files.zip diff --git a/apps/server/src/services/import/samples/New note.excalidraw b/packages/trilium-core/src/services/import/samples/New note.excalidraw similarity index 100% rename from apps/server/src/services/import/samples/New note.excalidraw rename to packages/trilium-core/src/services/import/samples/New note.excalidraw diff --git a/apps/server/src/services/import/samples/New note.mermaid b/packages/trilium-core/src/services/import/samples/New note.mermaid similarity index 100% rename from apps/server/src/services/import/samples/New note.mermaid rename to packages/trilium-core/src/services/import/samples/New note.mermaid diff --git a/apps/server/src/services/import/samples/New note.mmd b/packages/trilium-core/src/services/import/samples/New note.mmd similarity index 100% rename from apps/server/src/services/import/samples/New note.mmd rename to packages/trilium-core/src/services/import/samples/New note.mmd diff --git a/apps/server/src/services/import/samples/Text Note.mdx b/packages/trilium-core/src/services/import/samples/Text Note.mdx similarity index 100% rename from apps/server/src/services/import/samples/Text Note.mdx rename to packages/trilium-core/src/services/import/samples/Text Note.mdx diff --git a/apps/server/src/services/import/samples/UTF-16LE Code Note.json b/packages/trilium-core/src/services/import/samples/UTF-16LE Code Note.json similarity index 100% rename from apps/server/src/services/import/samples/UTF-16LE Code Note.json rename to packages/trilium-core/src/services/import/samples/UTF-16LE Code Note.json diff --git a/apps/server/src/services/import/samples/UTF-16LE Text Note.md b/packages/trilium-core/src/services/import/samples/UTF-16LE Text Note.md similarity index 100% rename from apps/server/src/services/import/samples/UTF-16LE Text Note.md rename to packages/trilium-core/src/services/import/samples/UTF-16LE Text Note.md diff --git a/apps/server/src/services/import/samples/UTF-16LE Text Note.txt b/packages/trilium-core/src/services/import/samples/UTF-16LE Text Note.txt similarity index 100% rename from apps/server/src/services/import/samples/UTF-16LE Text Note.txt rename to packages/trilium-core/src/services/import/samples/UTF-16LE Text Note.txt diff --git a/apps/server/src/services/import/samples/geomap.zip b/packages/trilium-core/src/services/import/samples/geomap.zip similarity index 100% rename from apps/server/src/services/import/samples/geomap.zip rename to packages/trilium-core/src/services/import/samples/geomap.zip diff --git a/apps/server/src/services/import/samples/mdx.zip b/packages/trilium-core/src/services/import/samples/mdx.zip similarity index 100% rename from apps/server/src/services/import/samples/mdx.zip rename to packages/trilium-core/src/services/import/samples/mdx.zip diff --git a/apps/server/src/services/import/samples/silverbullet.zip b/packages/trilium-core/src/services/import/samples/silverbullet.zip similarity index 100% rename from apps/server/src/services/import/samples/silverbullet.zip rename to packages/trilium-core/src/services/import/samples/silverbullet.zip diff --git a/apps/server/src/services/import/single.spec.ts b/packages/trilium-core/src/services/import/single.spec.ts similarity index 100% rename from apps/server/src/services/import/single.spec.ts rename to packages/trilium-core/src/services/import/single.spec.ts diff --git a/apps/server/src/services/import/single.ts b/packages/trilium-core/src/services/import/single.ts similarity index 100% rename from apps/server/src/services/import/single.ts rename to packages/trilium-core/src/services/import/single.ts diff --git a/apps/server/src/services/import/utils.spec.ts b/packages/trilium-core/src/services/import/utils.spec.ts similarity index 100% rename from apps/server/src/services/import/utils.spec.ts rename to packages/trilium-core/src/services/import/utils.spec.ts diff --git a/apps/server/src/services/import/utils.ts b/packages/trilium-core/src/services/import/utils.ts similarity index 100% rename from apps/server/src/services/import/utils.ts rename to packages/trilium-core/src/services/import/utils.ts diff --git a/apps/server/src/services/import/zip.spec.ts b/packages/trilium-core/src/services/import/zip.spec.ts similarity index 100% rename from apps/server/src/services/import/zip.spec.ts rename to packages/trilium-core/src/services/import/zip.spec.ts diff --git a/apps/server/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts similarity index 100% rename from apps/server/src/services/import/zip.ts rename to packages/trilium-core/src/services/import/zip.ts From ecb467f2b7e47f929baee398e8ebe4543c2bc365 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 11:40:48 +0200 Subject: [PATCH 02/18] chore(import): fix a few type errors --- apps/server/package.json | 4 +-- apps/server/src/services/utils.ts | 34 ++----------------- packages/trilium-core/package.json | 2 ++ packages/trilium-core/src/services/image.ts | 10 +++++- .../trilium-core/src/services/import/enex.ts | 26 ++++++++------ .../src/services/import/markdown.ts | 10 +++--- .../trilium-core/src/services/import/opml.ts | 6 ++-- .../src/services/import/single.spec.ts | 8 +++-- .../src/services/import/single.ts | 21 ++++++------ .../trilium-core/src/services/import/utils.ts | 4 +-- .../src/services/import/zip.spec.ts | 4 +-- .../trilium-core/src/services/import/zip.ts | 32 ++++++++--------- .../trilium-core/src/services/utils/binary.ts | 32 +++++++++++++++++ pnpm-lock.yaml | 16 ++++----- 14 files changed, 110 insertions(+), 99 deletions(-) diff --git a/apps/server/package.json b/apps/server/package.json index 243a987b41..f6fac1a4d8 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -68,7 +68,6 @@ "axios": "1.13.6", "bindings": "1.5.0", "bootstrap": "5.3.8", - "chardet": "2.1.1", "cheerio": "1.2.0", "chokidar": "5.0.0", "cls-hooked": "4.2.2", @@ -108,8 +107,7 @@ "safe-compare": "1.1.4", "sax": "1.6.0", "serve-favicon": "2.5.1", - "stream-throttle": "0.1.3", - "strip-bom": "5.0.0", + "stream-throttle": "0.1.3", "striptags": "3.2.0", "supertest": "7.2.2", "swagger-jsdoc": "6.2.8", diff --git a/apps/server/src/services/utils.ts b/apps/server/src/services/utils.ts index 7b4d6cd1d3..db7b4d38f9 100644 --- a/apps/server/src/services/utils.ts +++ b/apps/server/src/services/utils.ts @@ -1,9 +1,7 @@ -import { getCrypto,utils as coreUtils } from "@triliumnext/core"; -import chardet from "chardet"; +import { binary_utils,getCrypto, utils as coreUtils } from "@triliumnext/core"; import crypto from "crypto"; import { release as osRelease } from "os"; import path from "path"; -import stripBom from "strip-bom"; const osVersion = osRelease().split('.').map(Number); @@ -138,35 +136,6 @@ export function getResourceDir() { return path.join(__dirname, ".."); } -/** - * For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped. - * - * For strings, they are returned immediately without any transformation. - * - * For nullish values, an empty string is returned. - * - * @param data the string or buffer to process. - * @returns the string representation of the buffer, or the same string is it's a string. - */ -export function processStringOrBuffer(data: string | Buffer | null) { - if (!data) { - return ""; - } - - if (!Buffer.isBuffer(data)) { - return data; - } - - const detectedEncoding = chardet.detect(data); - switch (detectedEncoding) { - case "UTF-16LE": - return stripBom(data.toString("utf-16le")); - case "UTF-8": - default: - return data.toString("utf-8"); - } -} - /** @deprecated */ export const escapeHtml = coreUtils.escapeHtml; /** @deprecated */ @@ -183,6 +152,7 @@ export const isEmptyOrWhitespace = coreUtils.isEmptyOrWhitespace; export const normalizeUrl = coreUtils.normalizeUrl; export const timeLimit = coreUtils.timeLimit; export const sanitizeSqlIdentifier = coreUtils.sanitizeSqlIdentifier; +export const processStringOrBuffer = binary_utils.processStringOrBuffer; export function waitForStreamToFinish(stream: any): Promise { return new Promise((resolve, reject) => { diff --git a/packages/trilium-core/package.json b/packages/trilium-core/package.json index 2a1133b32e..3a79abdcb0 100644 --- a/packages/trilium-core/package.json +++ b/packages/trilium-core/package.json @@ -10,12 +10,14 @@ "@braintree/sanitize-url": "7.1.1", "@triliumnext/commons": "workspace:*", "async-mutex": "0.5.0", + "chardet": "2.1.1", "escape-html": "1.0.3", "i18next": "25.10.10", "mime-types": "3.0.2", "node-html-parser": "7.1.0", "sanitize-filename": "1.6.4", "sanitize-html": "2.17.2", + "strip-bom": "5.0.0", "unescape": "1.0.1" }, "devDependencies": { diff --git a/packages/trilium-core/src/services/image.ts b/packages/trilium-core/src/services/image.ts index 605c3cec21..7ccd1ff5ad 100644 --- a/packages/trilium-core/src/services/image.ts +++ b/packages/trilium-core/src/services/image.ts @@ -1,5 +1,5 @@ export default { - saveImageToAttachment(noteId: string, imageBuffer: Uint8Array, title: string, b1: boolean, b2: boolean) { + saveImageToAttachment(noteId: string, imageBuffer: Uint8Array, title: string, b1?: boolean, b2?: boolean) { console.warn("Image save ignored", noteId, title); return { @@ -10,5 +10,13 @@ export default { updateImage(noteId: string, imageBuffer: Uint8Array, title: string) { console.warn("Image update ignored", noteId, title); + }, + + saveImage(noteId: string, imageBuffer: Uint8Array, title: string, b1?: boolean, b2?: boolean) { + console.warn("Image save ignored", noteId, title); + + return { + note: null + }; } } diff --git a/packages/trilium-core/src/services/import/enex.ts b/packages/trilium-core/src/services/import/enex.ts index d50f6e4174..18ab4e888d 100644 --- a/packages/trilium-core/src/services/import/enex.ts +++ b/packages/trilium-core/src/services/import/enex.ts @@ -1,20 +1,22 @@ import type { AttributeType } from "@triliumnext/commons"; import { dayjs } from "@triliumnext/commons"; -import { sanitize, utils } from "@triliumnext/core"; import sax from "sax"; import stream from "stream"; import { Throttle } from "stream-throttle"; import type BNote from "../../becca/entities/bnote.js"; -import date_utils from "../date_utils.js"; +import date_utils from "../utils/date.js"; +import * as utils from "../utils/index.js"; import imageService from "../image.js"; -import log from "../log.js"; +import { getLog } from "../log.js"; import noteService from "../notes.js"; import protectedSessionService from "../protected_session.js"; -import sql from "../sql.js"; import type TaskContext from "../task_context.js"; -import { escapeHtml, fromBase64,md5 } from "../utils.js"; +import { escapeHtml, md5 } from "../utils/index.js"; +import { decodeBase64 } from "../utils/binary.js"; import type { File } from "./common.js"; +import { sanitizeHtml } from "../sanitizer.js"; +import { getSql } from "../sql/index.js"; /** * date format is e.g. 20181121T193703Z or 2013-04-14T16:19:00.000Z (Mac evernote, see #3496) @@ -38,7 +40,7 @@ interface Attribute { interface Resource { title: string; - content?: Buffer | string; + content?: Uint8Array | string; mime?: string; attributes: Attribute[]; } @@ -117,7 +119,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN "\u2611 " ); - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); return content; } @@ -138,7 +140,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN saxStream.on("error", (e) => { // unhandled errors will throw, since this is a proper node event emitter. - log.error(`error when parsing ENEX file: ${e}`); + getLog().error(`error when parsing ENEX file: ${e}`); // clear the error (saxStream._parser as any).error = null; saxStream._parser.resume(); @@ -235,6 +237,8 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } }); + const sql = getSql(); + function updateDates(note: BNote, utcDateCreated?: string, utcDateModified?: string) { // it's difficult to force custom dateCreated and dateModified to Note entity, so we do it post-creation with SQL const dateCreated = formatDateTimeToLocalDbFormat(utcDateCreated, false); @@ -295,7 +299,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } if (typeof resource.content === "string") { - resource.content = fromBase64(resource.content); + resource.content = decodeBase64(resource.content); } const hash = md5(resource.content); @@ -359,7 +363,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN content += imageLink; } } catch (e: any) { - log.error(`error when saving image from ENEX file: ${e.message}`); + getLog().error(`error when saving image from ENEX file: ${e.message}`); createFileNote(); } } else { @@ -367,7 +371,7 @@ function importEnex(taskContext: TaskContext<"importNotes">, file: File, parentN } } - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); // save updated content with links to files/images noteEntity.setContent(content); diff --git a/packages/trilium-core/src/services/import/markdown.ts b/packages/trilium-core/src/services/import/markdown.ts index dec4a68e7f..35bc7cf8d4 100644 --- a/packages/trilium-core/src/services/import/markdown.ts +++ b/packages/trilium-core/src/services/import/markdown.ts @@ -1,17 +1,15 @@ - - import { getMimeTypeFromMarkdownName, MIME_TYPE_AUTO } from "@triliumnext/commons"; import { normalizeMimeTypeForCKEditor } from "@triliumnext/commons"; -import { sanitize } from "@triliumnext/core"; import { parse, Renderer, type Tokens,use } from "marked"; import { ADMONITION_TYPE_MAPPINGS } from "../export/markdown.js"; -import utils from "../utils.js"; import wikiLinkInternalLink from "./markdown/wikilink_internal_link.js"; import wikiLinkTransclusion from "./markdown/wikilink_transclusion.js"; import importUtils from "./utils.js"; +import { escapeHtml } from "../utils/index.js"; +import { sanitizeHtml } from "../sanitizer.js"; -const escape = utils.escapeHtml; +const escape = escapeHtml; /** * Keep renderer code up to date with https://github.com/markedjs/marked/blob/master/src/Renderer.ts. @@ -151,7 +149,7 @@ function renderToHtml(content: string, title: string) { // h1 handling needs to come before sanitization html = importUtils.handleH1(html, title); - html = sanitize.sanitizeHtml(html); + html = sanitizeHtml(html); // Add a trailing semicolon to CSS styles. html = html.replaceAll(/(<(img|figure|col).*?style=".*?)"/g, "$1;\""); diff --git a/packages/trilium-core/src/services/import/opml.ts b/packages/trilium-core/src/services/import/opml.ts index d9a2ed8fbe..f44371e95e 100644 --- a/packages/trilium-core/src/services/import/opml.ts +++ b/packages/trilium-core/src/services/import/opml.ts @@ -1,12 +1,10 @@ - - -import { sanitize } from "@triliumnext/core"; import xml2js from "xml2js"; import type BNote from "../../becca/entities/bnote.js"; import noteService from "../../services/notes.js"; import protectedSessionService from "../protected_session.js"; import type TaskContext from "../task_context.js"; +import { sanitizeHtml } from "../sanitizer.js"; const parseString = xml2js.parseString; interface OpmlXml { @@ -65,7 +63,7 @@ async function importOpml(taskContext: TaskContext<"importNotes">, fileBuffer: s throw new Error(`Unrecognized OPML version ${opmlVersion}`); } - content = sanitize.sanitizeHtml(content || ""); + content = sanitizeHtml(content || ""); const { note } = noteService.createNewNote({ parentNoteId, diff --git a/packages/trilium-core/src/services/import/single.spec.ts b/packages/trilium-core/src/services/import/single.spec.ts index 866a0874d6..4720bff43e 100644 --- a/packages/trilium-core/src/services/import/single.spec.ts +++ b/packages/trilium-core/src/services/import/single.spec.ts @@ -6,10 +6,10 @@ import { dirname } from "path"; import becca from "../../becca/becca.js"; import BNote from "../../becca/entities/bnote.js"; import TaskContext from "../task_context.js"; -import cls from "../cls.js"; import sql_init from "../sql_init.js"; import single from "./single.js"; import stripBom from "strip-bom"; +import { getContext } from "../context.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); async function testImport(fileName: string, mimetype: string) { @@ -20,7 +20,7 @@ async function testImport(fileName: string, mimetype: string) { }); return new Promise<{ buffer: Buffer; importedNote: BNote }>((resolve, reject) => { - cls.init(async () => { + getContext().init(async () => { const rootNote = becca.getNote("root"); if (!rootNote) { reject("Missing root note."); @@ -36,6 +36,10 @@ async function testImport(fileName: string, mimetype: string) { }, rootNote as BNote ); + if (importedNote === null) { + reject("Import failed."); + return; + } resolve({ buffer, importedNote diff --git a/packages/trilium-core/src/services/import/single.ts b/packages/trilium-core/src/services/import/single.ts index 7fba5024a5..87cb07da64 100644 --- a/packages/trilium-core/src/services/import/single.ts +++ b/packages/trilium-core/src/services/import/single.ts @@ -1,16 +1,17 @@ import type { NoteType } from "@triliumnext/commons"; -import { sanitize, utils } from "@triliumnext/core"; import type BNote from "../../becca/entities/bnote.js"; import imageService from "../../services/image.js"; import noteService from "../../services/notes.js"; -import { processStringOrBuffer } from "../../services/utils.js"; import protectedSessionService from "../protected_session.js"; import type TaskContext from "../task_context.js"; import type { File } from "./common.js"; import markdownService from "./markdown.js"; import mimeService from "./mime.js"; import importUtils from "./utils.js"; +import { getNoteTitle } from "../utils/index.js"; +import { sanitizeHtml } from "../sanitizer.js"; +import { processStringOrBuffer } from "../utils/binary.js"; function importSingleFile(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { const mime = mimeService.getMime(file.originalname) || file.mimetype; @@ -57,7 +58,7 @@ function importFile(taskContext: TaskContext<"importNotes">, file: File, parentN const mime = mimeService.getMime(originalName) || file.mimetype; const { note } = noteService.createNewNote({ parentNoteId: parentNote.noteId, - title: utils.getNoteTitle(originalName, mime === "application/pdf", { mime }), + title: getNoteTitle(originalName, mime === "application/pdf", { mime }), content: file.buffer, isProtected: parentNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), type: "file", @@ -72,7 +73,7 @@ function importFile(taskContext: TaskContext<"importNotes">, file: File, parentN } function importCodeNote(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const content = processStringOrBuffer(file.buffer); const detectedMime = mimeService.getMime(file.originalname) || file.mimetype; const mime = mimeService.normalizeMimeType(detectedMime); @@ -97,7 +98,7 @@ function importCodeNote(taskContext: TaskContext<"importNotes">, file: File, par } function importCustomType(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote, type: NoteType, mime: string) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const content = processStringOrBuffer(file.buffer); const { note } = noteService.createNewNote({ @@ -115,7 +116,7 @@ function importCustomType(taskContext: TaskContext<"importNotes">, file: File, p } function importPlainText(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const plainTextContent = processStringOrBuffer(file.buffer); const htmlContent = convertTextToHtml(plainTextContent); @@ -150,13 +151,13 @@ function convertTextToHtml(text: string) { } function importMarkdown(taskContext: TaskContext<"importNotes">, file: File, parentNote: BNote) { - const title = utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); const markdownContent = processStringOrBuffer(file.buffer); let htmlContent = markdownService.renderToHtml(markdownContent, title); if (taskContext.data?.safeImport) { - htmlContent = sanitize.sanitizeHtml(htmlContent); + htmlContent = sanitizeHtml(htmlContent); } const { note } = noteService.createNewNote({ @@ -179,12 +180,12 @@ function importHtml(taskContext: TaskContext<"importNotes">, file: File, parentN // Try to get title from HTML first, fall back to filename // We do this before sanitization since that turns all

s into

const htmlTitle = importUtils.extractHtmlTitle(content); - const title = htmlTitle || utils.getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); + const title = htmlTitle || getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces); content = importUtils.handleH1(content, title); if (taskContext?.data?.safeImport) { - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); } const { note } = noteService.createNewNote({ diff --git a/packages/trilium-core/src/services/import/utils.ts b/packages/trilium-core/src/services/import/utils.ts index 110e521cc5..3c5a73d9a0 100644 --- a/packages/trilium-core/src/services/import/utils.ts +++ b/packages/trilium-core/src/services/import/utils.ts @@ -1,6 +1,4 @@ -"use strict"; - -import { unescapeHtml } from "../utils.js"; +import { unescapeHtml } from "../utils"; function handleH1(content: string, title: string) { let isFirstH1Handled = false; diff --git a/packages/trilium-core/src/services/import/zip.spec.ts b/packages/trilium-core/src/services/import/zip.spec.ts index ecc070c6a9..397bce0db1 100644 --- a/packages/trilium-core/src/services/import/zip.spec.ts +++ b/packages/trilium-core/src/services/import/zip.spec.ts @@ -7,9 +7,9 @@ import zip, { removeTriliumTags } from "./zip.js"; import becca from "../../becca/becca.js"; import BNote from "../../becca/entities/bnote.js"; import TaskContext from "../task_context.js"; -import cls from "../cls.js"; import sql_init from "../sql_init.js"; import { trimIndentation } from "@triliumnext/commons"; +import { getContext } from "../context.js"; const scriptDir = dirname(fileURLToPath(import.meta.url)); async function testImport(fileName: string) { @@ -19,7 +19,7 @@ async function testImport(fileName: string) { }); return new Promise<{ importedNote: BNote; rootNote: BNote }>((resolve, reject) => { - cls.init(async () => { + getContext().init(async () => { const rootNote = becca.getNote("root"); if (!rootNote) { expect(rootNote).toBeTruthy(); diff --git a/packages/trilium-core/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts index 29fce3430a..30aec07ada 100644 --- a/packages/trilium-core/src/services/import/zip.ts +++ b/packages/trilium-core/src/services/import/zip.ts @@ -1,7 +1,4 @@ - - import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons"; -import { sanitize, utils } from "@triliumnext/core"; import path from "path"; import type { Stream } from "stream"; import yauzl from "yauzl"; @@ -12,16 +9,17 @@ import BAttribute from "../../becca/entities/battribute.js"; import BBranch from "../../becca/entities/bbranch.js"; import type BNote from "../../becca/entities/bnote.js"; import attributeService from "../../services/attributes.js"; -import log from "../../services/log.js"; +import { getLog } from "../../services/log.js"; import noteService from "../../services/notes.js"; -import { newEntityId, processStringOrBuffer, unescapeHtml } from "../../services/utils.js"; -import type AttributeMeta from "../meta/attribute_meta.js"; -import type NoteMeta from "../meta/note_meta.js"; +import { getNoteTitle, newEntityId, removeFileExtension, unescapeHtml } from "../../services/utils/index.js"; +import { processStringOrBuffer } from "../../services/utils/binary.js"; import protectedSessionService from "../protected_session.js"; import type TaskContext from "../task_context.js"; import treeService from "../tree.js"; import markdownService from "./markdown.js"; import mimeService from "./mime.js"; +import { AttributeMeta, NoteMeta } from "../../meta.js"; +import { sanitizeHtml } from "../sanitizer.js"; interface MetaFile { files: NoteMeta[]; @@ -162,7 +160,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu // in case we lack metadata, we treat e.g. "Programming.html" and "Programming" as the same note // (one data file, the other directory for children) - const filePathNoExt = utils.removeFileExtension(filePath); + const filePathNoExt = removeFileExtension(filePath); if (filePathNoExt in createdPaths) { return createdPaths[filePathNoExt]; @@ -199,7 +197,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu } if (!attributeService.isAttributeType(attr.type)) { - log.error(`Unrecognized attribute type ${attr.type}`); + getLog().error(`Unrecognized attribute type ${attr.type}`); continue; } @@ -217,8 +215,8 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu } if (taskContext.data?.safeImport) { - attr.name = sanitize.sanitizeHtml(attr.name); - attr.value = sanitize.sanitizeHtml(attr.value); + attr.name = sanitizeHtml(attr.name); + attr.value = sanitizeHtml(attr.value); } attributes.push(attr); @@ -234,7 +232,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu return; } - const noteTitle = utils.getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta); + const noteTitle = getNoteTitle(filePath, !!taskContext.data?.replaceUnderscoresWithSpaces, noteMeta); const parentNoteId = getParentNoteId(filePath, parentNoteMeta); if (!parentNoteId) { @@ -318,7 +316,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu }); if (taskContext.data?.safeImport) { - content = sanitize.sanitizeHtml(content); + content = sanitizeHtml(content); } content = content.replace(/]*>/gis, ""); @@ -333,7 +331,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu try { url = decodeURIComponent(url).trim(); } catch (e: any) { - log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`); + getLog().error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`); return `src="${url}"`; } @@ -356,7 +354,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu try { url = decodeURIComponent(url).trim(); } catch (e: any) { - log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`); + getLog().error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`); return `href="${url}"`; } @@ -467,7 +465,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu content = processStringOrBuffer(content); } - const noteTitle = utils.getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); + const noteTitle = getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); content = processNoteContent(noteMeta, type, mime, content, noteTitle || "", filePath); @@ -613,7 +611,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Bu if (attr.type !== "relation" || attr.value in becca.notes) { new BAttribute(attr).save(); } else { - log.info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`); + getLog().info(`Relation not imported since the target note doesn't exist: ${JSON.stringify(attr)}`); } } diff --git a/packages/trilium-core/src/services/utils/binary.ts b/packages/trilium-core/src/services/utils/binary.ts index 34b2909b9a..dc31b46af0 100644 --- a/packages/trilium-core/src/services/utils/binary.ts +++ b/packages/trilium-core/src/services/utils/binary.ts @@ -1,3 +1,6 @@ +import chardet from "chardet"; +import stripBom from "strip-bom"; + const utf8Decoder = new TextDecoder("utf-8"); const utf8Encoder = new TextEncoder(); @@ -59,3 +62,32 @@ export function wrapStringOrBuffer(stringOrBuffer: string | Uint8Array) { return stringOrBuffer; } } + +/** + * For buffers, they are scanned for a supported encoding and decoded (UTF-8, UTF-16). In some cases, the BOM is also stripped. + * + * For strings, they are returned immediately without any transformation. + * + * For nullish values, an empty string is returned. + * + * @param data the string or buffer to process. + * @returns the string representation of the buffer, or the same string is it's a string. + */ +export function processStringOrBuffer(data: string | Buffer | null) { + if (!data) { + return ""; + } + + if (!Buffer.isBuffer(data)) { + return data; + } + + const detectedEncoding = chardet.detect(data); + switch (detectedEncoding) { + case "UTF-16LE": + return stripBom(data.toString("utf-16le")); + case "UTF-8": + default: + return data.toString("utf-8"); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8aa762e626..123ff9f72b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -878,9 +878,6 @@ importers: bootstrap: specifier: 5.3.8 version: 5.3.8(@popperjs/core@2.11.8) - chardet: - specifier: 2.1.1 - version: 2.1.1 cheerio: specifier: 1.2.0 version: 1.2.0 @@ -1001,9 +998,6 @@ importers: stream-throttle: specifier: 0.1.3 version: 0.1.3 - strip-bom: - specifier: 5.0.0 - version: 5.0.0 striptags: specifier: 3.2.0 version: 3.2.0 @@ -1707,6 +1701,9 @@ importers: async-mutex: specifier: 0.5.0 version: 0.5.0 + chardet: + specifier: 2.1.1 + version: 2.1.1 escape-html: specifier: 1.0.3 version: 1.0.3 @@ -1725,6 +1722,9 @@ importers: sanitize-html: specifier: 2.17.2 version: 2.17.2 + strip-bom: + specifier: 5.0.0 + version: 5.0.0 unescape: specifier: 1.0.1 version: 1.0.1 @@ -17625,6 +17625,8 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 + transitivePeerDependencies: + - supports-color '@ckeditor/ckeditor5-import-word@47.6.1': dependencies: @@ -17752,8 +17754,6 @@ snapshots: '@ckeditor/ckeditor5-utils': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 - transitivePeerDependencies: - - supports-color '@ckeditor/ckeditor5-merge-fields@47.6.1': dependencies: From 6ce1d31cebe7f22d719e0c9552a99d2f5351bb64 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 11:45:02 +0200 Subject: [PATCH 03/18] chore(import): integrate import route into core --- apps/server/src/routes/routes.ts | 3 --- apps/server/src/services/cls.ts | 7 +++---- .../trilium-core}/src/routes/api/import.ts | 13 +++++++------ packages/trilium-core/src/routes/index.ts | 4 ++++ packages/trilium-core/src/services/context.ts | 4 ++++ 5 files changed, 18 insertions(+), 13 deletions(-) rename {apps/server => packages/trilium-core}/src/routes/api/import.ts (94%) diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index c1ad909d61..3bc8bf3cb3 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -25,7 +25,6 @@ import etapiTokensApiRoutes from "./api/etapi_tokens.js"; import exportRoute from "./api/export.js"; import filesRoute from "./api/files.js"; import fontsRoute from "./api/fonts.js"; -import importRoute from "./api/import.js"; import loginApiRoute from "./api/login.js"; import metricsRoute from "./api/metrics.js"; import otherRoute from "./api/other.js"; @@ -130,8 +129,6 @@ function register(app: express.Application) { // route(GET, "/api/revisions/:revisionId/download", [auth.checkApiAuthOrElectron], revisionsApiRoute.downloadRevision); route(GET, "/api/branches/:branchId/export/:type/:format/:version/:taskId", [auth.checkApiAuthOrElectron], exportRoute.exportBranch); - asyncRoute(PST, "/api/notes/:parentNoteId/notes-import", [auth.checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], importRoute.importNotesToBranch, apiResultHandler); - route(PST, "/api/notes/:parentNoteId/attachments-import", [auth.checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], importRoute.importAttachmentsToNote, apiResultHandler); // :filename is not used by trilium, but instead used for "save as" to assign a human-readable filename diff --git a/apps/server/src/services/cls.ts b/apps/server/src/services/cls.ts index dd0930872c..314b16a62f 100644 --- a/apps/server/src/services/cls.ts +++ b/apps/server/src/services/cls.ts @@ -46,10 +46,6 @@ function putEntityChange(entityChange: EntityChange) { cls.putEntityChange(entityChange); } -function ignoreEntityChangeIds() { - cls.getContext().set("ignoreEntityChangeIds", true); -} - function get(key: string) { return cls.getContext().get(key); } @@ -62,7 +58,10 @@ function reset() { cls.getContext().reset(); } +/** @deprecated */ export const wrap = cls.wrap; +/** @deprecated */ +export const ignoreEntityChangeIds = cls.ignoreEntityChangeIds; export default { init, diff --git a/apps/server/src/routes/api/import.ts b/packages/trilium-core/src/routes/api/import.ts similarity index 94% rename from apps/server/src/routes/api/import.ts rename to packages/trilium-core/src/routes/api/import.ts index e6686a86f7..923520887f 100644 --- a/apps/server/src/routes/api/import.ts +++ b/packages/trilium-core/src/routes/api/import.ts @@ -1,17 +1,18 @@ -import { becca_loader, ValidationError } from "@triliumnext/core"; import type { Request } from "express"; import path from "path"; import becca from "../../becca/becca.js"; import type BNote from "../../becca/entities/bnote.js"; -import cls from "../../services/cls.js"; import enexImportService from "../../services/import/enex.js"; import opmlImportService from "../../services/import/opml.js"; import singleImportService from "../../services/import/single.js"; import zipImportService from "../../services/import/zip.js"; -import log from "../../services/log.js"; +import log, { getLog } from "../../services/log.js"; import TaskContext from "../../services/task_context.js"; -import { safeExtractMessageAndStackFromError } from "../../services/utils.js"; +import { safeExtractMessageAndStackFromError } from "../../services/utils/index.js"; +import * as cls from "../../services/context.js"; +import { ValidationError } from "../../errors.js"; +import becca_loader from "../../becca/becca_loader.js"; async function importNotesToBranch(req: Request<{ parentNoteId: string }>) { const { parentNoteId } = req.params; @@ -72,7 +73,7 @@ async function importNotesToBranch(req: Request<{ parentNoteId: string }>) { const message = `Import failed with following error: '${errMessage}'. More details might be in the logs.`; taskContext.reportError(message); - log.error(message + errStack); + getLog().error(message + errStack); return [500, message]; } @@ -126,7 +127,7 @@ function importAttachmentsToNote(req: Request<{ parentNoteId: string }>) { const message = `Import failed with following error: '${errMessage}'. More details might be in the logs.`; taskContext.reportError(message); - log.error(message + errStack); + getLog().error(message + errStack); return [500, message]; } diff --git a/packages/trilium-core/src/routes/index.ts b/packages/trilium-core/src/routes/index.ts index 7f2b526f92..8f1ae2de27 100644 --- a/packages/trilium-core/src/routes/index.ts +++ b/packages/trilium-core/src/routes/index.ts @@ -25,6 +25,7 @@ import similarNotesRoute from "./api/similar_notes"; import imageRoute from "./api/image"; import setupApiRoute from "./api/setup"; import filesRoute from "./api/files"; +import importRoute from "./api/import"; // TODO: Deduplicate with routes.ts const GET = "get", @@ -136,6 +137,9 @@ export function buildSharedApiRoutes({ route, asyncRoute, apiRoute, asyncApiRout route(PST, "/api/sync/queue-sector/:entityName/:sector", [checkApiAuth], syncApiRoute.queueSector, apiResultHandler); route(GET, "/api/sync/stats", [], syncApiRoute.getStats, apiResultHandler); + asyncRoute(PST, "/api/notes/:parentNoteId/notes-import", [checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], importRoute.importNotesToBranch, apiResultHandler); + route(PST, "/api/notes/:parentNoteId/attachments-import", [checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], importRoute.importAttachmentsToNote, apiResultHandler); + apiRoute(GET, "/api/quick-search/:searchString", searchRoute.quickSearch); apiRoute(GET, "/api/search-note/:noteId", searchRoute.searchFromNote); apiRoute(PST, "/api/search-and-execute-note/:noteId", searchRoute.searchAndExecute); diff --git a/packages/trilium-core/src/services/context.ts b/packages/trilium-core/src/services/context.ts index f8565c54de..ba54900912 100644 --- a/packages/trilium-core/src/services/context.ts +++ b/packages/trilium-core/src/services/context.ts @@ -77,3 +77,7 @@ export function getAndClearEntityChangeIds() { return entityChangeIds; } + +export function ignoreEntityChangeIds() { + getContext().set("ignoreEntityChangeIds", true); +} From ce1f7a427464ce675416991480e8124e2411992c Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 14:28:40 +0200 Subject: [PATCH 04/18] chore(scripts): deduplicate errors listing --- scripts/filter-tsc-output.mts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/filter-tsc-output.mts b/scripts/filter-tsc-output.mts index bdc9263e4b..4652a55873 100644 --- a/scripts/filter-tsc-output.mts +++ b/scripts/filter-tsc-output.mts @@ -26,14 +26,24 @@ const filtered = lines.filter( let errorIndex = 0; const numbered: string[] = []; +const seen = new Set(); +let skipContinuation = false; for (const line of filtered) { if (ERROR_LINE_PATTERN.test(line)) { + if (seen.has(line)) { + skipContinuation = true; + continue; + } + seen.add(line); + skipContinuation = false; errorIndex++; numbered.push(`[${errorIndex}] ${line}`); } else if (line.trim()) { // Continuation line (indented context for multi-line errors) - numbered.push(line); + if (!skipContinuation) { + numbered.push(line); + } } } From f1d7d34f1afeb98cf0e7d9000f19e60a10225bc3 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 14:28:48 +0200 Subject: [PATCH 05/18] chore(core): align tsconfig --- packages/trilium-core/tsconfig.lib.json | 1 + packages/trilium-core/tsconfig.spec.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/trilium-core/tsconfig.lib.json b/packages/trilium-core/tsconfig.lib.json index 3ae44d7205..e0625b3c80 100644 --- a/packages/trilium-core/tsconfig.lib.json +++ b/packages/trilium-core/tsconfig.lib.json @@ -3,6 +3,7 @@ "compilerOptions": { "module": "ESNext", "moduleResolution": "bundler", + "target": "ES2020", "rootDir": "src", "outDir": "dist", "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo", diff --git a/packages/trilium-core/tsconfig.spec.json b/packages/trilium-core/tsconfig.spec.json index 92448e8fa1..6026943caa 100644 --- a/packages/trilium-core/tsconfig.spec.json +++ b/packages/trilium-core/tsconfig.spec.json @@ -2,6 +2,9 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "outDir": "./out-tsc/vitest", + "module": "ESNext", + "moduleResolution": "bundler", + "target": "ES2020", "types": [ "vitest" ], From f81369d6437789fc0d86adc5cae34ae1385cab05 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 14:33:52 +0200 Subject: [PATCH 06/18] feat(core): support md5 hash --- apps/client-standalone/package.json | 1 + .../src/lightweight/crypto_provider.ts | 12 ++++++++++-- apps/server/src/crypto_provider.ts | 2 +- apps/server/src/services/utils.ts | 5 ----- .../trilium-core/src/services/encryption/crypto.ts | 2 +- packages/trilium-core/src/services/utils/index.ts | 5 +++++ pnpm-lock.yaml | 12 ++++++++++++ 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/apps/client-standalone/package.json b/apps/client-standalone/package.json index 9e72dc5357..dc06104c1f 100644 --- a/apps/client-standalone/package.json +++ b/apps/client-standalone/package.json @@ -46,6 +46,7 @@ "i18next-http-backend": "3.0.2", "jquery": "4.0.0", "jquery.fancytree": "2.38.5", + "js-md5": "0.8.3", "js-sha1": "0.7.0", "js-sha256": "0.11.1", "js-sha512": "0.9.0", diff --git a/apps/client-standalone/src/lightweight/crypto_provider.ts b/apps/client-standalone/src/lightweight/crypto_provider.ts index 0acfc1944c..0d1a6d8066 100644 --- a/apps/client-standalone/src/lightweight/crypto_provider.ts +++ b/apps/client-standalone/src/lightweight/crypto_provider.ts @@ -2,6 +2,7 @@ import type { CryptoProvider } from "@triliumnext/core"; import { sha1 } from "js-sha1"; import { sha256 } from "js-sha256"; import { sha512 } from "js-sha512"; +import { md5 } from "js-md5"; interface Cipher { update(data: Uint8Array): Uint8Array; @@ -15,11 +16,18 @@ const CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; */ export default class BrowserCryptoProvider implements CryptoProvider { - createHash(algorithm: "sha1" | "sha512", content: string | Uint8Array): Uint8Array { + createHash(algorithm: "md5" | "sha1" | "sha512", content: string | Uint8Array): Uint8Array { const data = typeof content === "string" ? content : new TextDecoder().decode(content); - const hexHash = algorithm === "sha1" ? sha1(data) : sha512(data); + let hexHash: string; + if (algorithm === "md5") { + hexHash = md5(data); + } else if (algorithm === "sha1") { + hexHash = sha1(data); + } else { + hexHash = sha512(data); + } // Convert hex string to Uint8Array const bytes = new Uint8Array(hexHash.length / 2); diff --git a/apps/server/src/crypto_provider.ts b/apps/server/src/crypto_provider.ts index c32f301797..5643b9ecf4 100644 --- a/apps/server/src/crypto_provider.ts +++ b/apps/server/src/crypto_provider.ts @@ -6,7 +6,7 @@ const randtoken = generator({ source: "crypto" }); export default class NodejsCryptoProvider implements CryptoProvider { - createHash(algorithm: "sha1", content: string | Uint8Array): Uint8Array { + createHash(algorithm: "md5" | "sha1" | "sha512", content: string | Uint8Array): Uint8Array { return crypto.createHash(algorithm).update(content).digest(); } diff --git a/apps/server/src/services/utils.ts b/apps/server/src/services/utils.ts index db7b4d38f9..789f8af3d3 100644 --- a/apps/server/src/services/utils.ts +++ b/apps/server/src/services/utils.ts @@ -25,10 +25,6 @@ export function randomString(length: number): string { return coreUtils.randomString(length); } -export function md5(content: crypto.BinaryLike) { - return crypto.createHash("md5").update(content).digest("hex"); -} - /** @deprecated */ export function hashedBlobId(content: string | Buffer) { return coreUtils.hashedBlobId(content); @@ -177,7 +173,6 @@ export default { isMac, isStringNote, isWindows, - md5, newEntityId, normalize, quoteRegex, diff --git a/packages/trilium-core/src/services/encryption/crypto.ts b/packages/trilium-core/src/services/encryption/crypto.ts index 4f747dd9c6..0f3c54dddb 100644 --- a/packages/trilium-core/src/services/encryption/crypto.ts +++ b/packages/trilium-core/src/services/encryption/crypto.ts @@ -5,7 +5,7 @@ interface Cipher { export interface CryptoProvider { - createHash(algorithm: "sha1" | "sha512", content: string | Uint8Array): Uint8Array; + createHash(algorithm: "md5" | "sha1" | "sha512", content: string | Uint8Array): Uint8Array; randomBytes(size: number): Uint8Array; randomString(length: number): string; createCipheriv(algorithm: "aes-128-cbc", key: Uint8Array, iv: Uint8Array): Cipher; diff --git a/packages/trilium-core/src/services/utils/index.ts b/packages/trilium-core/src/services/utils/index.ts index 0a73c6b8e3..ec64a693f5 100644 --- a/packages/trilium-core/src/services/utils/index.ts +++ b/packages/trilium-core/src/services/utils/index.ts @@ -20,6 +20,11 @@ export function hash(text: string) { return encodeBase64(getCrypto().createHash("sha1", text.normalize())); } +export function md5(content: string | Uint8Array) { + const bytes = getCrypto().createHash("md5", content); + return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join(""); +} + export function isStringNote(type: string | undefined, mime: string) { return (type && STRING_NOTE_TYPES.has(type)) || mime.startsWith("text/") || STRING_MIME_TYPES.has(mime); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 123ff9f72b..8b9940114e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -507,6 +507,9 @@ importers: jquery.fancytree: specifier: 2.38.5 version: 2.38.5(jquery@4.0.0) + js-md5: + specifier: 0.8.3 + version: 0.8.3 js-sha1: specifier: 0.7.0 version: 0.7.0 @@ -11173,6 +11176,9 @@ packages: resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} engines: {node: '>=0.10.0'} + js-md5@0.8.3: + resolution: {integrity: sha512-qR0HB5uP6wCuRMrWPTrkMaev7MJZwJuuw4fnwAzRgP4J4/F8RwtodOKpGp4XpqsLBFzzgqIO42efFAyz2Et6KQ==} + js-sha1@0.7.0: resolution: {integrity: sha512-oQZ1Mo7440BfLSv9TX87VNEyU52pXPVG19F9PL3gTgNt0tVxlZ8F4O6yze3CLuLx28TxotxvlyepCNaaV0ZjMw==} @@ -17650,6 +17656,8 @@ snapshots: '@ckeditor/ckeditor5-ui': 47.6.1 '@ckeditor/ckeditor5-utils': 47.6.1 ckeditor5: 47.6.1 + transitivePeerDependencies: + - supports-color '@ckeditor/ckeditor5-inspector@5.0.0': {} @@ -17660,6 +17668,8 @@ snapshots: '@ckeditor/ckeditor5-ui': 47.6.1 '@ckeditor/ckeditor5-utils': 47.6.1 ckeditor5: 47.6.1 + transitivePeerDependencies: + - supports-color '@ckeditor/ckeditor5-line-height@47.6.1': dependencies: @@ -29147,6 +29157,8 @@ snapshots: js-levenshtein@1.1.6: {} + js-md5@0.8.3: {} + js-sha1@0.7.0: {} js-sha256@0.11.1: {} From f069b41df6da1cfdf1c938534e0d210f10a24588 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 16:40:23 +0200 Subject: [PATCH 07/18] chore(standalone): upload middleware with error handling --- .../src/lightweight/browser_router.ts | 37 +++++++++++++++++-- .../src/lightweight/browser_routes.ts | 5 ++- apps/server/src/routes/routes.ts | 4 +- .../trilium-core/src/routes/api/import.ts | 11 ++++-- packages/trilium-core/src/routes/index.ts | 4 +- .../src/services/import/common.ts | 2 +- .../trilium-core/src/services/import/opml.ts | 2 +- 7 files changed, 54 insertions(+), 11 deletions(-) diff --git a/apps/client-standalone/src/lightweight/browser_router.ts b/apps/client-standalone/src/lightweight/browser_router.ts index caa698f0e5..a5bdfea79f 100644 --- a/apps/client-standalone/src/lightweight/browser_router.ts +++ b/apps/client-standalone/src/lightweight/browser_router.ts @@ -5,6 +5,12 @@ import { getContext, routes } from "@triliumnext/core"; +export interface UploadedFile { + originalname: string; + mimetype: string; + buffer: Uint8Array; +} + export interface BrowserRequest { method: string; url: string; @@ -13,6 +19,7 @@ export interface BrowserRequest { query: Record; headers?: Record; body?: unknown; + file?: UploadedFile; } export interface BrowserResponse { @@ -154,8 +161,9 @@ export class BrowserRouter { const query = parseQuery(url.search); const upperMethod = method.toUpperCase(); - // Parse JSON body if it's an ArrayBuffer and content-type suggests JSON + // Parse body based on content-type let parsedBody = body; + let uploadedFile: UploadedFile | undefined; if (body instanceof ArrayBuffer && headers) { const contentType = headers['content-type'] || headers['Content-Type'] || ''; if (contentType.includes('application/json')) { @@ -166,9 +174,31 @@ export class BrowserRouter { } } catch (e) { console.warn('[Router] Failed to parse JSON body:', e); - // Keep original body if JSON parsing fails parsedBody = body; } + } else if (contentType.includes('multipart/form-data')) { + try { + // Reconstruct a Response so we can use the native FormData parser + const response = new Response(body, { headers: { 'content-type': contentType } }); + const formData = await response.formData(); + const formFields: Record = {}; + for (const [key, value] of formData.entries()) { + if (typeof value === 'string') { + formFields[key] = value; + } else { + // File field (Blob) — multer uses the field name "upload" + const fileBuffer = new Uint8Array(await value.arrayBuffer()); + uploadedFile = { + originalname: value.name, + mimetype: value.type || 'application/octet-stream', + buffer: fileBuffer + }; + } + } + parsedBody = formFields; + } catch (e) { + console.warn('[Router] Failed to parse multipart body:', e); + } } } // Find matching route @@ -191,7 +221,8 @@ export class BrowserRouter { params, query, headers: headers ?? {}, - body: parsedBody + body: parsedBody, + file: uploadedFile }; try { diff --git a/apps/client-standalone/src/lightweight/browser_routes.ts b/apps/client-standalone/src/lightweight/browser_routes.ts index 8eed93c4ea..e6492f7462 100644 --- a/apps/client-standalone/src/lightweight/browser_routes.ts +++ b/apps/client-standalone/src/lightweight/browser_routes.ts @@ -35,6 +35,7 @@ function toExpressLikeReq(req: BrowserRequest) { body: req.body, headers: req.headers ?? {}, method: req.method, + file: req.file, get originalUrl() { return req.url; } }; } @@ -227,7 +228,9 @@ export function registerRoutes(router: BrowserRouter): void { checkApiAuthOrElectron: noopMiddleware, checkAppNotInitialized, checkCredentials: noopMiddleware, - loginRateLimiter: noopMiddleware + loginRateLimiter: noopMiddleware, + uploadMiddlewareWithErrorHandling: noopMiddleware, + csrfMiddleware: noopMiddleware }); apiRoute('get', '/bootstrap', bootstrapRoute); diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index 3bc8bf3cb3..b34a610957 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -88,7 +88,9 @@ function register(app: express.Application) { checkApiAuthOrElectron: auth.checkApiAuthOrElectron, checkAppNotInitialized: auth.checkAppNotInitialized, checkCredentials: auth.checkCredentials, - loginRateLimiter + loginRateLimiter, + uploadMiddlewareWithErrorHandling, + csrfMiddleware }); route(PUT, "/api/notes/:noteId/file", [auth.checkApiAuthOrElectron, uploadMiddlewareWithErrorHandling, csrfMiddleware], filesRoute.updateFile, apiResultHandler); diff --git a/packages/trilium-core/src/routes/api/import.ts b/packages/trilium-core/src/routes/api/import.ts index 923520887f..4d68e504f4 100644 --- a/packages/trilium-core/src/routes/api/import.ts +++ b/packages/trilium-core/src/routes/api/import.ts @@ -1,5 +1,10 @@ import type { Request } from "express"; import path from "path"; +import type { File } from "../../services/import/common.js"; + +interface ImportRequest

extends Request

{ + file?: File; +} import becca from "../../becca/becca.js"; import type BNote from "../../becca/entities/bnote.js"; @@ -14,7 +19,7 @@ import * as cls from "../../services/context.js"; import { ValidationError } from "../../errors.js"; import becca_loader from "../../becca/becca_loader.js"; -async function importNotesToBranch(req: Request<{ parentNoteId: string }>) { +async function importNotesToBranch(req: ImportRequest<{ parentNoteId: string }>) { const { parentNoteId } = req.params; const { taskId, last } = req.body; @@ -66,7 +71,7 @@ async function importNotesToBranch(req: Request<{ parentNoteId: string }>) { return importResult; } } else { - note = await singleImportService.importSingleFile(taskContext, file, parentNote); + note = singleImportService.importSingleFile(taskContext, file, parentNote); } } catch (e: unknown) { const [errMessage, errStack] = safeExtractMessageAndStackFromError(e); @@ -100,7 +105,7 @@ async function importNotesToBranch(req: Request<{ parentNoteId: string }>) { return note.getPojo(); } -function importAttachmentsToNote(req: Request<{ parentNoteId: string }>) { +function importAttachmentsToNote(req: ImportRequest<{ parentNoteId: string }>) { const { parentNoteId } = req.params; const { taskId, last } = req.body; diff --git a/packages/trilium-core/src/routes/index.ts b/packages/trilium-core/src/routes/index.ts index 8f1ae2de27..f5e9e3e6c7 100644 --- a/packages/trilium-core/src/routes/index.ts +++ b/packages/trilium-core/src/routes/index.ts @@ -45,9 +45,11 @@ interface SharedApiRoutesContext { checkAppNotInitialized: any; loginRateLimiter: any; checkCredentials: any; + uploadMiddlewareWithErrorHandling: any; + csrfMiddleware: any; } -export function buildSharedApiRoutes({ route, asyncRoute, apiRoute, asyncApiRoute, checkApiAuth, apiResultHandler, checkApiAuthOrElectron, checkAppNotInitialized, checkCredentials, loginRateLimiter }: SharedApiRoutesContext) { +export function buildSharedApiRoutes({ route, asyncRoute, apiRoute, asyncApiRoute, checkApiAuth, apiResultHandler, checkApiAuthOrElectron, checkAppNotInitialized, checkCredentials, loginRateLimiter, uploadMiddlewareWithErrorHandling, csrfMiddleware }: SharedApiRoutesContext) { apiRoute(GET, '/api/tree', treeApiRoute.getTree); apiRoute(PST, '/api/tree/load', treeApiRoute.load); diff --git a/packages/trilium-core/src/services/import/common.ts b/packages/trilium-core/src/services/import/common.ts index 72b5eff4df..114f529458 100644 --- a/packages/trilium-core/src/services/import/common.ts +++ b/packages/trilium-core/src/services/import/common.ts @@ -1,5 +1,5 @@ export interface File { originalname: string; mimetype: string; - buffer: string | Buffer; + buffer: string | Buffer | Uint8Array; } diff --git a/packages/trilium-core/src/services/import/opml.ts b/packages/trilium-core/src/services/import/opml.ts index f44371e95e..370d150f5c 100644 --- a/packages/trilium-core/src/services/import/opml.ts +++ b/packages/trilium-core/src/services/import/opml.ts @@ -27,7 +27,7 @@ interface OpmlOutline { outline: OpmlOutline[]; } -async function importOpml(taskContext: TaskContext<"importNotes">, fileBuffer: string | Buffer, parentNote: BNote) { +async function importOpml(taskContext: TaskContext<"importNotes">, fileBuffer: string | Uint8Array, parentNote: BNote) { const xml = await new Promise((resolve, reject) => { parseString(fileBuffer, (err: any, result: OpmlXml) => { if (err) { From 55f09fe21a6126cc82c645b3e5e18dd95fe146bb Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 16:45:44 +0200 Subject: [PATCH 08/18] chore(core): fix usage of Buffer --- packages/trilium-core/src/services/import/zip.ts | 6 +++--- packages/trilium-core/src/services/utils/binary.ts | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/trilium-core/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts index 30aec07ada..0aa29ab568 100644 --- a/packages/trilium-core/src/services/import/zip.ts +++ b/packages/trilium-core/src/services/import/zip.ts @@ -29,7 +29,7 @@ interface ImportZipOpts { preserveIds?: boolean; } -async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Buffer, importRootNote: BNote, opts?: ImportZipOpts): Promise { +async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Uint8Array, importRootNote: BNote, opts?: ImportZipOpts): Promise { /** maps from original noteId (in ZIP file) to newly generated noteId */ const noteIdMap: Record = {}; /** type maps from original attachmentId (in ZIP file) to newly generated attachmentId */ @@ -655,9 +655,9 @@ export function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise }); } -export function readZipFile(buffer: Buffer, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => Promise) { +export function readZipFile(buffer: Uint8Array, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => Promise) { return new Promise((res, rej) => { - yauzl.fromBuffer(buffer, { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => { + yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => { if (err) rej(err); if (!zipfile) throw new Error("Unable to read zip file."); diff --git a/packages/trilium-core/src/services/utils/binary.ts b/packages/trilium-core/src/services/utils/binary.ts index dc31b46af0..721119080f 100644 --- a/packages/trilium-core/src/services/utils/binary.ts +++ b/packages/trilium-core/src/services/utils/binary.ts @@ -73,21 +73,21 @@ export function wrapStringOrBuffer(stringOrBuffer: string | Uint8Array) { * @param data the string or buffer to process. * @returns the string representation of the buffer, or the same string is it's a string. */ -export function processStringOrBuffer(data: string | Buffer | null) { +export function processStringOrBuffer(data: string | Uint8Array | null) { if (!data) { return ""; } - if (!Buffer.isBuffer(data)) { + if (typeof data === "string") { return data; } const detectedEncoding = chardet.detect(data); switch (detectedEncoding) { case "UTF-16LE": - return stripBom(data.toString("utf-16le")); + return stripBom(new TextDecoder("utf-16le").decode(data)); case "UTF-8": default: - return data.toString("utf-8"); + return utf8Decoder.decode(data); } } From 050cdd0a857ee2b7613fbafb208f823b33ef0526 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 17:00:13 +0200 Subject: [PATCH 09/18] chore(core): add a few missing constants --- packages/trilium-core/src/services/export/markdown.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 packages/trilium-core/src/services/export/markdown.ts diff --git a/packages/trilium-core/src/services/export/markdown.ts b/packages/trilium-core/src/services/export/markdown.ts new file mode 100644 index 0000000000..5889033a5f --- /dev/null +++ b/packages/trilium-core/src/services/export/markdown.ts @@ -0,0 +1,10 @@ +// TODO: Move this to a dedicated file someday. +export const ADMONITION_TYPE_MAPPINGS: Record = { + note: "NOTE", + tip: "TIP", + important: "IMPORTANT", + caution: "CAUTION", + warning: "WARNING" +}; + +export const DEFAULT_ADMONITION_TYPE = ADMONITION_TYPE_MAPPINGS.note; From a0573c439b69e0c11f3cc783edddeea69f34bb0d Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 17:32:32 +0200 Subject: [PATCH 10/18] fix(core): extension lookup failing in standalone --- packages/trilium-core/src/routes/api/import.ts | 6 +++--- packages/trilium-core/src/routes/api/revisions.ts | 4 ++-- packages/trilium-core/src/services/import/mime.ts | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/trilium-core/src/routes/api/import.ts b/packages/trilium-core/src/routes/api/import.ts index 4d68e504f4..b3bcdff357 100644 --- a/packages/trilium-core/src/routes/api/import.ts +++ b/packages/trilium-core/src/routes/api/import.ts @@ -1,5 +1,4 @@ import type { Request } from "express"; -import path from "path"; import type { File } from "../../services/import/common.js"; interface ImportRequest

extends Request

{ @@ -12,12 +11,13 @@ import enexImportService from "../../services/import/enex.js"; import opmlImportService from "../../services/import/opml.js"; import singleImportService from "../../services/import/single.js"; import zipImportService from "../../services/import/zip.js"; -import log, { getLog } from "../../services/log.js"; +import { getLog } from "../../services/log.js"; import TaskContext from "../../services/task_context.js"; import { safeExtractMessageAndStackFromError } from "../../services/utils/index.js"; import * as cls from "../../services/context.js"; import { ValidationError } from "../../errors.js"; import becca_loader from "../../becca/becca_loader.js"; +import { extname } from "../../services/utils/path.js"; async function importNotesToBranch(req: ImportRequest<{ parentNoteId: string }>) { const { parentNoteId } = req.params; @@ -40,7 +40,7 @@ async function importNotesToBranch(req: ImportRequest<{ parentNoteId: string }>) const parentNote = becca.getNoteOrThrow(parentNoteId); - const extension = path.extname(file.originalname).toLowerCase(); + const extension = extname(file.originalname).toLowerCase(); // running all the event handlers on imported notes (and attributes) is slow // and may produce unintended consequences diff --git a/packages/trilium-core/src/routes/api/revisions.ts b/packages/trilium-core/src/routes/api/revisions.ts index 5620cb1400..22b62ffcba 100644 --- a/packages/trilium-core/src/routes/api/revisions.ts +++ b/packages/trilium-core/src/routes/api/revisions.ts @@ -1,6 +1,5 @@ import { EditedNotesResponse, RevisionItem, RevisionPojo } from "@triliumnext/commons"; import type { Request, Response } from "express"; -import path from "path"; import becca from "../../becca/becca.js"; import type BNote from "../../becca/entities/bnote.js"; @@ -10,6 +9,7 @@ import eraseService from "../../services/erase.js"; import { NotePojo } from "../../becca/becca-interface.js"; import { becca_service, binary_utils, cls, getSql } from "../../index.js"; import { formatDownloadTitle, getContentDisposition } from "../../services/utils/index.js"; +import { extname } from "../../services/utils/path.js"; interface NotePath { noteId: string; @@ -67,7 +67,7 @@ function getRevisionFilename(revision: BRevision) { throw new Error("Missing creation date for revision."); } - const extension = path.extname(filename); + const extension = extname(filename); const date = revision.dateCreated .substr(0, 19) .replace(" ", "_") diff --git a/packages/trilium-core/src/services/import/mime.ts b/packages/trilium-core/src/services/import/mime.ts index b25e98926d..9ce41324ce 100644 --- a/packages/trilium-core/src/services/import/mime.ts +++ b/packages/trilium-core/src/services/import/mime.ts @@ -1,8 +1,8 @@ "use strict"; -import mimeTypes from "mime-types"; -import path from "path"; +import { types as extToMime } from "mime-types"; import type { NoteType, TaskData } from "@triliumnext/commons"; +import { extname } from "../utils/path"; const CODE_MIME_TYPES = new Set([ "application/json", @@ -84,10 +84,10 @@ function getMime(fileName: string) { return "text/x-dockerfile"; } - const ext = path.extname(fileNameLc); + const ext = extname(fileNameLc); const mimeFromExt = EXTENSION_TO_MIME.get(ext); - return mimeFromExt || mimeTypes.lookup(fileNameLc); + return mimeFromExt || extToMime[ext.slice(1)] || false; } function getType(options: TaskData<"importNotes">, mime: string): NoteType { From 22c86cf3b575d0dedab9d42b9bdc3db4d4d5d2ad Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 18:11:59 +0200 Subject: [PATCH 11/18] feat(standalone): basic ZIP support --- apps/client-standalone/package.json | 1 + .../src/lightweight/zip_provider.ts | 27 +++++++ .../src/local-server-worker.ts | 5 ++ apps/desktop/src/main.ts | 2 + apps/server/spec/setup.ts | 2 + apps/server/src/main.ts | 2 + apps/server/src/zip_provider.ts | 46 +++++++++++ packages/trilium-core/src/index.ts | 5 +- .../trilium-core/src/services/import/zip.ts | 76 +++++-------------- .../src/services/import/zip_provider.ts | 25 ++++++ .../trilium-core/src/services/utils/path.ts | 9 +++ pnpm-lock.yaml | 5 ++ 12 files changed, 145 insertions(+), 60 deletions(-) create mode 100644 apps/client-standalone/src/lightweight/zip_provider.ts create mode 100644 apps/server/src/zip_provider.ts create mode 100644 packages/trilium-core/src/services/import/zip_provider.ts diff --git a/apps/client-standalone/package.json b/apps/client-standalone/package.json index dc06104c1f..40c5730612 100644 --- a/apps/client-standalone/package.json +++ b/apps/client-standalone/package.json @@ -40,6 +40,7 @@ "color": "5.0.3", "debounce": "3.0.0", "draggabilly": "3.0.0", + "fflate": "0.8.2", "force-graph": "1.51.2", "globals": "17.4.0", "i18next": "25.10.10", diff --git a/apps/client-standalone/src/lightweight/zip_provider.ts b/apps/client-standalone/src/lightweight/zip_provider.ts new file mode 100644 index 0000000000..47a1c20f0c --- /dev/null +++ b/apps/client-standalone/src/lightweight/zip_provider.ts @@ -0,0 +1,27 @@ +import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js"; +import { unzip } from "fflate"; + +export default class BrowserZipProvider implements ZipProvider { + readZipFile( + buffer: Uint8Array, + processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise + ): Promise { + return new Promise((res, rej) => { + unzip(buffer, async (err, files) => { + if (err) { rej(err); return; } + + try { + for (const [fileName, data] of Object.entries(files)) { + await processEntry( + { fileName }, + () => Promise.resolve(data) + ); + } + res(); + } catch (e) { + rej(e); + } + }); + }); + } +} diff --git a/apps/client-standalone/src/local-server-worker.ts b/apps/client-standalone/src/local-server-worker.ts index be86cf8391..6a31561117 100644 --- a/apps/client-standalone/src/local-server-worker.ts +++ b/apps/client-standalone/src/local-server-worker.ts @@ -55,6 +55,7 @@ let BrowserSqlProvider: typeof import('./lightweight/sql_provider').default; let WorkerMessagingProvider: typeof import('./lightweight/messaging_provider').default; let BrowserExecutionContext: typeof import('./lightweight/cls_provider').default; let BrowserCryptoProvider: typeof import('./lightweight/crypto_provider').default; +let BrowserZipProvider: typeof import('./lightweight/zip_provider').default; let FetchRequestProvider: typeof import('./lightweight/request_provider').default; let StandalonePlatformProvider: typeof import('./lightweight/platform_provider').default; let translationProvider: typeof import('./lightweight/translation_provider').default; @@ -82,6 +83,7 @@ async function loadModules(): Promise { messagingModule, clsModule, cryptoModule, + zipModule, requestModule, platformModule, translationModule, @@ -91,6 +93,7 @@ async function loadModules(): Promise { import('./lightweight/messaging_provider.js'), import('./lightweight/cls_provider.js'), import('./lightweight/crypto_provider.js'), + import('./lightweight/zip_provider.js'), import('./lightweight/request_provider.js'), import('./lightweight/platform_provider.js'), import('./lightweight/translation_provider.js'), @@ -101,6 +104,7 @@ async function loadModules(): Promise { WorkerMessagingProvider = messagingModule.default; BrowserExecutionContext = clsModule.default; BrowserCryptoProvider = cryptoModule.default; + BrowserZipProvider = zipModule.default; FetchRequestProvider = requestModule.default; StandalonePlatformProvider = platformModule.default; translationProvider = translationModule.default; @@ -152,6 +156,7 @@ async function initialize(): Promise { await coreModule.initializeCore({ executionContext: new BrowserExecutionContext(), crypto: new BrowserCryptoProvider(), + zip: new BrowserZipProvider(), messaging: messagingProvider!, request: new FetchRequestProvider(), platform: new StandalonePlatformProvider(queryString), diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index b8d7d33d19..22c20aa513 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,6 +1,7 @@ import { getLog, initializeCore, sql_init } from "@triliumnext/core"; import ClsHookedExecutionContext from "@triliumnext/server/src/cls_provider.js"; import NodejsCryptoProvider from "@triliumnext/server/src/crypto_provider.js"; +import NodejsZipProvider from "@triliumnext/server/src/zip_provider.js"; import dataDirs from "@triliumnext/server/src/services/data_dir.js"; import options from "@triliumnext/server/src/services/options.js"; import port from "@triliumnext/server/src/services/port.js"; @@ -133,6 +134,7 @@ async function main() { } }, crypto: new NodejsCryptoProvider(), + zip: new NodejsZipProvider(), request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), messaging: new WebSocketMessagingProvider(), diff --git a/apps/server/spec/setup.ts b/apps/server/spec/setup.ts index 437ff8a2f8..cd0ef855cc 100644 --- a/apps/server/spec/setup.ts +++ b/apps/server/spec/setup.ts @@ -4,6 +4,7 @@ import { join } from "path"; import { initializeCore } from "@triliumnext/core"; import ClsHookedExecutionContext from "../src/cls_provider.js"; import NodejsCryptoProvider from "../src/crypto_provider.js"; +import NodejsZipProvider from "../src/zip_provider.js"; import ServerPlatformProvider from "../src/platform_provider.js"; import BetterSqlite3Provider from "../src/sql_provider.js"; import { initializeTranslations } from "../src/services/i18n.js"; @@ -27,6 +28,7 @@ beforeAll(async () => { onTransactionRollback() {} }, crypto: new NodejsCryptoProvider(), + zip: new NodejsZipProvider(), executionContext: new ClsHookedExecutionContext(), schema: readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"), platform: new ServerPlatformProvider(), diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index c7bb8d3798..d09ef82ebd 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -10,6 +10,7 @@ import path from "path"; import ClsHookedExecutionContext from "./cls_provider.js"; import NodejsCryptoProvider from "./crypto_provider.js"; +import NodejsZipProvider from "./zip_provider.js"; import ServerPlatformProvider from "./platform_provider.js"; import dataDirs from "./services/data_dir.js"; import port from "./services/port.js"; @@ -51,6 +52,7 @@ async function startApplication() { } }, crypto: new NodejsCryptoProvider(), + zip: new NodejsZipProvider(), request: new NodeRequestProvider(), executionContext: new ClsHookedExecutionContext(), messaging: new WebSocketMessagingProvider(), diff --git a/apps/server/src/zip_provider.ts b/apps/server/src/zip_provider.ts new file mode 100644 index 0000000000..1f858e8de4 --- /dev/null +++ b/apps/server/src/zip_provider.ts @@ -0,0 +1,46 @@ +import type { ZipEntry, ZipProvider } from "@triliumnext/core/src/services/import/zip_provider.js"; +import type { Stream } from "stream"; +import yauzl from "yauzl"; + +function streamToBuffer(stream: Stream): Promise { + const chunks: Uint8Array[] = []; + stream.on("data", (chunk: Uint8Array) => chunks.push(chunk)); + return new Promise((res, rej) => { + stream.on("end", () => res(Buffer.concat(chunks))); + stream.on("error", rej); + }); +} + +export default class NodejsZipProvider implements ZipProvider { + readZipFile( + buffer: Uint8Array, + processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise + ): Promise { + return new Promise((res, rej) => { + yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => { + if (err) { rej(err); return; } + if (!zipfile) { rej(new Error("Unable to read zip file.")); return; } + + zipfile.readEntry(); + zipfile.on("entry", async (entry: yauzl.Entry) => { + try { + const readContent = () => new Promise((res, rej) => { + zipfile.openReadStream(entry, (err, readStream) => { + if (err) { rej(err); return; } + if (!readStream) { rej(new Error("Unable to read content.")); return; } + streamToBuffer(readStream).then(res, rej); + }); + }); + + await processEntry({ fileName: entry.fileName }, readContent); + } catch (e) { + rej(e); + } + zipfile.readEntry(); + }); + zipfile.on("end", res); + zipfile.on("error", rej); + }); + }); + } +} diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index df899145e8..c4fbcb0920 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -9,6 +9,7 @@ import { initTranslations, TranslationProvider } from "./services/i18n"; import { initSchema } from "./services/sql_init"; import appInfo from "./services/app_info"; import { type PlatformProvider, initPlatform } from "./services/platform"; +import { type ZipProvider, initZipProvider } from "./services/import/zip_provider"; export { getLog } from "./services/log"; export type * from "./services/sql/types"; @@ -102,10 +103,11 @@ export * as routeHelpers from "./routes/helpers"; export * as becca_easy_mocking from "./test/becca_easy_mocking"; export * as becca_mocking from "./test/becca_mocking"; -export async function initializeCore({ dbConfig, executionContext, crypto, translations, messaging, request, schema, extraAppInfo, platform }: { +export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform }: { dbConfig: SqlServiceParams, executionContext: ExecutionContext, crypto: CryptoProvider, + zip: ZipProvider, translations: TranslationProvider, platform: PlatformProvider, schema: string, @@ -120,6 +122,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, trans initLog(); await initTranslations(translations); initCrypto(crypto); + initZipProvider(zip); initContext(executionContext); initSql(new SqlService(dbConfig, getLog())); initSchema(schema); diff --git a/packages/trilium-core/src/services/import/zip.ts b/packages/trilium-core/src/services/import/zip.ts index 0aa29ab568..94df44ce32 100644 --- a/packages/trilium-core/src/services/import/zip.ts +++ b/packages/trilium-core/src/services/import/zip.ts @@ -1,7 +1,6 @@ import { ALLOWED_NOTE_TYPES, type NoteType } from "@triliumnext/commons"; -import path from "path"; -import type { Stream } from "stream"; -import yauzl from "yauzl"; +import { basename, dirname } from "../utils/path.js"; +import { getZipProvider } from "./zip_provider.js"; import becca from "../../becca/becca.js"; import BAttachment from "../../becca/entities/battachment.js"; @@ -138,7 +137,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui if (parentNoteMeta?.noteId) { parentNoteId = parentNoteMeta.isImportRoot ? importRootNote.noteId : getNewNoteId(parentNoteMeta.noteId); } else { - const parentPath = path.dirname(filePath); + const parentPath = dirname(filePath); if (parentPath === ".") { parentNoteId = importRootNote.noteId; @@ -267,10 +266,10 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui url = url.substr(2); } - absUrl = path.dirname(filePath); + absUrl = dirname(filePath); while (url.startsWith("../")) { - absUrl = path.dirname(absUrl); + absUrl = dirname(absUrl); url = url.substr(3); } @@ -342,9 +341,9 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui const target = getEntityIdFromRelativeUrl(url, filePath); if (target.attachmentId) { - return `src="api/attachments/${target.attachmentId}/image/${path.basename(url)}"`; + return `src="api/attachments/${target.attachmentId}/image/${basename(url)}"`; } else if (target.noteId) { - return `src="api/images/${target.noteId}/${path.basename(url)}"`; + return `src="api/images/${target.noteId}/${basename(url)}"`; } return match; @@ -390,7 +389,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui return content; } - function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Buffer, noteTitle: string, filePath: string) { + function processNoteContent(noteMeta: NoteMeta | undefined, type: string, mime: string, content: string | Uint8Array, noteTitle: string, filePath: string) { if ((noteMeta?.format === "markdown" || (!noteMeta && taskContext.data?.textImportedAsText && ["text/markdown", "text/x-markdown", "text/mdx"].includes(mime))) && typeof content === "string") { content = markdownService.renderToHtml(content, noteTitle); } @@ -412,7 +411,7 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui return content; } - function saveNote(filePath: string, content: string | Buffer) { + function saveNote(filePath: string, content: string | Uint8Array) { const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath); if (noteMeta?.noImport) { @@ -549,46 +548,42 @@ async function importZip(taskContext: TaskContext<"importNotes">, fileBuffer: Ui noteId, type: "label", name: "originalFileName", - value: path.basename(filePath) + value: basename(filePath) }); } } // we're running two passes in order to obtain critical information first (meta file and root) const topLevelItems = new Set(); - await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => { + const zipProvider = getZipProvider(); + + await zipProvider.readZipFile(fileBuffer, async (entry, readContent) => { const filePath = normalizeFilePath(entry.fileName); // make sure that the meta file is loaded before the rest of the files is processed. if (filePath === "!!!meta.json") { - const content = await readContent(zipfile, entry); - - metaFile = JSON.parse(content.toString("utf-8")); + const content = await readContent(); + metaFile = JSON.parse(new TextDecoder("utf-8").decode(content)); } // determine the root of the .zip (i.e. if it has only one top-level folder then the root is that folder, or the root of the archive if there are multiple top-level folders). const firstSlash = filePath.indexOf("/"); const topLevelPath = (firstSlash !== -1 ? filePath.substring(0, firstSlash) : filePath); topLevelItems.add(topLevelPath); - - zipfile.readEntry(); }); topLevelPath = (topLevelItems.size > 1 ? "" : topLevelItems.values().next().value ?? ""); - await readZipFile(fileBuffer, async (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => { + await zipProvider.readZipFile(fileBuffer, async (entry, readContent) => { const filePath = normalizeFilePath(entry.fileName); if (/\/$/.test(entry.fileName)) { saveDirectory(filePath); } else if (filePath !== "!!!meta.json") { - const content = await readContent(zipfile, entry); - - saveNote(filePath, content); + saveNote(filePath, await readContent()); } taskContext.increaseProgressCount(); - zipfile.readEntry(); }); for (const noteId of createdNoteIds) { @@ -637,43 +632,6 @@ function normalizeFilePath(filePath: string): string { return filePath; } -function streamToBuffer(stream: Stream): Promise { - const chunks: Uint8Array[] = []; - stream.on("data", (chunk) => chunks.push(chunk)); - - return new Promise((res, rej) => stream.on("end", () => res(Buffer.concat(chunks)))); -} - -export function readContent(zipfile: yauzl.ZipFile, entry: yauzl.Entry): Promise { - return new Promise((res, rej) => { - zipfile.openReadStream(entry, (err, readStream) => { - if (err) rej(err); - if (!readStream) throw new Error("Unable to read content."); - - streamToBuffer(readStream).then(res); - }); - }); -} - -export function readZipFile(buffer: Uint8Array, processEntryCallback: (zipfile: yauzl.ZipFile, entry: yauzl.Entry) => Promise) { - return new Promise((res, rej) => { - yauzl.fromBuffer(Buffer.from(buffer), { lazyEntries: true, validateEntrySizes: false }, (err, zipfile) => { - if (err) rej(err); - if (!zipfile) throw new Error("Unable to read zip file."); - - zipfile.readEntry(); - zipfile.on("entry", async (entry) => { - try { - await processEntryCallback(zipfile, entry); - } catch (e) { - rej(e); - } - }); - zipfile.on("end", res); - }); - }); -} - function resolveNoteType(type: string | undefined): NoteType { // BC for ZIPs created in Trilium 0.57 and older switch (type) { diff --git a/packages/trilium-core/src/services/import/zip_provider.ts b/packages/trilium-core/src/services/import/zip_provider.ts new file mode 100644 index 0000000000..7b6cc9d04e --- /dev/null +++ b/packages/trilium-core/src/services/import/zip_provider.ts @@ -0,0 +1,25 @@ +export interface ZipEntry { + fileName: string; +} + +export interface ZipProvider { + /** + * Iterates over every entry in a ZIP buffer, calling `processEntry` for each one. + * `readContent()` inside the callback reads the raw bytes of that entry on demand. + */ + readZipFile( + buffer: Uint8Array, + processEntry: (entry: ZipEntry, readContent: () => Promise) => Promise + ): Promise; +} + +let zipProvider: ZipProvider | null = null; + +export function initZipProvider(provider: ZipProvider) { + zipProvider = provider; +} + +export function getZipProvider(): ZipProvider { + if (!zipProvider) throw new Error("ZipProvider not initialized."); + return zipProvider; +} diff --git a/packages/trilium-core/src/services/utils/path.ts b/packages/trilium-core/src/services/utils/path.ts index d470b5dc16..da2497cdcd 100644 --- a/packages/trilium-core/src/services/utils/path.ts +++ b/packages/trilium-core/src/services/utils/path.ts @@ -16,3 +16,12 @@ export function basename(filePath: string): string { const lastSlash = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\")); return filePath.substring(lastSlash + 1); } + +/** Returns the directory part of a file path, or "." if there is none. */ +export function dirname(filePath: string): string { + const normalized = filePath.replace(/\\/g, "/"); + const lastSlash = normalized.lastIndexOf("/"); + if (lastSlash === -1) return "."; + if (lastSlash === 0) return "/"; + return normalized.substring(0, lastSlash); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8b9940114e..63bc0cd559 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -489,6 +489,9 @@ importers: draggabilly: specifier: 3.0.0 version: 3.0.0 + fflate: + specifier: 0.8.2 + version: 0.8.2 force-graph: specifier: 1.51.2 version: 1.51.2 @@ -17694,6 +17697,8 @@ snapshots: '@ckeditor/ckeditor5-widget': 47.6.1 ckeditor5: 47.6.1 es-toolkit: 1.39.5 + transitivePeerDependencies: + - supports-color '@ckeditor/ckeditor5-list-multi-level@47.6.1': dependencies: From 91d526b15f0033b8bfd63b36489fdcf72fa85c68 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 18:27:19 +0200 Subject: [PATCH 12/18] feat(standalone/import): improve importing speed --- .../src/lightweight/browser_routes.ts | 41 ++++++++++++++++++- .../src/lightweight/sql_provider.ts | 9 ++-- .../trilium-core/src/routes/api/import.ts | 14 +++---- packages/trilium-core/src/services/sql/sql.ts | 20 +++++++++ 4 files changed, 73 insertions(+), 11 deletions(-) diff --git a/apps/client-standalone/src/lightweight/browser_routes.ts b/apps/client-standalone/src/lightweight/browser_routes.ts index e6492f7462..739fae625e 100644 --- a/apps/client-standalone/src/lightweight/browser_routes.ts +++ b/apps/client-standalone/src/lightweight/browser_routes.ts @@ -122,6 +122,45 @@ function createRoute(router: BrowserRouter) { }; } +/** + * Async variant of createRoute for handlers that return Promises (e.g. import). + * Uses transactionalAsync (manual BEGIN/COMMIT/ROLLBACK) instead of the synchronous + * transactional() wrapper, which would commit an empty transaction immediately when + * passed an async callback. + */ +function createAsyncRoute(router: BrowserRouter) { + return (method: HttpMethod, path: string, _middleware: any[], handler: (req: any, res: any) => Promise, resultHandler?: ((req: any, res: any, result: unknown) => unknown) | null) => { + router.register(method, path, (req: BrowserRequest) => { + return getContext().init(async () => { + setContextFromHeaders(req); + const expressLikeReq = toExpressLikeReq(req); + const mockRes = createMockExpressResponse(); + const result = await getSql().transactionalAsync(() => handler(expressLikeReq, mockRes)); + + // If the handler used the mock response (e.g. image routes that call res.send()), + // return it as a raw response so BrowserRouter doesn't JSON-serialize it. + if (mockRes._used) { + return { + [RAW_RESPONSE]: true as const, + status: mockRes._status, + headers: mockRes._headers, + body: mockRes._body + }; + } + + if (resultHandler) { + // Create a minimal response object that captures what apiResultHandler sets. + const res = createResultHandlerResponse(); + resultHandler(expressLikeReq, res, result); + return res.result; + } + + return result; + }); + }); + }; +} + /** * Creates a mock Express response object that captures calls to set(), send(), sendStatus(), etc. * Used for route handlers (like image routes) that write directly to the response. @@ -220,7 +259,7 @@ export function registerRoutes(router: BrowserRouter): void { const apiRoute = createApiRoute(router, true); routes.buildSharedApiRoutes({ route: createRoute(router), - asyncRoute: createRoute(router), + asyncRoute: createAsyncRoute(router), apiRoute, asyncApiRoute: createApiRoute(router, false), apiResultHandler, diff --git a/apps/client-standalone/src/lightweight/sql_provider.ts b/apps/client-standalone/src/lightweight/sql_provider.ts index e7bd80ba34..4b22833de4 100644 --- a/apps/client-standalone/src/lightweight/sql_provider.ts +++ b/apps/client-standalone/src/lightweight/sql_provider.ts @@ -501,9 +501,12 @@ export default class BrowserSqlProvider implements DatabaseProvider { // Helper function to execute within a transaction const executeTransaction = (beginStatement: string, ...args: unknown[]): T => { - // If we're already in a transaction, use SAVEPOINTs for nesting - // This mimics better-sqlite3's behavior - if (self._inTransaction) { + // If we're already in a transaction (either tracked via JS flag or via actual SQLite + // autocommit state), use SAVEPOINTs for nesting — this handles the case where a manual + // BEGIN was issued directly (e.g. transactionalAsync) without going through transaction(). + const sqliteInTransaction = self.db?.pointer !== undefined + && (self.sqlite3!.capi as any).sqlite3_get_autocommit(self.db!.pointer) === 0; + if (self._inTransaction || sqliteInTransaction) { const savepointName = `sp_${++savepointCounter}_${Date.now()}`; self.db!.exec(`SAVEPOINT ${savepointName}`); try { diff --git a/packages/trilium-core/src/routes/api/import.ts b/packages/trilium-core/src/routes/api/import.ts index b3bcdff357..ea78365928 100644 --- a/packages/trilium-core/src/routes/api/import.ts +++ b/packages/trilium-core/src/routes/api/import.ts @@ -7,7 +7,7 @@ interface ImportRequest

extends Request

{ import becca from "../../becca/becca.js"; import type BNote from "../../becca/entities/bnote.js"; -import enexImportService from "../../services/import/enex.js"; +// import enexImportService from "../../services/import/enex.js"; import opmlImportService from "../../services/import/opml.js"; import singleImportService from "../../services/import/single.js"; import zipImportService from "../../services/import/zip.js"; @@ -64,12 +64,12 @@ async function importNotesToBranch(req: ImportRequest<{ parentNoteId: string }>) return importResult; } } else if (extension === ".enex" && options.explodeArchives) { - const importResult = await enexImportService.importEnex(taskContext, file, parentNote); - if (!Array.isArray(importResult)) { - note = importResult; - } else { - return importResult; - } + // const importResult = await enexImportService.importEnex(taskContext, file, parentNote); + // if (!Array.isArray(importResult)) { + // note = importResult; + // } else { + // return importResult; + // } } else { note = singleImportService.importSingleFile(taskContext, file, parentNote); } diff --git a/packages/trilium-core/src/services/sql/sql.ts b/packages/trilium-core/src/services/sql/sql.ts index d878936f2c..90631c5bfd 100644 --- a/packages/trilium-core/src/services/sql/sql.ts +++ b/packages/trilium-core/src/services/sql/sql.ts @@ -312,6 +312,26 @@ export class SqlService { } } + /** + * Async-safe transaction wrapper for use in Web Workers and other single-threaded async contexts. + * Uses manual BEGIN/COMMIT/ROLLBACK because the synchronous `transactional()` cannot await promises. + */ + async transactionalAsync(func: () => Promise): Promise { + this.execute("BEGIN IMMEDIATE"); + try { + const result = await func(); + this.execute("COMMIT"); + if (!this.dbConnection.inTransaction) { + this.params.onTransactionCommit(); + } + return result; + } catch (e) { + this.execute("ROLLBACK"); + this.params.onTransactionRollback(); + throw e; + } + } + fillParamList(paramIds: string[] | Set, truncate = true) { if ("length" in paramIds && paramIds.length === 0) { return; From d29d1428ed94ef2bdd26bb109b8fda3d23186f78 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 18:55:18 +0200 Subject: [PATCH 13/18] feat(standalone/import): import demo DB --- .../src/local-server-worker.ts | 5 +++++ apps/desktop/src/main.ts | 1 + apps/server/src/main.ts | 1 + packages/trilium-core/src/index.ts | 8 +++++-- .../trilium-core/src/services/sql_init.ts | 21 ++++++++++++------- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/apps/client-standalone/src/local-server-worker.ts b/apps/client-standalone/src/local-server-worker.ts index 6a31561117..6b54a947a5 100644 --- a/apps/client-standalone/src/local-server-worker.ts +++ b/apps/client-standalone/src/local-server-worker.ts @@ -162,6 +162,11 @@ async function initialize(): Promise { platform: new StandalonePlatformProvider(queryString), translations: translationProvider, schema: schemaModule.default, + getDemoArchive: async () => { + const response = await fetch("/server-assets/db/demo.zip"); + if (!response.ok) return null; + return new Uint8Array(await response.arrayBuffer()); + }, dbConfig: { provider: sqlProvider!, isReadOnly: false, diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 22c20aa513..8aef99bae6 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -141,6 +141,7 @@ async function main() { schema: fs.readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"), platform: new DesktopPlatformProvider(), translations: (await import("@triliumnext/server/src/services/i18n.js")).initializeTranslations, + getDemoArchive: async () => fs.readFileSync(require.resolve("@triliumnext/server/src/assets/db/demo.zip")), extraAppInfo: { nodeVersion: process.version, dataDirectory: path.resolve(dataDirs.TRILIUM_DATA_DIR) diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index d09ef82ebd..5a5c58c150 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -59,6 +59,7 @@ async function startApplication() { schema: fs.readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"), platform: new ServerPlatformProvider(), translations: (await import("./services/i18n.js")).initializeTranslations, + getDemoArchive: async () => fs.readFileSync(require.resolve("@triliumnext/server/src/assets/db/demo.zip")), extraAppInfo: { nodeVersion: process.version, dataDirectory: path.resolve(dataDirs.TRILIUM_DATA_DIR) diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index c4fbcb0920..5625bb69f6 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -6,7 +6,7 @@ import { SqlService, SqlServiceParams } from "./services/sql/sql"; import { initMessaging, MessagingProvider } from "./services/messaging/index"; import { initRequest, RequestProvider } from "./services/request"; import { initTranslations, TranslationProvider } from "./services/i18n"; -import { initSchema } from "./services/sql_init"; +import { initSchema, initDemoArchive } from "./services/sql_init"; import appInfo from "./services/app_info"; import { type PlatformProvider, initPlatform } from "./services/platform"; import { type ZipProvider, initZipProvider } from "./services/import/zip_provider"; @@ -103,7 +103,7 @@ export * as routeHelpers from "./routes/helpers"; export * as becca_easy_mocking from "./test/becca_easy_mocking"; export * as becca_mocking from "./test/becca_mocking"; -export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform }: { +export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform, getDemoArchive }: { dbConfig: SqlServiceParams, executionContext: ExecutionContext, crypto: CryptoProvider, @@ -113,6 +113,7 @@ export async function initializeCore({ dbConfig, executionContext, crypto, zip, schema: string, messaging?: MessagingProvider, request?: RequestProvider, + getDemoArchive?: () => Promise, extraAppInfo?: { nodeVersion: string; dataDirectory: string; @@ -126,6 +127,9 @@ export async function initializeCore({ dbConfig, executionContext, crypto, zip, initContext(executionContext); initSql(new SqlService(dbConfig, getLog())); initSchema(schema); + if (getDemoArchive) { + initDemoArchive(getDemoArchive); + } Object.assign(appInfo, extraAppInfo); if (messaging) { initMessaging(messaging); diff --git a/packages/trilium-core/src/services/sql_init.ts b/packages/trilium-core/src/services/sql_init.ts index b5d8a88fda..d53cac5716 100644 --- a/packages/trilium-core/src/services/sql_init.ts +++ b/packages/trilium-core/src/services/sql_init.ts @@ -15,11 +15,16 @@ import migrationService from "./migration"; export const dbReady = deferred(); let schema: string; +let getDemoArchive: (() => Promise) | null = null; export function initSchema(schemaStr: string) { schema = schemaStr; } +export function initDemoArchive(fn: () => Promise) { + getDemoArchive = fn; +} + function schemaExists() { return !!getSql().getValue(/*sql*/`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'options'`); @@ -177,13 +182,15 @@ async function createInitialDatabase(skipDemoDb?: boolean) { }); // Import demo content. - log.info("Importing demo content..."); - - const dummyTaskContext = new TaskContext("no-progress-reporting", "importNotes", null); - - // if (demoFile) { - // await zipImportService.importZip(dummyTaskContext, demoFile, rootNote); - // } + if (getDemoArchive) { + log.info("Importing demo content..."); + const demoFile = await getDemoArchive(); + if (demoFile) { + const { default: zipImportService } = await import("./import/zip.js"); + const dummyTaskContext = new TaskContext("no-progress-reporting", "importNotes", null); + await zipImportService.importZip(dummyTaskContext, demoFile, rootNote); + } + } // Post-demo. sql.transactional(() => { From 24650edd62600af7e7bc041bb8edd05bf9703b6c Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 19:03:39 +0200 Subject: [PATCH 14/18] fix(setup): demo DB not respected --- packages/trilium-core/src/routes/api/setup.ts | 2 +- packages/trilium-core/src/services/sql_init.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/trilium-core/src/routes/api/setup.ts b/packages/trilium-core/src/routes/api/setup.ts index 64f440e5e8..09a08189f2 100644 --- a/packages/trilium-core/src/routes/api/setup.ts +++ b/packages/trilium-core/src/routes/api/setup.ts @@ -15,7 +15,7 @@ function getStatus() { async function setupNewDocument(req: Request) { const { skipDemoDb } = req.query; - await sqlInit.createInitialDatabase(!!skipDemoDb); + await sqlInit.createInitialDatabase(skipDemoDb !== undefined); } function setupSyncFromServer(req: Request): Promise { diff --git a/packages/trilium-core/src/services/sql_init.ts b/packages/trilium-core/src/services/sql_init.ts index d53cac5716..758549f144 100644 --- a/packages/trilium-core/src/services/sql_init.ts +++ b/packages/trilium-core/src/services/sql_init.ts @@ -182,7 +182,7 @@ async function createInitialDatabase(skipDemoDb?: boolean) { }); // Import demo content. - if (getDemoArchive) { + if (!skipDemoDb && getDemoArchive) { log.info("Importing demo content..."); const demoFile = await getDemoArchive(); if (demoFile) { From 5b3fb315d771b4ec66604194480a3c1955e81edb Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 19:16:23 +0200 Subject: [PATCH 15/18] fix(core): on new database, opening hidden notes instead of the root --- packages/trilium-core/src/services/sql_init.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/trilium-core/src/services/sql_init.ts b/packages/trilium-core/src/services/sql_init.ts index 758549f144..29654cf97c 100644 --- a/packages/trilium-core/src/services/sql_init.ts +++ b/packages/trilium-core/src/services/sql_init.ts @@ -192,13 +192,13 @@ async function createInitialDatabase(skipDemoDb?: boolean) { } } - // Post-demo. + // Post-demo: pick the first visible (non-system) child of root as the start note. + // System notes have IDs starting with "_" and should not be navigated to on startup. + // Falls back to "root" if no visible child exists (e.g. empty database). sql.transactional(() => { - // this needs to happen after ZIP import, - // the previous solution was to move option initialization here, but then the important parts of initialization - // are not all in one transaction (because ZIP import is async and thus not transactional) - - const startNoteId = sql.getValue("SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 ORDER BY notePosition"); + const startNoteId = sql.getValue( + "SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 AND substr(noteId, 1, 1) != '_' ORDER BY notePosition" + ) ?? "root"; optionService.setOption( "openNoteContexts", From 2d4027c21498f4c640003a0d1b7e23a1dbc895cc Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 19:18:31 +0200 Subject: [PATCH 16/18] fix(server): depending on unexported zip import service --- apps/server/src/etapi/notes.ts | 3 +-- packages/trilium-core/src/index.ts | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/server/src/etapi/notes.ts b/apps/server/src/etapi/notes.ts index a0d3cf49e1..8c5e7d35b8 100644 --- a/apps/server/src/etapi/notes.ts +++ b/apps/server/src/etapi/notes.ts @@ -1,11 +1,10 @@ -import { NoteParams, SearchParams } from "@triliumnext/core"; +import { NoteParams, SearchParams, zipImportService } from "@triliumnext/core"; import type { Request, Router } from "express"; import type { ParsedQs } from "qs"; import becca from "../becca/becca.js"; import zipExportService from "../services/export/zip.js"; import type { ExportFormat } from "../services/export/zip/abstract_provider.js"; -import zipImportService from "../services/import/zip.js"; import noteService from "../services/notes.js"; import SearchContext from "../services/search/search_context.js"; import searchService from "../services/search/services/search.js"; diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index 5625bb69f6..037291f494 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -100,6 +100,8 @@ export type { RequestProvider, ExecOpts, CookieJar } from "./services/request"; export type * from "./meta"; export * as routeHelpers from "./routes/helpers"; +export { default as zipImportService } from "./services/import/zip"; + export * as becca_easy_mocking from "./test/becca_easy_mocking"; export * as becca_mocking from "./test/becca_mocking"; From 0211535f734a9a58244cc15b4a7d13e74d4948f0 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 19:26:27 +0200 Subject: [PATCH 17/18] fix(edit-docs): missing zip primitives --- apps/edit-docs/src/utils.ts | 15 +++++++-------- packages/trilium-core/src/index.ts | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/edit-docs/src/utils.ts b/apps/edit-docs/src/utils.ts index 3b41a70fd5..833028dd77 100644 --- a/apps/edit-docs/src/utils.ts +++ b/apps/edit-docs/src/utils.ts @@ -62,7 +62,7 @@ export function startElectron(callback: () => void): DeferredPromise { export async function importData(path: string) { const buffer = await createImportZip(path); - const importService = (await import("@triliumnext/server/src/services/import/zip.js")).default; + const { zipImportService } = (await import("@triliumnext/core")); const context = new TaskContext("no-progress-reporting", "importNotes", null); const becca = (await import("@triliumnext/server/src/becca/becca.js")).default; @@ -70,7 +70,7 @@ export async function importData(path: string) { if (!rootNote) { throw new Error("Missing root note for import."); } - await importService.importZip(context, buffer, rootNote, { + await zipImportService.importZip(context, buffer, rootNote, { preserveIds: true }); } @@ -106,19 +106,18 @@ function waitForEnd(archive: Archiver, stream: WriteStream) { export async function extractZip(zipFilePath: string, outputPath: string, ignoredFiles?: Set) { const promise = deferred(); setTimeout(async () => { - // Then extract the zip. - const { readZipFile, readContent } = (await import("@triliumnext/server/src/services/import/zip.js")); - await readZipFile(await fs.readFile(zipFilePath), async (zip, entry) => { + const { getZipProvider } = (await import("@triliumnext/core")); + const zipProvider = getZipProvider(); + const buffer = await fs.readFile(zipFilePath); + await zipProvider.readZipFile(buffer, async (entry, readContent) => { // We ignore directories since they can appear out of order anyway. if (!entry.fileName.endsWith("/") && !ignoredFiles?.has(entry.fileName)) { const destPath = path.join(outputPath, entry.fileName); - const fileContent = await readContent(zip, entry); + const fileContent = await readContent(); await fsExtra.mkdirs(path.dirname(destPath)); await fs.writeFile(destPath, fileContent); } - - zip.readEntry(); }); promise.resolve(); }, 1000); diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index 037291f494..c9f01f127b 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -100,6 +100,7 @@ export type { RequestProvider, ExecOpts, CookieJar } from "./services/request"; export type * from "./meta"; export * as routeHelpers from "./routes/helpers"; +export { getZipProvider, type ZipProvider } from "./services/import/zip_provider"; export { default as zipImportService } from "./services/import/zip"; export * as becca_easy_mocking from "./test/becca_easy_mocking"; From 6c75df70e0e59e865daf880710105b548ea2f3ec Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Fri, 27 Mar 2026 21:40:19 +0200 Subject: [PATCH 18/18] chore: solve type errors --- apps/server/src/routes/api/other.ts | 6 +++--- packages/trilium-core/src/index.ts | 3 +++ packages/trilium-core/src/routes/api/import.ts | 5 ++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/server/src/routes/api/other.ts b/apps/server/src/routes/api/other.ts index 0b8d852493..3481c1647b 100644 --- a/apps/server/src/routes/api/other.ts +++ b/apps/server/src/routes/api/other.ts @@ -1,8 +1,8 @@ +import { RenderMarkdownResponse, ToMarkdownResponse } from "@triliumnext/commons"; +import { markdownImportService } from "@triliumnext/core"; import type { Request } from "express"; -import markdownService from "../../services/import/markdown.js"; import markdown from "../../services/export/markdown.js"; -import { RenderMarkdownResponse, ToMarkdownResponse } from "@triliumnext/commons"; function renderMarkdown(req: Request) { const { markdownContent } = req.body; @@ -10,7 +10,7 @@ function renderMarkdown(req: Request) { throw new Error('markdownContent parameter is required and must be a string'); } return { - htmlContent: markdownService.renderToHtml(markdownContent, "") + htmlContent: markdownImportService.renderToHtml(markdownContent, "") } satisfies RenderMarkdownResponse; } diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.ts index c9f01f127b..d1f713884e 100644 --- a/packages/trilium-core/src/index.ts +++ b/packages/trilium-core/src/index.ts @@ -10,6 +10,7 @@ import { initSchema, initDemoArchive } from "./services/sql_init"; import appInfo from "./services/app_info"; import { type PlatformProvider, initPlatform } from "./services/platform"; import { type ZipProvider, initZipProvider } from "./services/import/zip_provider"; +import markdown from "./services/import/markdown"; export { getLog } from "./services/log"; export type * from "./services/sql/types"; @@ -106,6 +107,8 @@ export { default as zipImportService } from "./services/import/zip"; export * as becca_easy_mocking from "./test/becca_easy_mocking"; export * as becca_mocking from "./test/becca_mocking"; +export { default as markdownImportService } from "./services/import/markdown"; + export async function initializeCore({ dbConfig, executionContext, crypto, zip, translations, messaging, request, schema, extraAppInfo, platform, getDemoArchive }: { dbConfig: SqlServiceParams, executionContext: ExecutionContext, diff --git a/packages/trilium-core/src/routes/api/import.ts b/packages/trilium-core/src/routes/api/import.ts index ea78365928..89b3018881 100644 --- a/packages/trilium-core/src/routes/api/import.ts +++ b/packages/trilium-core/src/routes/api/import.ts @@ -1,9 +1,7 @@ import type { Request } from "express"; import type { File } from "../../services/import/common.js"; -interface ImportRequest

extends Request

{ - file?: File; -} +type ImportRequest

= Omit, "file"> & { file?: File }; import becca from "../../becca/becca.js"; import type BNote from "../../becca/entities/bnote.js"; @@ -64,6 +62,7 @@ async function importNotesToBranch(req: ImportRequest<{ parentNoteId: string }>) return importResult; } } else if (extension === ".enex" && options.explodeArchives) { + throw "ENEX import is currently not supported. Please use the desktop app to import ENEX files and then sync with the server."; // const importResult = await enexImportService.importEnex(taskContext, file, parentNote); // if (!Array.isArray(importResult)) { // note = importResult;