Merge branch 'main' into feature/toc_improvements

.github/workflows/dev.yml

@@ -40,32 +40,11 @@ jobs:
       - name: Run the client-side tests
         run: pnpm run --filter=client test
 
-      - name: Upload client test report
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: client-test-report
-          path: apps/client/test-output/vitest/html/
-          retention-days: 30
-
       - name: Run the server-side tests
         run: pnpm run --filter=server test
 
-      - name: Upload server test report
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: server-test-report
-          path: apps/server/test-output/vitest/html/
-          retention-days: 30
-
-      - name: Run CKEditor e2e tests
-        run: |
-          pnpm run --filter=ckeditor5-mermaid test
-          pnpm run --filter=ckeditor5-math test
-
       - name: Run the rest of the tests
-        run: pnpm run --filter=\!client --filter=\!server --filter=\!ckeditor5-mermaid --filter=\!ckeditor5-math test
+        run: pnpm run --filter=\!client --filter=\!server test"
 
   build_docker:
     name: Build Docker image

apps/build-docs/package.json

@@ -16,7 +16,7 @@
   "license": "AGPL-3.0-only",
   "packageManager": "pnpm@10.30.3",
   "devDependencies": {
-    "@redocly/cli": "2.20.4",
+    "@redocly/cli": "2.20.2",
     "archiver": "7.0.1",
     "fs-extra": "11.3.4",
     "js-yaml": "4.1.1",

apps/client/src/widgets/sidebar/TableOfContents.tsx

@@ -1,6 +1,6 @@
 import "./TableOfContents.css";
 
-import { CKTextEditor, ModelElement } from "@triliumnext/ckeditor5";
+import { attributeChangeAffectsHeading, CKTextEditor, ModelElement } from "@triliumnext/ckeditor5";
 import clsx from "clsx";
 import { useCallback, useEffect, useRef, useState } from "preact/hooks";
 
@@ -170,11 +170,14 @@ function EditableTextTableOfContents() {
 
             const affectsHeadings = changes.some( change => {
                 return (
-                    change.type === 'insert' || change.type === 'remove' || (change.type === 'attribute' && change.attributeKey === 'headingLevel')
+                    change.type === 'insert' || change.type === 'remove' ||
+                    (change.type === 'attribute' && attributeChangeAffectsHeading(change, textEditor))
                 );
             });
             if (affectsHeadings) {
-                setHeadings(extractTocFromTextEditor(textEditor));
+                requestAnimationFrame(() => {
+                    setHeadings(extractTocFromTextEditor(textEditor));
+                });
             }
         };
 

apps/client/vite.config.mts

@@ -120,11 +120,7 @@ export default defineConfig(() => ({
         environment: "happy-dom",
         setupFiles: [
             "./src/test/setup.ts"
-        ],
-        reporters: [
-            "verbose",
-            ["html", { outputFile: "./test-output/vitest/html/index.html" }]
-        ],
+        ]
     },
     commonjsOptions: {
         transformMixedEsModules: true,

apps/server/src/app.ts

@@ -86,9 +86,8 @@ export default async function buildApp() {
     app.use(`/robots.txt`, express.static(path.join(publicAssetsDir, "robots.txt")));
     app.use(`/icon.png`, express.static(path.join(publicAssetsDir, "icon.png")));
 
-    const { default: sessionParser, startSessionCleanup } = await import("./routes/session_parser.js");
+    const sessionParser = (await import("./routes/session_parser.js")).default;
     app.use(sessionParser);
-    startSessionCleanup();
     app.use(favicon(path.join(assetsDir, isDev ? "icon-dev.ico" : "icon.ico")));
 
     if (openID.isOpenIDEnabled())
@@ -99,16 +98,16 @@ export default async function buildApp() {
     custom.register(app);
     error_handlers.register(app);
 
-    const { startSyncTimer } = await import("./services/sync.js");
-    startSyncTimer();
+    // triggers sync timer
+    await import("./services/sync.js");
 
+    // triggers backup timer
     await import("./services/backup.js");
 
-    const { startConsistencyChecks } = await import("./services/consistency_checks.js");
-    startConsistencyChecks();
+    // trigger consistency checks timer
+    await import("./services/consistency_checks.js");
 
-    const { startScheduler } = await import("./services/scheduler.js");
-    startScheduler();
+    await import("./services/scheduler.js");
 
     startScheduledCleanup();
 

apps/server/src/routes/login.spec.ts

@@ -2,7 +2,7 @@ import { dayjs } from "@triliumnext/commons";
 import type { Application } from "express";
 import { SessionData } from "express-session";
 import supertest, { type Response } from "supertest";
-import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { beforeAll, describe, expect, it, vi } from "vitest";
 
 import cls from "../services/cls.js";
 import { type SQLiteSessionStore } from "./session_parser.js";
@@ -20,10 +20,6 @@ describe("Login Route test", () => {
         ({ sessionStore, CLEAN_UP_INTERVAL } = (await import("./session_parser.js")));
     });
 
-    afterAll(() => {
-        vi.useRealTimers();
-    });
-
     it("should return the login page, when using a GET request", async () => {
 
         // RegExp for login page specific string in HTML

apps/server/src/routes/session_parser.ts

@@ -113,13 +113,11 @@ const sessionParser: express.RequestHandler = session({
     store: sessionStore
 });
 
-export function startSessionCleanup() {
-    setInterval(() => {
-        // Clean up expired sessions.
-        const now = Date.now();
-        const result = sql.execute(/*sql*/`DELETE FROM sessions WHERE expires < ?`, now);
-        console.log("Cleaning up expired sessions: ", result.changes);
-    }, CLEAN_UP_INTERVAL);
-}
+setInterval(() => {
+    // Clean up expired sesions.
+    const now = Date.now();
+    const result = sql.execute(/*sql*/`DELETE FROM sessions WHERE expires < ?`, now);
+    console.log("Cleaning up expired sessions: ", result.changes);
+}, CLEAN_UP_INTERVAL);
 
 export default sessionParser;

apps/server/src/services/consistency_checks.ts

@@ -953,14 +953,12 @@ function runEntityChangesChecks() {
     consistencyChecks.findEntityChangeIssues();
 }
 
-export function startConsistencyChecks() {
-    sqlInit.dbReady.then(() => {
-        setInterval(cls.wrap(runPeriodicChecks), 60 * 60 * 1000);
+sqlInit.dbReady.then(() => {
+    setInterval(cls.wrap(runPeriodicChecks), 60 * 60 * 1000);
 
-        // kickoff checks soon after startup (to not block the initial load)
-        setTimeout(cls.wrap(runPeriodicChecks), 4 * 1000);
-    });
-}
+    // kickoff checks soon after startup (to not block the initial load)
+    setTimeout(cls.wrap(runPeriodicChecks), 4 * 1000);
+});
 
 export default {
     runOnDemandChecks,

apps/server/src/services/import/single.spec.ts

@@ -24,7 +24,6 @@ async function testImport(fileName: string, mimetype: string) {
             const rootNote = becca.getNote("root");
             if (!rootNote) {
                 reject("Missing root note.");
-                return;
             }
 
             const importedNote = single.importSingleFile(

apps/server/src/services/scheduler.ts

@@ -35,41 +35,39 @@ function runNotesWithLabel(runAttrValue: string) {
     }
 }
 
-export function startScheduler() {
-    // If the database is already initialized, we need to check the hidden subtree. Otherwise, hidden subtree
-    // is also checked before importing the demo.zip, so no need to do it again.
-    if (sqlInit.isDbInitialized()) {
-        console.log("Checking hidden subtree.");
-        sqlInit.dbReady.then(() => cls.init(() => hiddenSubtreeService.checkHiddenSubtree()));
+// If the database is already initialized, we need to check the hidden subtree. Otherwise, hidden subtree
+// is also checked before importing the demo.zip, so no need to do it again.
+if (sqlInit.isDbInitialized()) {
+    console.log("Checking hidden subtree.");
+    sqlInit.dbReady.then(() => cls.init(() => hiddenSubtreeService.checkHiddenSubtree()));
+}
+
+// Periodic checks.
+sqlInit.dbReady.then(() => {
+    if (!process.env.TRILIUM_SAFE_MODE) {
+        setTimeout(
+            cls.wrap(() => runNotesWithLabel("backendStartup")),
+            10 * 1000
+        );
+
+        setInterval(
+            cls.wrap(() => runNotesWithLabel("hourly")),
+            3600 * 1000
+        );
+
+        setInterval(
+            cls.wrap(() => runNotesWithLabel("daily")),
+            24 * 3600 * 1000
+        );
+
+        setInterval(
+            cls.wrap(() => hiddenSubtreeService.checkHiddenSubtree()),
+            7 * 3600 * 1000
+        );
     }
 
-    // Periodic checks.
-    sqlInit.dbReady.then(() => {
-        if (!process.env.TRILIUM_SAFE_MODE) {
-            setTimeout(
-                cls.wrap(() => runNotesWithLabel("backendStartup")),
-                10 * 1000
-            );
-
-            setInterval(
-                cls.wrap(() => runNotesWithLabel("hourly")),
-                3600 * 1000
-            );
-
-            setInterval(
-                cls.wrap(() => runNotesWithLabel("daily")),
-                24 * 3600 * 1000
-            );
-
-            setInterval(
-                cls.wrap(() => hiddenSubtreeService.checkHiddenSubtree()),
-                7 * 3600 * 1000
-            );
-        }
-
-        setInterval(() => checkProtectedSessionExpiration(), 30000);
-    });
-}
+    setInterval(() => checkProtectedSessionExpiration(), 30000);
+});
 
 function checkProtectedSessionExpiration() {
     const protectedSessionTimeout = options.getOptionInt("protectedSessionTimeout");

apps/server/src/services/sql_init.ts

@@ -50,7 +50,7 @@ async function initDbConnection() {
 
     await migrationService.migrateIfNecessary();
 
-    sql.execute('CREATE TEMP TABLE IF NOT EXISTS "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)');
+    sql.execute('CREATE TEMP TABLE "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)');
 
     sql.execute(`
     CREATE TABLE IF NOT EXISTS "user_data"

apps/server/src/services/sync.ts

@@ -446,17 +446,15 @@ function getOutstandingPullCount() {
     return outstandingPullCount;
 }
 
-export function startSyncTimer() {
-    becca_loader.beccaLoaded.then(() => {
-        setInterval(cls.wrap(sync), 60000);
+becca_loader.beccaLoaded.then(() => {
+    setInterval(cls.wrap(sync), 60000);
 
-        // kickoff initial sync immediately, but should happen after initial consistency checks
-        setTimeout(cls.wrap(sync), 5000);
+    // kickoff initial sync immediately, but should happen after initial consistency checks
+    setTimeout(cls.wrap(sync), 5000);
 
-        // called just so ws.setLastSyncedPush() is called
-        getLastSyncedPush();
-    });
-}
+    // called just so ws.setLastSyncedPush() is called
+    getLastSyncedPush();
+});
 
 export default {
     sync,

apps/server/src/share/routes.spec.ts

@@ -1,6 +1,6 @@
 import type { Application, NextFunction,Request, Response } from "express";
 import supertest from "supertest";
-import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 
 import { safeExtractMessageAndStackFromError } from "../services/utils.js";
 
@@ -23,10 +23,6 @@ describe("Share API test", () => {
         });
     });
 
-    afterAll(() => {
-        vi.useRealTimers();
-    });
-
     beforeEach(() => {
         cannotSetHeadersCount = 0;
     });

apps/server/vite.config.mts

@@ -19,18 +19,16 @@ export default defineConfig(() => ({
     exclude: [
       "spec/build-checks/**",
     ],
-    hookTimeout: 20_000,
-    testTimeout: 40_000,
+    hookTimeout: 20000,
     reporters: [
-      "verbose",
-      ["html", { outputFile: "./test-output/vitest/html/index.html" }]
+      "verbose"
     ],
     coverage: {
       reportsDirectory: './test-output/vitest/coverage',
       provider: 'v8' as const,
       reporter: [ "text", "html" ]
     },
-    pool: "forks",
-    maxWorkers: 6
+    pool: "vmForks",
+    maxWorkers: 3
   },
 }));

packages/ckeditor5/src/index.ts

@@ -11,6 +11,7 @@ export type { EditorConfig, MentionFeed, MentionFeedObjectItem, ModelNode, Model
 export type { TemplateDefinition } from "ckeditor5-premium-features";
 export { default as buildExtraCommands } from "./extra_slash_commands.js";
 export { default as getCkLocale } from "./i18n.js";
+export * from "./utils.js";
 
 // Import with sideffects to ensure that type augmentations are present.
 import "@triliumnext/ckeditor5-math";

packages/ckeditor5/src/utils.ts

@@ -0,0 +1,28 @@
+import type { DifferItemAttribute, Editor, ModelDocumentFragment, ModelElement, ModelNode } from "ckeditor5";
+
+function hasHeadingAncestor(node: ModelElement | ModelNode | ModelDocumentFragment | null): boolean {
+    let current: ModelElement | ModelNode | ModelDocumentFragment | null = node;
+    while (current) {
+        if (!!current && current.is('element') && (current as ModelElement).name.startsWith("heading")) return true;
+        current = current.parent;
+    }
+    return false;
+}
+
+export function attributeChangeAffectsHeading(change: DifferItemAttribute, editor: Editor): boolean {
+    if (change.type !== "attribute") return false;
+
+    // Fast checks on range boundaries
+    if (hasHeadingAncestor(change.range.start.parent) || hasHeadingAncestor(change.range.end.parent)) {
+        return true;
+    }
+
+    // Robust check across the whole changed range
+    const range = editor.model.createRange(change.range.start, change.range.end);
+    for (const item of range.getItems()) {
+        const baseNode = item.is("$textProxy") ? item.parent : item;
+        if (hasHeadingAncestor(baseNode)) return true;
+    }
+
+    return false;
+}

pnpm-lock.yaml

@@ -156,8 +156,8 @@ importers:
   apps/build-docs:
     devDependencies:
       '@redocly/cli':
-        specifier: 2.20.4
-        version: 2.20.4(@opentelemetry/api@1.9.0)(bufferutil@4.0.9)(core-js@3.46.0)(encoding@0.1.13)(utf-8-validate@6.0.5)
+        specifier: 2.20.2
+        version: 2.20.2(@opentelemetry/api@1.9.0)(bufferutil@4.0.9)(core-js@3.46.0)(encoding@0.1.13)(utf-8-validate@6.0.5)
       archiver:
         specifier: 7.0.1
         version: 7.0.1
@@ -5122,8 +5122,8 @@ packages:
   '@redocly/ajv@8.18.0':
     resolution: {integrity: sha512-F+LMD2IDIXuHxgpLJh3nkLj9+tSaEzoUWd+7fONGq5pe2169FUDjpEkOfEpoGLz1sbZni/69p07OsecNfAOpqA==}
 
-  '@redocly/cli@2.20.4':
-    resolution: {integrity: sha512-ahZmcTfEvw2ghWBm418A3anN3XS88Izbqn+HyH6fm/iJ/x04vFwW2KSv/V2K2JZON5ImpTCQBHPQd9bSmmZw9A==}
+  '@redocly/cli@2.20.2':
+    resolution: {integrity: sha512-wfEoGFoXq1vZjd9uEtTd9mIixF5I5Ci1rusK/9HHcS6UGy3o2kuvrSn1daDHl1T3KcG5YFqQoLzzEgcL4Je7KQ==}
     engines: {node: '>=22.12.0 || >=20.19.0 <21.0.0', npm: '>=10'}
     hasBin: true
 
@@ -5137,12 +5137,12 @@ packages:
     resolution: {integrity: sha512-0EbE8LRbkogtcCXU7liAyC00n9uNG9hJ+eMyHFdUsy9lB/WGqnEBgwjA9q2cyzAVcdTkQqTBBU1XePNnN3OijA==}
     engines: {node: '>=18.17.0', npm: '>=9.5.0'}
 
-  '@redocly/openapi-core@2.20.4':
-    resolution: {integrity: sha512-3WZh8dPF6MrxLDbTG4GXtV81EOqHrpMWlOhELWBIICRieMMt/LKcGFuOBRdLEp/KMU2ypwQLKYHrKYCeUNwO3Q==}
+  '@redocly/openapi-core@2.20.2':
+    resolution: {integrity: sha512-L3rzEZWMxq9SpAHP8k9C+/Fqxex4vxXgSZ5hQ+dg7++LXre67ZbT3RkhKsmYadJB/EUdbS6Z+h1Ont5TAI6zyQ==}
     engines: {node: '>=22.12.0 || >=20.19.0 <21.0.0', npm: '>=10'}
 
-  '@redocly/respect-core@2.20.4':
-    resolution: {integrity: sha512-kOM1cCDQtTwZkD58Lx3IIUPcFuRD8bnlkCFUvi0iigy6JJKfpRwgXuzkzFf6FHVO9Z5LwJuYgIP58q3TftWrqg==}
+  '@redocly/respect-core@2.20.2':
+    resolution: {integrity: sha512-oUCp+H83py0DEq5DJ+XmGPJQW9Yoq/tgml4OGwQrPEt1sC7QF7GjMRCA1HnRS4/dQ9psWGyFj9FwI1fCupQ8uw==}
     engines: {node: '>=22.12.0 || >=20.19.0 <21.0.0', npm: '>=10'}
 
   '@replit/codemirror-indentation-markers@6.5.3':
@@ -9851,9 +9851,6 @@ packages:
   fast-uri@3.1.0:
     resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==}
 
-  fast-xml-builder@1.0.0:
-    resolution: {integrity: sha512-fpZuDogrAgnyt9oDDz+5DBz0zgPdPZz6D4IR7iESxRXElrlGTRkHJ9eEt+SACRJwT0FNFrt71DFQIUFBJfX/uQ==}
-
   fast-xml-parser@4.4.1:
     resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==}
     hasBin: true
@@ -9866,10 +9863,6 @@ packages:
     resolution: {integrity: sha512-QNI3sAvSvaOiaMl8FYU4trnEzCwiRr8XMWgAHzlrWpTSj+QaCSvOf1h82OEP1s4hiAXhnbXSyFWCf4ldZzZRVA==}
     hasBin: true
 
-  fast-xml-parser@5.4.2:
-    resolution: {integrity: sha512-pw/6pIl4k0CSpElPEJhDppLzaixDEuWui2CUQQBH/ECDf7+y6YwA4Gf7Tyb0Rfe4DIMuZipYj4AEL0nACKglvQ==}
-    hasBin: true
-
   fastest-levenshtein@1.0.16:
     resolution: {integrity: sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==}
     engines: {node: '>= 4.9.1'}
@@ -12669,9 +12662,6 @@ packages:
   openapi-sampler@1.7.0:
     resolution: {integrity: sha512-fWq32F5vqGpgRJYIarC/9Y1wC9tKnRDcCOjsDJ7MIcSv2HsE7kNifcXIZ8FVtNStBUWxYrEk/MKqVF0SwZ5gog==}
 
-  openapi-sampler@1.7.1:
-    resolution: {integrity: sha512-pKFRROcYyxRt9GIn0NmS+GkWPS19l0CLQRYAnHk4m1Qp+G43ssVNcfRMs1sLkGrVMuFWO4P4F6YMXeXnfyFGuQ==}
-
   openapi-types@12.1.3:
     resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==}
 
@@ -21517,14 +21507,14 @@ snapshots:
       json-schema-traverse: 1.0.0
       require-from-string: 2.0.2
 
-  '@redocly/cli@2.20.4(@opentelemetry/api@1.9.0)(bufferutil@4.0.9)(core-js@3.46.0)(encoding@0.1.13)(utf-8-validate@6.0.5)':
+  '@redocly/cli@2.20.2(@opentelemetry/api@1.9.0)(bufferutil@4.0.9)(core-js@3.46.0)(encoding@0.1.13)(utf-8-validate@6.0.5)':
     dependencies:
       '@opentelemetry/exporter-trace-otlp-http': 0.202.0(@opentelemetry/api@1.9.0)
       '@opentelemetry/resources': 2.0.1(@opentelemetry/api@1.9.0)
       '@opentelemetry/sdk-trace-node': 2.0.1(@opentelemetry/api@1.9.0)
       '@opentelemetry/semantic-conventions': 1.34.0
-      '@redocly/openapi-core': 2.20.4
-      '@redocly/respect-core': 2.20.4
+      '@redocly/openapi-core': 2.20.2
+      '@redocly/respect-core': 2.20.2
       abort-controller: 3.0.0
       ajv: '@redocly/ajv@8.18.0'
       ajv-formats: 3.0.1(@redocly/ajv@8.18.0)
@@ -21576,7 +21566,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  '@redocly/openapi-core@2.20.4':
+  '@redocly/openapi-core@2.20.2':
     dependencies:
       '@redocly/ajv': 8.18.0
       '@redocly/config': 0.44.0
@@ -21589,18 +21579,18 @@ snapshots:
       pluralize: 8.0.0
       yaml-ast-parser: 0.0.43
 
-  '@redocly/respect-core@2.20.4':
+  '@redocly/respect-core@2.20.2':
     dependencies:
       '@faker-js/faker': 7.6.0
       '@noble/hashes': 1.8.0
       '@redocly/ajv': 8.18.0
-      '@redocly/openapi-core': 2.20.4
+      '@redocly/openapi-core': 2.20.2
       ajv: '@redocly/ajv@8.18.0'
       better-ajv-errors: 1.2.0(@redocly/ajv@8.18.0)
       colorette: 2.0.20
       json-pointer: 0.6.2
       jsonpath-rfc9535: 1.3.0
-      openapi-sampler: 1.7.1
+      openapi-sampler: 1.7.0
       outdent: 0.8.0
       picomatch: 4.0.3
 
@@ -28326,8 +28316,6 @@ snapshots:
 
   fast-uri@3.1.0: {}
 
-  fast-xml-builder@1.0.0: {}
-
   fast-xml-parser@4.4.1:
     dependencies:
       strnum: 1.1.2
@@ -28340,11 +28328,6 @@ snapshots:
     dependencies:
       strnum: 2.1.2
 
-  fast-xml-parser@5.4.2:
-    dependencies:
-      fast-xml-builder: 1.0.0
-      strnum: 2.1.2
-
   fastest-levenshtein@1.0.16: {}
 
   fastq@1.19.1:
@@ -31716,12 +31699,6 @@ snapshots:
       fast-xml-parser: 5.3.6
       json-pointer: 0.6.2
 
-  openapi-sampler@1.7.1:
-    dependencies:
-      '@types/json-schema': 7.0.15
-      fast-xml-parser: 5.4.2
-      json-pointer: 0.6.2
-
   openapi-types@12.1.3: {}
 
   opener@1.5.2: {}
@@ -32570,7 +32547,7 @@ snapshots:
 
   postcss@8.4.49:
     dependencies:
-      nanoid: 5.1.6
+      nanoid: 5.1.5
       picocolors: 1.1.1
       source-map-js: 1.2.1