From 09ff9ccc65d29f77f65b4c4f84b7c61fe9efa0c5 Mon Sep 17 00:00:00 2001 From: perfectra1n Date: Sat, 15 Nov 2025 15:32:55 -0800 Subject: [PATCH] feat(dev): add new stress test population script --- scripts/stress-test-populate.ts | 507 ++++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 scripts/stress-test-populate.ts diff --git a/scripts/stress-test-populate.ts b/scripts/stress-test-populate.ts new file mode 100644 index 000000000..991a7ac5c --- /dev/null +++ b/scripts/stress-test-populate.ts @@ -0,0 +1,507 @@ +#!/usr/bin/env tsx +/** + * Stress Test Database Population Script + * + * This script populates the Trilium database with a large number of diverse notes + * for performance testing, search testing, and stress testing purposes. + * + * Usage: + * pnpm tsx scripts/stress-test-populate.ts [options] + * + * Options: + * --notes=N Number of notes to create (default: 5000) + * --depth=N Maximum hierarchy depth (default: 10) + * --max-relations=N Maximum relations per note (default: 10) + * --max-labels=N Maximum labels per note (default: 8) + * --help Show this help message + * + * Note: This script requires an existing Trilium database. Run Trilium at least once + * before running this script to initialize the database. + */ + +// Set up environment variables like the server does +process.env.TRILIUM_ENV = "dev"; +process.env.TRILIUM_DATA_DIR = process.env.TRILIUM_DATA_DIR || "trilium-data"; + +import { initializeTranslations } from "../apps/server/src/services/i18n.js"; +import BNote from "../apps/server/src/becca/entities/bnote.js"; +import BBranch from "../apps/server/src/becca/entities/bbranch.js"; +import BAttribute from "../apps/server/src/becca/entities/battribute.js"; +import becca from "../apps/server/src/becca/becca.js"; +import { NoteBuilder, id, note } from "../apps/server/src/test/becca_mocking.js"; +import type { NoteType } from "@triliumnext/commons"; +import { dbReady } from "../apps/server/src/services/sql_init.js"; + +// Parse command line arguments +const args = process.argv.slice(2); +const config = { + noteCount: 5000, + maxDepth: 10, + maxRelations: 10, + maxLabels: 8, +}; + +for (const arg of args) { + if (arg === "--help" || arg === "-h") { + console.log(` +Stress Test Database Population Script + +This script populates the Trilium database with a large number of diverse notes +for performance testing, search testing, and stress testing purposes. + +Usage: + pnpm tsx scripts/stress-test-populate.ts [options] + +Options: + --notes=N Number of notes to create (default: ${config.noteCount}) + --depth=N Maximum hierarchy depth (default: ${config.maxDepth}) + --max-relations=N Maximum relations per note (default: ${config.maxRelations}) + --max-labels=N Maximum labels per note (default: ${config.maxLabels}) + --help, -h Show this help message + +Examples: + # Use defaults (5000 notes, depth 10) + pnpm tsx scripts/stress-test-populate.ts + + # Create 10000 notes with depth 15 + pnpm tsx scripts/stress-test-populate.ts --notes=10000 --depth=15 + + # Smaller test with 1000 notes and depth 5 + pnpm tsx scripts/stress-test-populate.ts --notes=1000 --depth=5 + `); + process.exit(0); + } + + const match = arg.match(/--(\w+)=(.+)/); + if (match) { + const [, key, value] = match; + switch (key) { + case "notes": + config.noteCount = parseInt(value, 10); + break; + case "depth": + config.maxDepth = parseInt(value, 10); + break; + case "max-relations": + config.maxRelations = parseInt(value, 10); + break; + case "max-labels": + config.maxLabels = parseInt(value, 10); + break; + } + } +} + +console.log("Stress Test Database Population"); +console.log("================================"); +console.log(`Target note count: ${config.noteCount}`); +console.log(`Maximum depth: ${config.maxDepth}`); +console.log(`Maximum relations per note: ${config.maxRelations}`); +console.log(`Maximum labels per note: ${config.maxLabels}`); +console.log(""); + +// Note type distribution (rough percentages) +const NOTE_TYPES: { type: NoteType; mime: string; weight: number }[] = [ + { type: "text", mime: "text/html", weight: 50 }, + { type: "code", mime: "text/javascript", weight: 15 }, + { type: "code", mime: "text/x-python", weight: 10 }, + { type: "code", mime: "application/json", weight: 5 }, + { type: "mermaid", mime: "text/mermaid", weight: 5 }, + { type: "book", mime: "text/html", weight: 5 }, + { type: "render", mime: "text/html", weight: 3 }, + { type: "relationMap", mime: "application/json", weight: 2 }, + { type: "search", mime: "application/json", weight: 2 }, + { type: "canvas", mime: "application/json", weight: 2 }, + { type: "doc", mime: "text/html", weight: 1 }, +]; + +// Sample content generators +const LOREM_IPSUM = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.`; + +const CODE_SAMPLES = { + "text/javascript": `function fibonacci(n) { + if (n <= 1) return n; + return fibonacci(n - 1) + fibonacci(n - 2); +} + +console.log(fibonacci(10));`, + + "text/x-python": `def quicksort(arr): + if len(arr) <= 1: + return arr + pivot = arr[len(arr) // 2] + left = [x for x in arr if x < pivot] + middle = [x for x in arr if x == pivot] + right = [x for x in arr if x > pivot] + return quicksort(left) + middle + quicksort(right) + +print(quicksort([3, 6, 8, 10, 1, 2, 1]))`, + + "application/json": `{ + "name": "example", + "version": "1.0.0", + "description": "A sample JSON document", + "keywords": ["example", "test", "stress"] +}`, +}; + +const MERMAID_SAMPLE = `graph TD + A[Start] --> B{Decision} + B -->|Yes| C[Process] + B -->|No| D[Alternative] + C --> E[End] + D --> E`; + +// Common label names and value patterns +const LABEL_NAMES = [ + "priority", "status", "category", "tag", "project", "version", "author", + "reviewed", "archived", "published", "draft", "language", "framework", + "difficulty", "rating", "year", "month", "country", "city", "department" +]; + +const LABEL_VALUES = { + priority: ["high", "medium", "low", "critical"], + status: ["active", "completed", "pending", "archived", "draft"], + category: ["personal", "work", "reference", "project", "research"], + rating: ["1", "2", "3", "4", "5"], + difficulty: ["beginner", "intermediate", "advanced", "expert"], + language: ["javascript", "python", "typescript", "rust", "go", "java"], + framework: ["react", "vue", "angular", "express", "django", "flask"], +}; + +// Relation names +const RELATION_NAMES = [ + "relatedTo", "dependsOn", "references", "implements", "extends", + "baseOn", "contains", "partOf", "author", "reviewer", "assignedTo", + "linkedWith", "similarTo", "contradicts", "supports" +]; + +// Title prefixes for different categories +const TITLE_PREFIXES = [ + "Documentation", "Tutorial", "Guide", "Reference", "API", "Concept", + "Example", "Pattern", "Architecture", "Design", "Implementation", + "Analysis", "Research", "Note", "Idea", "Project", "Task", "Feature", + "Bug", "Issue", "Discussion", "Meeting", "Review", "Proposal", "Spec" +]; + +const TITLE_SUBJECTS = [ + "Authentication", "Database", "API", "Frontend", "Backend", "Security", + "Performance", "Testing", "Deployment", "Configuration", "Monitoring", + "Logging", "Caching", "Scaling", "Optimization", "Refactoring", + "Integration", "Migration", "Upgrade", "Architecture", "Infrastructure" +]; + +/** + * Select a random item from array based on weights + */ +function weightedRandom(items: T[]): T { + const totalWeight = items.reduce((sum, item) => sum + item.weight, 0); + let random = Math.random() * totalWeight; + + for (const item of items) { + random -= item.weight; + if (random <= 0) { + return item; + } + } + + return items[items.length - 1]; +} + +/** + * Generate random integer between min and max (inclusive) + */ +function randomInt(min: number, max: number): number { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +/** + * Generate a random title + */ +function generateTitle(index: number): string { + if (Math.random() < 0.3) { + // Use structured title + const prefix = TITLE_PREFIXES[randomInt(0, TITLE_PREFIXES.length - 1)]; + const subject = TITLE_SUBJECTS[randomInt(0, TITLE_SUBJECTS.length - 1)]; + return `${prefix}: ${subject} #${index}`; + } else { + // Use simple title + return `Note ${index}`; + } +} + +/** + * Generate content based on note type + */ +function generateContent(type: NoteType, mime: string): string { + if (type === "code" && CODE_SAMPLES[mime as keyof typeof CODE_SAMPLES]) { + return CODE_SAMPLES[mime as keyof typeof CODE_SAMPLES]; + } else if (type === "mermaid") { + return MERMAID_SAMPLE; + } else if (type === "text" || type === "book" || type === "doc") { + // Generate multiple paragraphs + const paragraphs = randomInt(1, 5); + return Array(paragraphs).fill(LOREM_IPSUM).join("\n\n"); + } else if (mime === "application/json") { + return CODE_SAMPLES["application/json"]; + } + return ""; +} + +/** + * Generate random labels for a note + */ +function generateLabels(noteBuilder: NoteBuilder, count: number): void { + const labelsToAdd = Math.min(count, randomInt(0, config.maxLabels)); + + for (let i = 0; i < labelsToAdd; i++) { + const labelName = LABEL_NAMES[randomInt(0, LABEL_NAMES.length - 1)]; + let labelValue = ""; + + // Use predefined values if available + if (LABEL_VALUES[labelName as keyof typeof LABEL_VALUES]) { + const values = LABEL_VALUES[labelName as keyof typeof LABEL_VALUES]; + labelValue = values[randomInt(0, values.length - 1)]; + } else { + labelValue = `value${randomInt(1, 100)}`; + } + + const isInheritable = Math.random() < 0.2; // 20% chance of inheritable + noteBuilder.label(labelName, labelValue, isInheritable); + } +} + +/** + * Generate random relations for a note + */ +function generateRelations( + noteBuilder: NoteBuilder, + allNotes: BNote[], + maxRelations: number +): void { + if (allNotes.length === 0) return; + + const relationsToAdd = Math.min( + maxRelations, + randomInt(0, config.maxRelations) + ); + + for (let i = 0; i < relationsToAdd; i++) { + const relationName = RELATION_NAMES[randomInt(0, RELATION_NAMES.length - 1)]; + const targetNote = allNotes[randomInt(0, allNotes.length - 1)]; + + // Avoid self-relations + if (targetNote.noteId !== noteBuilder.note.noteId) { + noteBuilder.relation(relationName, targetNote); + } + } +} + +/** + * Create a note with random attributes + */ +function createRandomNote( + index: number, + allNotes: BNote[] +): NoteBuilder { + const noteType = weightedRandom(NOTE_TYPES); + const title = generateTitle(index); + + const noteBuilder = note(title, { + noteId: id(), + type: noteType.type, + mime: noteType.mime, + }); + + // Set content + const content = generateContent(noteType.type, noteType.mime); + if (content) { + noteBuilder.note.setContent(content, { forceSave: true }); + } + + // Add labels + generateLabels(noteBuilder, randomInt(0, config.maxLabels)); + + // Add relations (limit based on available notes) + const maxPossibleRelations = Math.min( + config.maxRelations, + Math.floor(allNotes.length / 10) // Limit to avoid too dense graphs + ); + generateRelations(noteBuilder, allNotes, maxPossibleRelations); + + // 5% chance of protected note + if (Math.random() < 0.05) { + noteBuilder.note.isProtected = true; + } + + // 10% chance of archived note + if (Math.random() < 0.1) { + noteBuilder.label("archived", "", true); + } + + return noteBuilder; +} + +/** + * Create notes recursively to build hierarchy + */ +function createNotesRecursively( + parent: NoteBuilder, + depth: number, + targetCount: number, + allNotes: BNote[] +): number { + let created = 0; + + if (depth >= config.maxDepth || targetCount <= 0) { + return 0; + } + + // Determine how many children at this level + // Decrease children count as depth increases to create pyramid structure + const maxChildrenAtDepth = Math.max(1, Math.floor(20 / (depth + 1))); + const childrenCount = Math.min( + targetCount, + randomInt(1, maxChildrenAtDepth) + ); + + for (let i = 0; i < childrenCount && created < targetCount; i++) { + const noteBuilder = createRandomNote(allNotes.length + 1, allNotes); + parent.child(noteBuilder); + allNotes.push(noteBuilder.note); + created++; + + // Log progress every 100 notes + if (allNotes.length % 100 === 0) { + console.log(` Created ${allNotes.length} notes...`); + } + + // Recursively create children + const remainingForSubtree = Math.floor((targetCount - created) / (childrenCount - i)); + const createdInSubtree = createNotesRecursively( + noteBuilder, + depth + 1, + remainingForSubtree, + allNotes + ); + created += createdInSubtree; + } + + return created; +} + +/** + * Main execution + */ +async function main() { + console.log("Initializing translations..."); + await initializeTranslations(); + + console.log("Initializing database connection..."); + + // Wait for database to be ready (initialized by sql.ts import) + await dbReady; + + console.log("Loading becca (backend cache)..."); + + // Dynamically import becca_loader to ensure proper initialization order + const { beccaLoaded } = await import("../apps/server/src/becca/becca_loader.js"); + await beccaLoaded; + + const rootNote = becca.getNote("root"); + if (!rootNote) { + throw new Error("Root note not found!"); + } + + // Create a container note for all stress test notes + const containerNote = note("Stress Test Notes", { + noteId: id(), + type: "book", + mime: "text/html", + }); + containerNote.note.setContent( + `

This note contains ${config.noteCount} notes generated for stress testing.

` + + `

Generated on: ${new Date().toISOString()}

` + + `

Configuration: depth=${config.maxDepth}, maxRelations=${config.maxRelations}, maxLabels=${config.maxLabels}

`, + { forceSave: true } + ); + + const rootBuilder = new NoteBuilder(rootNote); + rootBuilder.child(containerNote); + + console.log("\nCreating notes..."); + const startTime = Date.now(); + + const allNotes: BNote[] = [containerNote.note]; + + // Create notes recursively + const created = createNotesRecursively( + containerNote, + 0, + config.noteCount - 1, // -1 because we already created container + allNotes + ); + + const endTime = Date.now(); + const duration = (endTime - startTime) / 1000; + + console.log("\n================================"); + console.log("Stress Test Population Complete!"); + console.log("================================"); + console.log(`Total notes created: ${allNotes.length}`); + console.log(`Time taken: ${duration.toFixed(2)} seconds`); + console.log(`Notes per second: ${(allNotes.length / duration).toFixed(2)}`); + console.log(`Container note ID: ${containerNote.note.noteId}`); + console.log(""); + + // Print statistics + const noteTypeCount: Record = {}; + const labelCount: Record = {}; + let totalRelations = 0; + let protectedCount = 0; + let archivedCount = 0; + + for (const note of allNotes) { + // Count note types + noteTypeCount[note.type] = (noteTypeCount[note.type] || 0) + 1; + + // Count labels + for (const attr of note.getOwnedAttributes()) { + if (attr.type === "label") { + labelCount[attr.name] = (labelCount[attr.name] || 0) + 1; + if (attr.name === "archived") archivedCount++; + } else if (attr.type === "relation") { + totalRelations++; + } + } + + if (note.isProtected) protectedCount++; + } + + console.log("Note Type Distribution:"); + for (const [type, count] of Object.entries(noteTypeCount).sort((a, b) => b[1] - a[1])) { + console.log(` ${type}: ${count}`); + } + + console.log("\nTop 10 Label Names:"); + const sortedLabels = Object.entries(labelCount) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10); + for (const [name, count] of sortedLabels) { + console.log(` ${name}: ${count}`); + } + + console.log("\nOther Statistics:"); + console.log(` Total relations: ${totalRelations}`); + console.log(` Protected notes: ${protectedCount}`); + console.log(` Archived notes: ${archivedCount}`); + console.log(""); + console.log("You can find all generated notes under the 'Stress Test Notes' note in the tree."); +} + +// Run the script +main().catch((error) => { + console.error("Error during stress test population:"); + console.error(error); + process.exit(1); +});