mirror of
https://github.com/zadam/trilium.git
synced 2025-11-16 10:15:52 +01:00
508 lines
16 KiB
TypeScript
508 lines
16 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* Stress Test Database Population Script
|
|
*
|
|
* This script populates the Trilium database with a large number of diverse notes
|
|
* for performance testing, search testing, and stress testing purposes.
|
|
*
|
|
* Usage:
|
|
* pnpm tsx scripts/stress-test-populate.ts [options]
|
|
*
|
|
* Options:
|
|
* --notes=N Number of notes to create (default: 5000)
|
|
* --depth=N Maximum hierarchy depth (default: 10)
|
|
* --max-relations=N Maximum relations per note (default: 10)
|
|
* --max-labels=N Maximum labels per note (default: 8)
|
|
* --help Show this help message
|
|
*
|
|
* Note: This script requires an existing Trilium database. Run Trilium at least once
|
|
* before running this script to initialize the database.
|
|
*/
|
|
|
|
// Set up environment variables like the server does
|
|
process.env.TRILIUM_ENV = "dev";
|
|
process.env.TRILIUM_DATA_DIR = process.env.TRILIUM_DATA_DIR || "trilium-data";
|
|
|
|
import { initializeTranslations } from "../apps/server/src/services/i18n.js";
|
|
import BNote from "../apps/server/src/becca/entities/bnote.js";
|
|
import BBranch from "../apps/server/src/becca/entities/bbranch.js";
|
|
import BAttribute from "../apps/server/src/becca/entities/battribute.js";
|
|
import becca from "../apps/server/src/becca/becca.js";
|
|
import { NoteBuilder, id, note } from "../apps/server/src/test/becca_mocking.js";
|
|
import type { NoteType } from "@triliumnext/commons";
|
|
import { dbReady } from "../apps/server/src/services/sql_init.js";
|
|
|
|
// Parse command line arguments
|
|
const args = process.argv.slice(2);
|
|
const config = {
|
|
noteCount: 5000,
|
|
maxDepth: 10,
|
|
maxRelations: 10,
|
|
maxLabels: 8,
|
|
};
|
|
|
|
for (const arg of args) {
|
|
if (arg === "--help" || arg === "-h") {
|
|
console.log(`
|
|
Stress Test Database Population Script
|
|
|
|
This script populates the Trilium database with a large number of diverse notes
|
|
for performance testing, search testing, and stress testing purposes.
|
|
|
|
Usage:
|
|
pnpm tsx scripts/stress-test-populate.ts [options]
|
|
|
|
Options:
|
|
--notes=N Number of notes to create (default: ${config.noteCount})
|
|
--depth=N Maximum hierarchy depth (default: ${config.maxDepth})
|
|
--max-relations=N Maximum relations per note (default: ${config.maxRelations})
|
|
--max-labels=N Maximum labels per note (default: ${config.maxLabels})
|
|
--help, -h Show this help message
|
|
|
|
Examples:
|
|
# Use defaults (5000 notes, depth 10)
|
|
pnpm tsx scripts/stress-test-populate.ts
|
|
|
|
# Create 10000 notes with depth 15
|
|
pnpm tsx scripts/stress-test-populate.ts --notes=10000 --depth=15
|
|
|
|
# Smaller test with 1000 notes and depth 5
|
|
pnpm tsx scripts/stress-test-populate.ts --notes=1000 --depth=5
|
|
`);
|
|
process.exit(0);
|
|
}
|
|
|
|
const match = arg.match(/--(\w+)=(.+)/);
|
|
if (match) {
|
|
const [, key, value] = match;
|
|
switch (key) {
|
|
case "notes":
|
|
config.noteCount = parseInt(value, 10);
|
|
break;
|
|
case "depth":
|
|
config.maxDepth = parseInt(value, 10);
|
|
break;
|
|
case "max-relations":
|
|
config.maxRelations = parseInt(value, 10);
|
|
break;
|
|
case "max-labels":
|
|
config.maxLabels = parseInt(value, 10);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log("Stress Test Database Population");
|
|
console.log("================================");
|
|
console.log(`Target note count: ${config.noteCount}`);
|
|
console.log(`Maximum depth: ${config.maxDepth}`);
|
|
console.log(`Maximum relations per note: ${config.maxRelations}`);
|
|
console.log(`Maximum labels per note: ${config.maxLabels}`);
|
|
console.log("");
|
|
|
|
// Note type distribution (rough percentages)
|
|
const NOTE_TYPES: { type: NoteType; mime: string; weight: number }[] = [
|
|
{ type: "text", mime: "text/html", weight: 50 },
|
|
{ type: "code", mime: "text/javascript", weight: 15 },
|
|
{ type: "code", mime: "text/x-python", weight: 10 },
|
|
{ type: "code", mime: "application/json", weight: 5 },
|
|
{ type: "mermaid", mime: "text/mermaid", weight: 5 },
|
|
{ type: "book", mime: "text/html", weight: 5 },
|
|
{ type: "render", mime: "text/html", weight: 3 },
|
|
{ type: "relationMap", mime: "application/json", weight: 2 },
|
|
{ type: "search", mime: "application/json", weight: 2 },
|
|
{ type: "canvas", mime: "application/json", weight: 2 },
|
|
{ type: "doc", mime: "text/html", weight: 1 },
|
|
];
|
|
|
|
// Sample content generators
|
|
const LOREM_IPSUM = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.`;
|
|
|
|
const CODE_SAMPLES = {
|
|
"text/javascript": `function fibonacci(n) {
|
|
if (n <= 1) return n;
|
|
return fibonacci(n - 1) + fibonacci(n - 2);
|
|
}
|
|
|
|
console.log(fibonacci(10));`,
|
|
|
|
"text/x-python": `def quicksort(arr):
|
|
if len(arr) <= 1:
|
|
return arr
|
|
pivot = arr[len(arr) // 2]
|
|
left = [x for x in arr if x < pivot]
|
|
middle = [x for x in arr if x == pivot]
|
|
right = [x for x in arr if x > pivot]
|
|
return quicksort(left) + middle + quicksort(right)
|
|
|
|
print(quicksort([3, 6, 8, 10, 1, 2, 1]))`,
|
|
|
|
"application/json": `{
|
|
"name": "example",
|
|
"version": "1.0.0",
|
|
"description": "A sample JSON document",
|
|
"keywords": ["example", "test", "stress"]
|
|
}`,
|
|
};
|
|
|
|
const MERMAID_SAMPLE = `graph TD
|
|
A[Start] --> B{Decision}
|
|
B -->|Yes| C[Process]
|
|
B -->|No| D[Alternative]
|
|
C --> E[End]
|
|
D --> E`;
|
|
|
|
// Common label names and value patterns
|
|
const LABEL_NAMES = [
|
|
"priority", "status", "category", "tag", "project", "version", "author",
|
|
"reviewed", "archived", "published", "draft", "language", "framework",
|
|
"difficulty", "rating", "year", "month", "country", "city", "department"
|
|
];
|
|
|
|
const LABEL_VALUES = {
|
|
priority: ["high", "medium", "low", "critical"],
|
|
status: ["active", "completed", "pending", "archived", "draft"],
|
|
category: ["personal", "work", "reference", "project", "research"],
|
|
rating: ["1", "2", "3", "4", "5"],
|
|
difficulty: ["beginner", "intermediate", "advanced", "expert"],
|
|
language: ["javascript", "python", "typescript", "rust", "go", "java"],
|
|
framework: ["react", "vue", "angular", "express", "django", "flask"],
|
|
};
|
|
|
|
// Relation names
|
|
const RELATION_NAMES = [
|
|
"relatedTo", "dependsOn", "references", "implements", "extends",
|
|
"baseOn", "contains", "partOf", "author", "reviewer", "assignedTo",
|
|
"linkedWith", "similarTo", "contradicts", "supports"
|
|
];
|
|
|
|
// Title prefixes for different categories
|
|
const TITLE_PREFIXES = [
|
|
"Documentation", "Tutorial", "Guide", "Reference", "API", "Concept",
|
|
"Example", "Pattern", "Architecture", "Design", "Implementation",
|
|
"Analysis", "Research", "Note", "Idea", "Project", "Task", "Feature",
|
|
"Bug", "Issue", "Discussion", "Meeting", "Review", "Proposal", "Spec"
|
|
];
|
|
|
|
const TITLE_SUBJECTS = [
|
|
"Authentication", "Database", "API", "Frontend", "Backend", "Security",
|
|
"Performance", "Testing", "Deployment", "Configuration", "Monitoring",
|
|
"Logging", "Caching", "Scaling", "Optimization", "Refactoring",
|
|
"Integration", "Migration", "Upgrade", "Architecture", "Infrastructure"
|
|
];
|
|
|
|
/**
|
|
* Select a random item from array based on weights
|
|
*/
|
|
function weightedRandom<T extends { weight: number }>(items: T[]): T {
|
|
const totalWeight = items.reduce((sum, item) => sum + item.weight, 0);
|
|
let random = Math.random() * totalWeight;
|
|
|
|
for (const item of items) {
|
|
random -= item.weight;
|
|
if (random <= 0) {
|
|
return item;
|
|
}
|
|
}
|
|
|
|
return items[items.length - 1];
|
|
}
|
|
|
|
/**
|
|
* Generate random integer between min and max (inclusive)
|
|
*/
|
|
function randomInt(min: number, max: number): number {
|
|
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
}
|
|
|
|
/**
|
|
* Generate a random title
|
|
*/
|
|
function generateTitle(index: number): string {
|
|
if (Math.random() < 0.3) {
|
|
// Use structured title
|
|
const prefix = TITLE_PREFIXES[randomInt(0, TITLE_PREFIXES.length - 1)];
|
|
const subject = TITLE_SUBJECTS[randomInt(0, TITLE_SUBJECTS.length - 1)];
|
|
return `${prefix}: ${subject} #${index}`;
|
|
} else {
|
|
// Use simple title
|
|
return `Note ${index}`;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate content based on note type
|
|
*/
|
|
function generateContent(type: NoteType, mime: string): string {
|
|
if (type === "code" && CODE_SAMPLES[mime as keyof typeof CODE_SAMPLES]) {
|
|
return CODE_SAMPLES[mime as keyof typeof CODE_SAMPLES];
|
|
} else if (type === "mermaid") {
|
|
return MERMAID_SAMPLE;
|
|
} else if (type === "text" || type === "book" || type === "doc") {
|
|
// Generate multiple paragraphs
|
|
const paragraphs = randomInt(1, 5);
|
|
return Array(paragraphs).fill(LOREM_IPSUM).join("\n\n");
|
|
} else if (mime === "application/json") {
|
|
return CODE_SAMPLES["application/json"];
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/**
|
|
* Generate random labels for a note
|
|
*/
|
|
function generateLabels(noteBuilder: NoteBuilder, count: number): void {
|
|
const labelsToAdd = Math.min(count, randomInt(0, config.maxLabels));
|
|
|
|
for (let i = 0; i < labelsToAdd; i++) {
|
|
const labelName = LABEL_NAMES[randomInt(0, LABEL_NAMES.length - 1)];
|
|
let labelValue = "";
|
|
|
|
// Use predefined values if available
|
|
if (LABEL_VALUES[labelName as keyof typeof LABEL_VALUES]) {
|
|
const values = LABEL_VALUES[labelName as keyof typeof LABEL_VALUES];
|
|
labelValue = values[randomInt(0, values.length - 1)];
|
|
} else {
|
|
labelValue = `value${randomInt(1, 100)}`;
|
|
}
|
|
|
|
const isInheritable = Math.random() < 0.2; // 20% chance of inheritable
|
|
noteBuilder.label(labelName, labelValue, isInheritable);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate random relations for a note
|
|
*/
|
|
function generateRelations(
|
|
noteBuilder: NoteBuilder,
|
|
allNotes: BNote[],
|
|
maxRelations: number
|
|
): void {
|
|
if (allNotes.length === 0) return;
|
|
|
|
const relationsToAdd = Math.min(
|
|
maxRelations,
|
|
randomInt(0, config.maxRelations)
|
|
);
|
|
|
|
for (let i = 0; i < relationsToAdd; i++) {
|
|
const relationName = RELATION_NAMES[randomInt(0, RELATION_NAMES.length - 1)];
|
|
const targetNote = allNotes[randomInt(0, allNotes.length - 1)];
|
|
|
|
// Avoid self-relations
|
|
if (targetNote.noteId !== noteBuilder.note.noteId) {
|
|
noteBuilder.relation(relationName, targetNote);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a note with random attributes
|
|
*/
|
|
function createRandomNote(
|
|
index: number,
|
|
allNotes: BNote[]
|
|
): NoteBuilder {
|
|
const noteType = weightedRandom(NOTE_TYPES);
|
|
const title = generateTitle(index);
|
|
|
|
const noteBuilder = note(title, {
|
|
noteId: id(),
|
|
type: noteType.type,
|
|
mime: noteType.mime,
|
|
});
|
|
|
|
// Set content
|
|
const content = generateContent(noteType.type, noteType.mime);
|
|
if (content) {
|
|
noteBuilder.note.setContent(content, { forceSave: true });
|
|
}
|
|
|
|
// Add labels
|
|
generateLabels(noteBuilder, randomInt(0, config.maxLabels));
|
|
|
|
// Add relations (limit based on available notes)
|
|
const maxPossibleRelations = Math.min(
|
|
config.maxRelations,
|
|
Math.floor(allNotes.length / 10) // Limit to avoid too dense graphs
|
|
);
|
|
generateRelations(noteBuilder, allNotes, maxPossibleRelations);
|
|
|
|
// 5% chance of protected note
|
|
if (Math.random() < 0.05) {
|
|
noteBuilder.note.isProtected = true;
|
|
}
|
|
|
|
// 10% chance of archived note
|
|
if (Math.random() < 0.1) {
|
|
noteBuilder.label("archived", "", true);
|
|
}
|
|
|
|
return noteBuilder;
|
|
}
|
|
|
|
/**
|
|
* Create notes recursively to build hierarchy
|
|
*/
|
|
function createNotesRecursively(
|
|
parent: NoteBuilder,
|
|
depth: number,
|
|
targetCount: number,
|
|
allNotes: BNote[]
|
|
): number {
|
|
let created = 0;
|
|
|
|
if (depth >= config.maxDepth || targetCount <= 0) {
|
|
return 0;
|
|
}
|
|
|
|
// Determine how many children at this level
|
|
// Decrease children count as depth increases to create pyramid structure
|
|
const maxChildrenAtDepth = Math.max(1, Math.floor(20 / (depth + 1)));
|
|
const childrenCount = Math.min(
|
|
targetCount,
|
|
randomInt(1, maxChildrenAtDepth)
|
|
);
|
|
|
|
for (let i = 0; i < childrenCount && created < targetCount; i++) {
|
|
const noteBuilder = createRandomNote(allNotes.length + 1, allNotes);
|
|
parent.child(noteBuilder);
|
|
allNotes.push(noteBuilder.note);
|
|
created++;
|
|
|
|
// Log progress every 100 notes
|
|
if (allNotes.length % 100 === 0) {
|
|
console.log(` Created ${allNotes.length} notes...`);
|
|
}
|
|
|
|
// Recursively create children
|
|
const remainingForSubtree = Math.floor((targetCount - created) / (childrenCount - i));
|
|
const createdInSubtree = createNotesRecursively(
|
|
noteBuilder,
|
|
depth + 1,
|
|
remainingForSubtree,
|
|
allNotes
|
|
);
|
|
created += createdInSubtree;
|
|
}
|
|
|
|
return created;
|
|
}
|
|
|
|
/**
|
|
* Main execution
|
|
*/
|
|
async function main() {
|
|
console.log("Initializing translations...");
|
|
await initializeTranslations();
|
|
|
|
console.log("Initializing database connection...");
|
|
|
|
// Wait for database to be ready (initialized by sql.ts import)
|
|
await dbReady;
|
|
|
|
console.log("Loading becca (backend cache)...");
|
|
|
|
// Dynamically import becca_loader to ensure proper initialization order
|
|
const { beccaLoaded } = await import("../apps/server/src/becca/becca_loader.js");
|
|
await beccaLoaded;
|
|
|
|
const rootNote = becca.getNote("root");
|
|
if (!rootNote) {
|
|
throw new Error("Root note not found!");
|
|
}
|
|
|
|
// Create a container note for all stress test notes
|
|
const containerNote = note("Stress Test Notes", {
|
|
noteId: id(),
|
|
type: "book",
|
|
mime: "text/html",
|
|
});
|
|
containerNote.note.setContent(
|
|
`<p>This note contains ${config.noteCount} notes generated for stress testing.</p>` +
|
|
`<p>Generated on: ${new Date().toISOString()}</p>` +
|
|
`<p>Configuration: depth=${config.maxDepth}, maxRelations=${config.maxRelations}, maxLabels=${config.maxLabels}</p>`,
|
|
{ forceSave: true }
|
|
);
|
|
|
|
const rootBuilder = new NoteBuilder(rootNote);
|
|
rootBuilder.child(containerNote);
|
|
|
|
console.log("\nCreating notes...");
|
|
const startTime = Date.now();
|
|
|
|
const allNotes: BNote[] = [containerNote.note];
|
|
|
|
// Create notes recursively
|
|
const created = createNotesRecursively(
|
|
containerNote,
|
|
0,
|
|
config.noteCount - 1, // -1 because we already created container
|
|
allNotes
|
|
);
|
|
|
|
const endTime = Date.now();
|
|
const duration = (endTime - startTime) / 1000;
|
|
|
|
console.log("\n================================");
|
|
console.log("Stress Test Population Complete!");
|
|
console.log("================================");
|
|
console.log(`Total notes created: ${allNotes.length}`);
|
|
console.log(`Time taken: ${duration.toFixed(2)} seconds`);
|
|
console.log(`Notes per second: ${(allNotes.length / duration).toFixed(2)}`);
|
|
console.log(`Container note ID: ${containerNote.note.noteId}`);
|
|
console.log("");
|
|
|
|
// Print statistics
|
|
const noteTypeCount: Record<string, number> = {};
|
|
const labelCount: Record<string, number> = {};
|
|
let totalRelations = 0;
|
|
let protectedCount = 0;
|
|
let archivedCount = 0;
|
|
|
|
for (const note of allNotes) {
|
|
// Count note types
|
|
noteTypeCount[note.type] = (noteTypeCount[note.type] || 0) + 1;
|
|
|
|
// Count labels
|
|
for (const attr of note.getOwnedAttributes()) {
|
|
if (attr.type === "label") {
|
|
labelCount[attr.name] = (labelCount[attr.name] || 0) + 1;
|
|
if (attr.name === "archived") archivedCount++;
|
|
} else if (attr.type === "relation") {
|
|
totalRelations++;
|
|
}
|
|
}
|
|
|
|
if (note.isProtected) protectedCount++;
|
|
}
|
|
|
|
console.log("Note Type Distribution:");
|
|
for (const [type, count] of Object.entries(noteTypeCount).sort((a, b) => b[1] - a[1])) {
|
|
console.log(` ${type}: ${count}`);
|
|
}
|
|
|
|
console.log("\nTop 10 Label Names:");
|
|
const sortedLabels = Object.entries(labelCount)
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 10);
|
|
for (const [name, count] of sortedLabels) {
|
|
console.log(` ${name}: ${count}`);
|
|
}
|
|
|
|
console.log("\nOther Statistics:");
|
|
console.log(` Total relations: ${totalRelations}`);
|
|
console.log(` Protected notes: ${protectedCount}`);
|
|
console.log(` Archived notes: ${archivedCount}`);
|
|
console.log("");
|
|
console.log("You can find all generated notes under the 'Stress Test Notes' note in the tree.");
|
|
}
|
|
|
|
// Run the script
|
|
main().catch((error) => {
|
|
console.error("Error during stress test population:");
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|