mirror of
https://github.com/zadam/trilium.git
synced 2025-11-16 18:25:51 +01:00
feat(dev): add new stress test population script
This commit is contained in:
perfectra1n
507
scripts/stress-test-populate.ts
Normal file
507
scripts/stress-test-populate.ts
Normal file
@@ -0,0 +1,507 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Stress Test Database Population Script
|
||||
*
|
||||
* This script populates the Trilium database with a large number of diverse notes
|
||||
* for performance testing, search testing, and stress testing purposes.
|
||||
*
|
||||
* Usage:
|
||||
* pnpm tsx scripts/stress-test-populate.ts [options]
|
||||
*
|
||||
* Options:
|
||||
* --notes=N Number of notes to create (default: 5000)
|
||||
* --depth=N Maximum hierarchy depth (default: 10)
|
||||
* --max-relations=N Maximum relations per note (default: 10)
|
||||
* --max-labels=N Maximum labels per note (default: 8)
|
||||
* --help Show this help message
|
||||
*
|
||||
* Note: This script requires an existing Trilium database. Run Trilium at least once
|
||||
* before running this script to initialize the database.
|
||||
*/
|
||||
|
||||
// Set up environment variables like the server does
|
||||
process.env.TRILIUM_ENV = "dev";
|
||||
process.env.TRILIUM_DATA_DIR = process.env.TRILIUM_DATA_DIR || "trilium-data";
|
||||
|
||||
import { initializeTranslations } from "../apps/server/src/services/i18n.js";
|
||||
import BNote from "../apps/server/src/becca/entities/bnote.js";
|
||||
import BBranch from "../apps/server/src/becca/entities/bbranch.js";
|
||||
import BAttribute from "../apps/server/src/becca/entities/battribute.js";
|
||||
import becca from "../apps/server/src/becca/becca.js";
|
||||
import { NoteBuilder, id, note } from "../apps/server/src/test/becca_mocking.js";
|
||||
import type { NoteType } from "@triliumnext/commons";
|
||||
import { dbReady } from "../apps/server/src/services/sql_init.js";
|
||||
|
||||
// Parse command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const config = {
|
||||
noteCount: 5000,
|
||||
maxDepth: 10,
|
||||
maxRelations: 10,
|
||||
maxLabels: 8,
|
||||
};
|
||||
|
||||
for (const arg of args) {
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Stress Test Database Population Script
|
||||
|
||||
This script populates the Trilium database with a large number of diverse notes
|
||||
for performance testing, search testing, and stress testing purposes.
|
||||
|
||||
Usage:
|
||||
pnpm tsx scripts/stress-test-populate.ts [options]
|
||||
|
||||
Options:
|
||||
--notes=N Number of notes to create (default: ${config.noteCount})
|
||||
--depth=N Maximum hierarchy depth (default: ${config.maxDepth})
|
||||
--max-relations=N Maximum relations per note (default: ${config.maxRelations})
|
||||
--max-labels=N Maximum labels per note (default: ${config.maxLabels})
|
||||
--help, -h Show this help message
|
||||
|
||||
Examples:
|
||||
# Use defaults (5000 notes, depth 10)
|
||||
pnpm tsx scripts/stress-test-populate.ts
|
||||
|
||||
# Create 10000 notes with depth 15
|
||||
pnpm tsx scripts/stress-test-populate.ts --notes=10000 --depth=15
|
||||
|
||||
# Smaller test with 1000 notes and depth 5
|
||||
pnpm tsx scripts/stress-test-populate.ts --notes=1000 --depth=5
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const match = arg.match(/--(\w+)=(.+)/);
|
||||
if (match) {
|
||||
const [, key, value] = match;
|
||||
switch (key) {
|
||||
case "notes":
|
||||
config.noteCount = parseInt(value, 10);
|
||||
break;
|
||||
case "depth":
|
||||
config.maxDepth = parseInt(value, 10);
|
||||
break;
|
||||
case "max-relations":
|
||||
config.maxRelations = parseInt(value, 10);
|
||||
break;
|
||||
case "max-labels":
|
||||
config.maxLabels = parseInt(value, 10);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Stress Test Database Population");
|
||||
console.log("================================");
|
||||
console.log(`Target note count: ${config.noteCount}`);
|
||||
console.log(`Maximum depth: ${config.maxDepth}`);
|
||||
console.log(`Maximum relations per note: ${config.maxRelations}`);
|
||||
console.log(`Maximum labels per note: ${config.maxLabels}`);
|
||||
console.log("");
|
||||
|
||||
// Note type distribution (rough percentages)
|
||||
const NOTE_TYPES: { type: NoteType; mime: string; weight: number }[] = [
|
||||
{ type: "text", mime: "text/html", weight: 50 },
|
||||
{ type: "code", mime: "text/javascript", weight: 15 },
|
||||
{ type: "code", mime: "text/x-python", weight: 10 },
|
||||
{ type: "code", mime: "application/json", weight: 5 },
|
||||
{ type: "mermaid", mime: "text/mermaid", weight: 5 },
|
||||
{ type: "book", mime: "text/html", weight: 5 },
|
||||
{ type: "render", mime: "text/html", weight: 3 },
|
||||
{ type: "relationMap", mime: "application/json", weight: 2 },
|
||||
{ type: "search", mime: "application/json", weight: 2 },
|
||||
{ type: "canvas", mime: "application/json", weight: 2 },
|
||||
{ type: "doc", mime: "text/html", weight: 1 },
|
||||
];
|
||||
|
||||
// Sample content generators
|
||||
const LOREM_IPSUM = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.`;
|
||||
|
||||
const CODE_SAMPLES = {
|
||||
"text/javascript": `function fibonacci(n) {
|
||||
if (n <= 1) return n;
|
||||
return fibonacci(n - 1) + fibonacci(n - 2);
|
||||
}
|
||||
|
||||
console.log(fibonacci(10));`,
|
||||
|
||||
"text/x-python": `def quicksort(arr):
|
||||
if len(arr) <= 1:
|
||||
return arr
|
||||
pivot = arr[len(arr) // 2]
|
||||
left = [x for x in arr if x < pivot]
|
||||
middle = [x for x in arr if x == pivot]
|
||||
right = [x for x in arr if x > pivot]
|
||||
return quicksort(left) + middle + quicksort(right)
|
||||
|
||||
print(quicksort([3, 6, 8, 10, 1, 2, 1]))`,
|
||||
|
||||
"application/json": `{
|
||||
"name": "example",
|
||||
"version": "1.0.0",
|
||||
"description": "A sample JSON document",
|
||||
"keywords": ["example", "test", "stress"]
|
||||
}`,
|
||||
};
|
||||
|
||||
const MERMAID_SAMPLE = `graph TD
|
||||
A[Start] --> B{Decision}
|
||||
B -->|Yes| C[Process]
|
||||
B -->|No| D[Alternative]
|
||||
C --> E[End]
|
||||
D --> E`;
|
||||
|
||||
// Common label names and value patterns
|
||||
const LABEL_NAMES = [
|
||||
"priority", "status", "category", "tag", "project", "version", "author",
|
||||
"reviewed", "archived", "published", "draft", "language", "framework",
|
||||
"difficulty", "rating", "year", "month", "country", "city", "department"
|
||||
];
|
||||
|
||||
const LABEL_VALUES = {
|
||||
priority: ["high", "medium", "low", "critical"],
|
||||
status: ["active", "completed", "pending", "archived", "draft"],
|
||||
category: ["personal", "work", "reference", "project", "research"],
|
||||
rating: ["1", "2", "3", "4", "5"],
|
||||
difficulty: ["beginner", "intermediate", "advanced", "expert"],
|
||||
language: ["javascript", "python", "typescript", "rust", "go", "java"],
|
||||
framework: ["react", "vue", "angular", "express", "django", "flask"],
|
||||
};
|
||||
|
||||
// Relation names
|
||||
const RELATION_NAMES = [
|
||||
"relatedTo", "dependsOn", "references", "implements", "extends",
|
||||
"baseOn", "contains", "partOf", "author", "reviewer", "assignedTo",
|
||||
"linkedWith", "similarTo", "contradicts", "supports"
|
||||
];
|
||||
|
||||
// Title prefixes for different categories
|
||||
const TITLE_PREFIXES = [
|
||||
"Documentation", "Tutorial", "Guide", "Reference", "API", "Concept",
|
||||
"Example", "Pattern", "Architecture", "Design", "Implementation",
|
||||
"Analysis", "Research", "Note", "Idea", "Project", "Task", "Feature",
|
||||
"Bug", "Issue", "Discussion", "Meeting", "Review", "Proposal", "Spec"
|
||||
];
|
||||
|
||||
const TITLE_SUBJECTS = [
|
||||
"Authentication", "Database", "API", "Frontend", "Backend", "Security",
|
||||
"Performance", "Testing", "Deployment", "Configuration", "Monitoring",
|
||||
"Logging", "Caching", "Scaling", "Optimization", "Refactoring",
|
||||
"Integration", "Migration", "Upgrade", "Architecture", "Infrastructure"
|
||||
];
|
||||
|
||||
/**
|
||||
* Select a random item from array based on weights
|
||||
*/
|
||||
function weightedRandom<T extends { weight: number }>(items: T[]): T {
|
||||
const totalWeight = items.reduce((sum, item) => sum + item.weight, 0);
|
||||
let random = Math.random() * totalWeight;
|
||||
|
||||
for (const item of items) {
|
||||
random -= item.weight;
|
||||
if (random <= 0) {
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
return items[items.length - 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate random integer between min and max (inclusive)
|
||||
*/
|
||||
function randomInt(min: number, max: number): number {
|
||||
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a random title
|
||||
*/
|
||||
function generateTitle(index: number): string {
|
||||
if (Math.random() < 0.3) {
|
||||
// Use structured title
|
||||
const prefix = TITLE_PREFIXES[randomInt(0, TITLE_PREFIXES.length - 1)];
|
||||
const subject = TITLE_SUBJECTS[randomInt(0, TITLE_SUBJECTS.length - 1)];
|
||||
return `${prefix}: ${subject} #${index}`;
|
||||
} else {
|
||||
// Use simple title
|
||||
return `Note ${index}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate content based on note type
|
||||
*/
|
||||
function generateContent(type: NoteType, mime: string): string {
|
||||
if (type === "code" && CODE_SAMPLES[mime as keyof typeof CODE_SAMPLES]) {
|
||||
return CODE_SAMPLES[mime as keyof typeof CODE_SAMPLES];
|
||||
} else if (type === "mermaid") {
|
||||
return MERMAID_SAMPLE;
|
||||
} else if (type === "text" || type === "book" || type === "doc") {
|
||||
// Generate multiple paragraphs
|
||||
const paragraphs = randomInt(1, 5);
|
||||
return Array(paragraphs).fill(LOREM_IPSUM).join("\n\n");
|
||||
} else if (mime === "application/json") {
|
||||
return CODE_SAMPLES["application/json"];
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate random labels for a note
|
||||
*/
|
||||
function generateLabels(noteBuilder: NoteBuilder, count: number): void {
|
||||
const labelsToAdd = Math.min(count, randomInt(0, config.maxLabels));
|
||||
|
||||
for (let i = 0; i < labelsToAdd; i++) {
|
||||
const labelName = LABEL_NAMES[randomInt(0, LABEL_NAMES.length - 1)];
|
||||
let labelValue = "";
|
||||
|
||||
// Use predefined values if available
|
||||
if (LABEL_VALUES[labelName as keyof typeof LABEL_VALUES]) {
|
||||
const values = LABEL_VALUES[labelName as keyof typeof LABEL_VALUES];
|
||||
labelValue = values[randomInt(0, values.length - 1)];
|
||||
} else {
|
||||
labelValue = `value${randomInt(1, 100)}`;
|
||||
}
|
||||
|
||||
const isInheritable = Math.random() < 0.2; // 20% chance of inheritable
|
||||
noteBuilder.label(labelName, labelValue, isInheritable);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate random relations for a note
|
||||
*/
|
||||
function generateRelations(
|
||||
noteBuilder: NoteBuilder,
|
||||
allNotes: BNote[],
|
||||
maxRelations: number
|
||||
): void {
|
||||
if (allNotes.length === 0) return;
|
||||
|
||||
const relationsToAdd = Math.min(
|
||||
maxRelations,
|
||||
randomInt(0, config.maxRelations)
|
||||
);
|
||||
|
||||
for (let i = 0; i < relationsToAdd; i++) {
|
||||
const relationName = RELATION_NAMES[randomInt(0, RELATION_NAMES.length - 1)];
|
||||
const targetNote = allNotes[randomInt(0, allNotes.length - 1)];
|
||||
|
||||
// Avoid self-relations
|
||||
if (targetNote.noteId !== noteBuilder.note.noteId) {
|
||||
noteBuilder.relation(relationName, targetNote);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a note with random attributes
|
||||
*/
|
||||
function createRandomNote(
|
||||
index: number,
|
||||
allNotes: BNote[]
|
||||
): NoteBuilder {
|
||||
const noteType = weightedRandom(NOTE_TYPES);
|
||||
const title = generateTitle(index);
|
||||
|
||||
const noteBuilder = note(title, {
|
||||
noteId: id(),
|
||||
type: noteType.type,
|
||||
mime: noteType.mime,
|
||||
});
|
||||
|
||||
// Set content
|
||||
const content = generateContent(noteType.type, noteType.mime);
|
||||
if (content) {
|
||||
noteBuilder.note.setContent(content, { forceSave: true });
|
||||
}
|
||||
|
||||
// Add labels
|
||||
generateLabels(noteBuilder, randomInt(0, config.maxLabels));
|
||||
|
||||
// Add relations (limit based on available notes)
|
||||
const maxPossibleRelations = Math.min(
|
||||
config.maxRelations,
|
||||
Math.floor(allNotes.length / 10) // Limit to avoid too dense graphs
|
||||
);
|
||||
generateRelations(noteBuilder, allNotes, maxPossibleRelations);
|
||||
|
||||
// 5% chance of protected note
|
||||
if (Math.random() < 0.05) {
|
||||
noteBuilder.note.isProtected = true;
|
||||
}
|
||||
|
||||
// 10% chance of archived note
|
||||
if (Math.random() < 0.1) {
|
||||
noteBuilder.label("archived", "", true);
|
||||
}
|
||||
|
||||
return noteBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create notes recursively to build hierarchy
|
||||
*/
|
||||
function createNotesRecursively(
|
||||
parent: NoteBuilder,
|
||||
depth: number,
|
||||
targetCount: number,
|
||||
allNotes: BNote[]
|
||||
): number {
|
||||
let created = 0;
|
||||
|
||||
if (depth >= config.maxDepth || targetCount <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Determine how many children at this level
|
||||
// Decrease children count as depth increases to create pyramid structure
|
||||
const maxChildrenAtDepth = Math.max(1, Math.floor(20 / (depth + 1)));
|
||||
const childrenCount = Math.min(
|
||||
targetCount,
|
||||
randomInt(1, maxChildrenAtDepth)
|
||||
);
|
||||
|
||||
for (let i = 0; i < childrenCount && created < targetCount; i++) {
|
||||
const noteBuilder = createRandomNote(allNotes.length + 1, allNotes);
|
||||
parent.child(noteBuilder);
|
||||
allNotes.push(noteBuilder.note);
|
||||
created++;
|
||||
|
||||
// Log progress every 100 notes
|
||||
if (allNotes.length % 100 === 0) {
|
||||
console.log(` Created ${allNotes.length} notes...`);
|
||||
}
|
||||
|
||||
// Recursively create children
|
||||
const remainingForSubtree = Math.floor((targetCount - created) / (childrenCount - i));
|
||||
const createdInSubtree = createNotesRecursively(
|
||||
noteBuilder,
|
||||
depth + 1,
|
||||
remainingForSubtree,
|
||||
allNotes
|
||||
);
|
||||
created += createdInSubtree;
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main execution
|
||||
*/
|
||||
async function main() {
|
||||
console.log("Initializing translations...");
|
||||
await initializeTranslations();
|
||||
|
||||
console.log("Initializing database connection...");
|
||||
|
||||
// Wait for database to be ready (initialized by sql.ts import)
|
||||
await dbReady;
|
||||
|
||||
console.log("Loading becca (backend cache)...");
|
||||
|
||||
// Dynamically import becca_loader to ensure proper initialization order
|
||||
const { beccaLoaded } = await import("../apps/server/src/becca/becca_loader.js");
|
||||
await beccaLoaded;
|
||||
|
||||
const rootNote = becca.getNote("root");
|
||||
if (!rootNote) {
|
||||
throw new Error("Root note not found!");
|
||||
}
|
||||
|
||||
// Create a container note for all stress test notes
|
||||
const containerNote = note("Stress Test Notes", {
|
||||
noteId: id(),
|
||||
type: "book",
|
||||
mime: "text/html",
|
||||
});
|
||||
containerNote.note.setContent(
|
||||
`<p>This note contains ${config.noteCount} notes generated for stress testing.</p>` +
|
||||
`<p>Generated on: ${new Date().toISOString()}</p>` +
|
||||
`<p>Configuration: depth=${config.maxDepth}, maxRelations=${config.maxRelations}, maxLabels=${config.maxLabels}</p>`,
|
||||
{ forceSave: true }
|
||||
);
|
||||
|
||||
const rootBuilder = new NoteBuilder(rootNote);
|
||||
rootBuilder.child(containerNote);
|
||||
|
||||
console.log("\nCreating notes...");
|
||||
const startTime = Date.now();
|
||||
|
||||
const allNotes: BNote[] = [containerNote.note];
|
||||
|
||||
// Create notes recursively
|
||||
const created = createNotesRecursively(
|
||||
containerNote,
|
||||
0,
|
||||
config.noteCount - 1, // -1 because we already created container
|
||||
allNotes
|
||||
);
|
||||
|
||||
const endTime = Date.now();
|
||||
const duration = (endTime - startTime) / 1000;
|
||||
|
||||
console.log("\n================================");
|
||||
console.log("Stress Test Population Complete!");
|
||||
console.log("================================");
|
||||
console.log(`Total notes created: ${allNotes.length}`);
|
||||
console.log(`Time taken: ${duration.toFixed(2)} seconds`);
|
||||
console.log(`Notes per second: ${(allNotes.length / duration).toFixed(2)}`);
|
||||
console.log(`Container note ID: ${containerNote.note.noteId}`);
|
||||
console.log("");
|
||||
|
||||
// Print statistics
|
||||
const noteTypeCount: Record<string, number> = {};
|
||||
const labelCount: Record<string, number> = {};
|
||||
let totalRelations = 0;
|
||||
let protectedCount = 0;
|
||||
let archivedCount = 0;
|
||||
|
||||
for (const note of allNotes) {
|
||||
// Count note types
|
||||
noteTypeCount[note.type] = (noteTypeCount[note.type] || 0) + 1;
|
||||
|
||||
// Count labels
|
||||
for (const attr of note.getOwnedAttributes()) {
|
||||
if (attr.type === "label") {
|
||||
labelCount[attr.name] = (labelCount[attr.name] || 0) + 1;
|
||||
if (attr.name === "archived") archivedCount++;
|
||||
} else if (attr.type === "relation") {
|
||||
totalRelations++;
|
||||
}
|
||||
}
|
||||
|
||||
if (note.isProtected) protectedCount++;
|
||||
}
|
||||
|
||||
console.log("Note Type Distribution:");
|
||||
for (const [type, count] of Object.entries(noteTypeCount).sort((a, b) => b[1] - a[1])) {
|
||||
console.log(` ${type}: ${count}`);
|
||||
}
|
||||
|
||||
console.log("\nTop 10 Label Names:");
|
||||
const sortedLabels = Object.entries(labelCount)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10);
|
||||
for (const [name, count] of sortedLabels) {
|
||||
console.log(` ${name}: ${count}`);
|
||||
}
|
||||
|
||||
console.log("\nOther Statistics:");
|
||||
console.log(` Total relations: ${totalRelations}`);
|
||||
console.log(` Protected notes: ${protectedCount}`);
|
||||
console.log(` Archived notes: ${archivedCount}`);
|
||||
console.log("");
|
||||
console.log("You can find all generated notes under the 'Stress Test Notes' note in the tree.");
|
||||
}
|
||||
|
||||
// Run the script
|
||||
main().catch((error) => {
|
||||
console.error("Error during stress test population:");
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user