Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-11-12 16:25:51 +01:00
Code Issues Packages Projects Releases Activity

Revert "feat(search): further improve fts search"

Browse Source 
This reverts commit 7c5553bd4b.
This commit is contained in:
perf3ct
2025-09-02 19:24:45 +00:00
parent 8572f82e0a
commit f529ddc601
3 changed files with 57 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
apps/server/src/migrations/0234__add_fts5_search.ts
View File

@@ -17,9 +17,18 @@ export default function addFTS5SearchAndPerformanceIndexes() {
// Create FTS5 virtual table with porter tokenizer
log.info("Creating FTS5 virtual table...");
// Note: Transaction-safe pragmas are excluded here.
// They should be set at database initialization, not during migration.
// Set optimal SQLite pragmas for FTS5 operations with millions of notes
sql.executeScript(`
-- Memory and performance pragmas for large-scale FTS operations
PRAGMA cache_size = -262144; -- 256MB cache for better performance
PRAGMA temp_store = MEMORY; -- Use RAM for temporary storage
PRAGMA mmap_size = 536870912; -- 512MB memory-mapped I/O
PRAGMA synchronous = NORMAL; -- Faster writes with good safety
PRAGMA journal_mode = WAL; -- Write-ahead logging for better concurrency
PRAGMA wal_autocheckpoint = 1000; -- Auto-checkpoint every 1000 pages
PRAGMA automatic_index = ON; -- Allow automatic indexes
PRAGMA threads = 4; -- Use multiple threads for sorting
-- Drop existing FTS tables if they exist
DROP TABLE IF EXISTS notes_fts;
DROP TABLE IF EXISTS notes_fts_trigram;
@@ -61,6 +70,11 @@ export default function addFTS5SearchAndPerformanceIndexes() {
// Process in optimized batches using a prepared statement
sql.transactional(() => {
// Prepare statement for batch inserts
const insertStmt = sql.prepare(`
INSERT OR REPLACE INTO notes_fts (noteId, title, content)
VALUES (?, ?, ?)
`);
let offset = 0;
while (offset < totalNotes) {
@@ -84,12 +98,9 @@ export default function addFTS5SearchAndPerformanceIndexes() {
break;
}
// Batch insert
// Batch insert using prepared statement
for (const note of notesBatch) {
sql.execute(
`INSERT OR REPLACE INTO notes_fts (noteId, title, content) VALUES (?, ?, ?)`,
[note.noteId, note.title, note.content]
);
insertStmt.run(note.noteId, note.title, note.content);
}
offset += notesBatch.length;
@@ -105,6 +116,9 @@ export default function addFTS5SearchAndPerformanceIndexes() {
break;
}
}
// Finalize prepared statement
insertStmt.finalize();
});
} catch (error) {
log.error(`Failed to populate FTS index: ${error}`);
@@ -229,11 +243,32 @@ export default function addFTS5SearchAndPerformanceIndexes() {
log.info("Optimizing FTS5 index...");
sql.execute(`INSERT INTO notes_fts(notes_fts) VALUES('optimize')`);
// Run ANALYZE on FTS-related tables (these are safe within transactions)
log.info("Analyzing FTS tables for query optimization...");
sql.execute(`ANALYZE notes_fts`);
sql.execute(`ANALYZE notes`);
sql.execute(`ANALYZE blobs`);
// Set comprehensive SQLite pragmas optimized for millions of notes
log.info("Configuring SQLite pragmas for large-scale FTS performance...");
sql.executeScript(`
-- Memory Management (Critical for large databases)
PRAGMA cache_size = -262144; -- 256MB cache (was 50MB) - critical for FTS performance
PRAGMA temp_store = MEMORY; -- Use memory for temporary tables and indices
PRAGMA mmap_size = 536870912; -- 512MB memory-mapped I/O for better read performance
-- Write Optimization (Important for batch operations)
PRAGMA synchronous = NORMAL; -- Balance between safety and performance (was FULL)
PRAGMA journal_mode = WAL; -- Write-Ahead Logging for better concurrency
PRAGMA wal_autocheckpoint = 1000; -- Checkpoint every 1000 pages for memory management
-- Query Optimization (Essential for FTS queries)
PRAGMA automatic_index = ON; -- Allow SQLite to create automatic indexes
PRAGMA optimize; -- Update query planner statistics
-- FTS-Specific Optimizations
PRAGMA threads = 4; -- Use multiple threads for FTS operations (if available)
-- Run comprehensive ANALYZE on all FTS-related tables
ANALYZE notes_fts;
ANALYZE notes;
ANALYZE blobs;
`);
log.info("FTS5 migration completed successfully");
}

3
apps/server/src/routes/api/search.ts
View File

@@ -141,8 +141,7 @@ function syncFtsIndex(req: Request) {
log.info(`FTS sync requested for ${noteIds?.length || 'all'} notes`);
// syncMissingNotes doesn't accept parameters - it syncs all missing notes
const syncedCount = ftsSearchService.syncMissingNotes();
const syncedCount = ftsSearchService.syncMissingNotes(noteIds);
return {
success: true,

78
apps/server/src/services/search/fts_search.ts
View File

@@ -30,7 +30,6 @@ export interface FTSSearchOptions {
limit?: number;
offset?: number;
searchProtected?: boolean;
includeSnippets?: boolean;
}
/**
@@ -290,12 +289,17 @@ class FTSSearchService {
// Insert missing notes using efficient batch processing
sql.transactional(() => {
// Use prepared statement for better performance
const insertStmt = sql.prepare(`
INSERT OR REPLACE INTO notes_fts (noteId, title, content)
VALUES (?, ?, ?)
`);
for (const note of missingNotes) {
sql.execute(
`INSERT OR REPLACE INTO notes_fts (noteId, title, content) VALUES (?, ?, ?)`,
[note.noteId, note.title, note.content]
);
insertStmt.run(note.noteId, note.title, note.content);
}
insertStmt.finalize();
});
log.info(`Synced ${missingNotes.length} missing notes to FTS index`);
@@ -481,70 +485,6 @@ class FTSSearchService {
return { documentCount: 0, indexSize: 0 };
}
}
/**
* Get FTS index statistics (alias for getStatistics for API compatibility)
*/
getIndexStats(): { totalDocuments: number; indexSize: number } {
const stats = this.getStatistics();
return {
totalDocuments: stats.documentCount,
indexSize: stats.indexSize
};
}
/**
* Rebuild the entire FTS index from scratch
*/
rebuildIndex(): void {
if (!this.checkFTS5Availability()) {
throw new FTSNotAvailableError();
}
try {
log.info("Starting FTS index rebuild");
sql.transactional(() => {
// Clear existing index
sql.execute(`DELETE FROM notes_fts`);
// Rebuild from all eligible notes
const notes = sql.getRows<{noteId: string, title: string, content: string}>(`
SELECT n.noteId, n.title, b.content
FROM notes n
LEFT JOIN blobs b ON n.blobId = b.blobId
WHERE n.type IN ('text', 'code', 'mermaid', 'canvas', 'mindMap')
AND n.isDeleted = 0
AND n.isProtected = 0
AND b.content IS NOT NULL
`);
if (notes && notes.length > 0) {
// Process in batches for better performance
const batchSize = FTS_CONFIG.BATCH_SIZE;
for (let i = 0; i < notes.length; i += batchSize) {
const batch = notes.slice(i, i + batchSize);
for (const note of batch) {
sql.execute(
`INSERT INTO notes_fts (noteId, title, content) VALUES (?, ?, ?)`,
[note.noteId, note.title, note.content]
);
}
}
log.info(`Rebuilt FTS index with ${notes.length} notes`);
}
});
// Optimize after rebuild
this.optimizeIndex();
} catch (error) {
log.error(`Error rebuilding FTS index: ${error}`);
throw new FTSError(`Failed to rebuild FTS index: ${error}`, 'FTS_REBUILD_ERROR');
}
}
}
// Export singleton instance