Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-11-03 20:06:08 +01:00
Code Issues Packages Projects Releases Activity

significantly faster sync with transactions

Browse Source
This commit is contained in:
zadam
2020-04-04 21:49:57 +02:00
parent ae50c9847d
commit b1bed18331
4 changed files with 31 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/routes/api/sync.js
View File

@@ -126,7 +126,7 @@ async function getChanged(req) {
}; };
if (ret.syncs.length > 0) { if (ret.syncs.length > 0) {
log.info(`Returning ${ret.syncs.length} in ${Date.now() - startTime}ms`); log.info(`Returning ${ret.syncs.length} sync records in ${Date.now() - startTime}ms`);
} }
return ret; return ret;

3
src/services/sql_init.js
View File

@@ -18,6 +18,9 @@ async function createConnection() {
const dbConnection = new Promise(async (resolve, reject) => { const dbConnection = new Promise(async (resolve, reject) => {
// no need to create new connection now since DB stays the same all the time // no need to create new connection now since DB stays the same all the time
const db = await createConnection(); const db = await createConnection();
db.run('PRAGMA journal_mode = WAL;');
sql.setDbConnection(db); sql.setDbConnection(db);
resolve(); resolve();

16
src/services/sync.js
View File

@@ -145,8 +145,7 @@ async function pullSync(syncContext) {
break; break;
} }
log.info(`Pulled ${rows.length} changes from ${changesUri} in ${Date.now() - startDate}ms`); await sql.transactional(async () => {
for (const {sync, entity} of rows) { for (const {sync, entity} of rows) {
if (!sourceIdService.isLocalSourceId(sync.sourceId)) { if (!sourceIdService.isLocalSourceId(sync.sourceId)) {
if (appliedPulls === 0 && sync.entity !== 'recent_notes') { // send only for first if (appliedPulls === 0 && sync.entity !== 'recent_notes') { // send only for first
@@ -155,19 +154,16 @@ async function pullSync(syncContext) {
appliedPulls++; appliedPulls++;
} }
const startTime = Date.now(); await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
const updated = await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
if (updated) {
log.info(`Updated ${sync.entityName} ${sync.entityId} in ${Date.now() - startTime}ms`);
}
} }
stats.outstandingPulls = resp.maxSyncId - sync.id; stats.outstandingPulls = resp.maxSyncId - sync.id;
} }
await setLastSyncedPull(rows[rows.length - 1].sync.id); await setLastSyncedPull(rows[rows.length - 1].sync.id);
});
log.info(`Pulled and updated ${rows.length} changes from ${changesUri} in ${Date.now() - startDate}ms`);
} }
if (appliedPulls > 0) { if (appliedPulls > 0) {
@@ -257,8 +253,6 @@ async function checkContentHash(syncContext) {
await syncTableService.addEntitySyncsForSector(entityName, entityPrimaryKey, sector); await syncTableService.addEntitySyncsForSector(entityName, entityPrimaryKey, sector);
await syncRequest(syncContext, 'POST', `/api/sync/queue-sector/${entityName}/${sector}`); await syncRequest(syncContext, 'POST', `/api/sync/queue-sector/${entityName}/${sector}`);
log.info(`Added sector ${sector} of ${entityName} to sync queue.`);
} }
return failedChecks.length > 0; return failedChecks.length > 0;

6
src/services/sync_table.js
View File

@@ -33,6 +33,9 @@ async function addEntitySync(entityName, entityId, sourceId, isSynced) {
} }
async function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) { async function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) {
const startTime = Date.now();
await sql.transactional(async () => {
const entityIds = await sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]); const entityIds = await sql.getColumn(`SELECT ${entityPrimaryKey} FROM ${entityName} WHERE SUBSTR(${entityPrimaryKey}, 1, 1) = ?`, [sector]);
for (const entityId of entityIds) { for (const entityId of entityIds) {
@@ -46,6 +49,9 @@ async function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) {
await insertEntitySync(entityName, entityId, 'content-check', true); await insertEntitySync(entityName, entityId, 'content-check', true);
} }
});
log.info(`Added sector ${sector} of ${entityName} to sync queue in ${Date.now() - startTime}ms.`);
} }
async function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) { async function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) {