Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-31 02:16:05 +01:00
Code Issues Packages Projects Releases Activity

blob erasure is not synced, need to clean them up before each content hash check

Browse Source
This commit is contained in:
zadam
2023-07-27 23:57:12 +02:00
parent 8edb5428e5
commit ce3834eb9e
8 changed files with 201 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
src/services/notes.js
View File

@@ -855,158 +855,6 @@ async function asyncPostProcessContent(note, content) {
scanForLinks(note, content);
}
function eraseNotes(noteIdsToErase) {
if (noteIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM notes WHERE noteId IN (???)`, noteIdsToErase);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'notes' AND entityId IN (???)`, noteIdsToErase));
// we also need to erase all "dependent" entities of the erased notes
const branchIdsToErase = sql.getManyRows(`SELECT branchId FROM branches WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.branchId);
eraseBranches(branchIdsToErase);
const attributeIdsToErase = sql.getManyRows(`SELECT attributeId FROM attributes WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.attributeId);
eraseAttributes(attributeIdsToErase);
const revisionIdsToErase = sql.getManyRows(`SELECT revisionId FROM revisions WHERE noteId IN (???)`, noteIdsToErase)
.map(row => row.revisionId);
revisionService.eraseRevisions(revisionIdsToErase);
log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
}
function setEntityChangesAsErased(entityChanges) {
for (const ec of entityChanges) {
ec.isErased = true;
entityChangesService.addEntityChange(ec);
}
}
function eraseBranches(branchIdsToErase) {
if (branchIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM branches WHERE branchId IN (???)`, branchIdsToErase);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'branches' AND entityId IN (???)`, branchIdsToErase));
log.info(`Erased branches: ${JSON.stringify(branchIdsToErase)}`);
}
function eraseAttributes(attributeIdsToErase) {
if (attributeIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM attributes WHERE attributeId IN (???)`, attributeIdsToErase);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'attributes' AND entityId IN (???)`, attributeIdsToErase));
log.info(`Erased attributes: ${JSON.stringify(attributeIdsToErase)}`);
}
function eraseAttachments(attachmentIdsToErase) {
if (attachmentIdsToErase.length === 0) {
return;
}
sql.executeMany(`DELETE FROM attachments WHERE attachmentId IN (???)`, attachmentIdsToErase);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'attachments' AND entityId IN (???)`, attachmentIdsToErase));
log.info(`Erased attachments: ${JSON.stringify(attachmentIdsToErase)}`);
}
function eraseUnusedBlobs() {
// this method is rather defense in depth - in normal operation, the unused blobs should be erased immediately
// after getting unused (handled in entity._setContent())
const unusedBlobIds = sql.getColumn(`
SELECT blobs.blobId
FROM blobs
LEFT JOIN notes ON notes.blobId = blobs.blobId
LEFT JOIN attachments ON attachments.blobId = blobs.blobId
LEFT JOIN revisions ON revisions.blobId = blobs.blobId
WHERE notes.noteId IS NULL
AND attachments.attachmentId IS NULL
AND revisions.revisionId IS NULL`);
if (unusedBlobIds.length === 0) {
return;
}
sql.executeMany(`DELETE FROM blobs WHERE blobId IN (???)`, unusedBlobIds);
setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'blobs' AND entityId IN (???)`, unusedBlobIds));
log.info(`Erased unused blobs: ${JSON.stringify(unusedBlobIds)}`);
}
function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
// this is important also so that the erased entity changes are sent to the connected clients
sql.transactional(() => {
if (eraseEntitiesAfterTimeInSeconds === null) {
eraseEntitiesAfterTimeInSeconds = optionService.getOptionInt('eraseEntitiesAfterTimeInSeconds');
}
const cutoffDate = new Date(Date.now() - eraseEntitiesAfterTimeInSeconds * 1000);
const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
eraseNotes(noteIdsToErase);
const branchIdsToErase = sql.getColumn("SELECT branchId FROM branches WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
eraseBranches(branchIdsToErase);
const attributeIdsToErase = sql.getColumn("SELECT attributeId FROM attributes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
eraseAttributes(attributeIdsToErase);
const attachmentIdsToErase = sql.getColumn("SELECT attachmentId FROM attachments WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateTimeStr(cutoffDate)]);
eraseAttachments(attachmentIdsToErase);
eraseUnusedBlobs();
});
}
function eraseNotesWithDeleteId(deleteId) {
const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND deleteId = ?", [deleteId]);
eraseNotes(noteIdsToErase);
const branchIdsToErase = sql.getColumn("SELECT branchId FROM branches WHERE isDeleted = 1 AND deleteId = ?", [deleteId]);
eraseBranches(branchIdsToErase);
const attributeIdsToErase = sql.getColumn("SELECT attributeId FROM attributes WHERE isDeleted = 1 AND deleteId = ?", [deleteId]);
eraseAttributes(attributeIdsToErase);
const attachmentIdsToErase = sql.getColumn("SELECT attachmentId FROM attachments WHERE isDeleted = 1 AND deleteId = ?", [deleteId]);
eraseAttachments(attachmentIdsToErase);
eraseUnusedBlobs();
}
function eraseDeletedNotesNow() {
eraseDeletedEntities(0);
}
function eraseUnusedAttachmentsNow() {
eraseScheduledAttachments(0);
}
// all keys should be replaced by the corresponding values
function replaceByMap(str, mapObj) {
const re = new RegExp(Object.keys(mapObj).join("|"),"g");
@@ -1138,26 +986,6 @@ function getNoteIdMapping(origNote) {
return noteIdMapping;
}
function eraseScheduledAttachments(eraseUnusedAttachmentsAfterSeconds = null) {
if (eraseUnusedAttachmentsAfterSeconds === null) {
eraseUnusedAttachmentsAfterSeconds = optionService.getOptionInt('eraseUnusedAttachmentsAfterSeconds');
}
const cutOffDate = dateUtils.utcDateTimeStr(new Date(Date.now() - (eraseUnusedAttachmentsAfterSeconds * 1000)));
const attachmentIdsToErase = sql.getColumn('SELECT attachmentId FROM attachments WHERE utcDateScheduledForErasureSince < ?', [cutOffDate]);
eraseAttachments(attachmentIdsToErase);
}
sqlInit.dbReady.then(() => {
// first cleanup kickoff 5 minutes after startup
setTimeout(cls.wrap(() => eraseDeletedEntities()), 5 * 60 * 1000);
setTimeout(cls.wrap(() => eraseScheduledAttachments()), 6 * 60 * 1000);
setInterval(cls.wrap(() => eraseDeletedEntities()), 4 * 3600 * 1000);
setInterval(cls.wrap(() => eraseScheduledAttachments()), 3600 * 1000);
});
module.exports = {
createNewNote,
createNewNoteWithTarget,
@@ -1168,9 +996,6 @@ module.exports = {
duplicateSubtreeWithoutRoot,
getUndeletedParentBranchIds,
triggerNoteTitleChanged,
eraseDeletedNotesNow,
eraseUnusedAttachmentsNow,
eraseNotesWithDeleteId,
saveRevisionIfNeeded,
downloadImages,
asyncPostProcessContent