Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-30 18:05:55 +01:00
Code Issues Packages Projects Releases Activity

image sync

Browse Source
This commit is contained in:
azivner
2018-01-06 15:56:00 -05:00
parent 91cf090820
commit 784cd62df1
10 changed files with 201 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
package-lock.json generated
View File

@@ -4167,6 +4167,34 @@
"es5-ext": "0.10.35" "es5-ext": "0.10.35"
} }
}, },
"exec-buffer": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/exec-buffer/-/exec-buffer-3.2.0.tgz",
"integrity": "sha512-wsiD+2Tp6BWHoVv3B+5Dcx6E7u5zky+hUwOHjuH2hKSLR3dvRmX8fk8UD8uqQixHs4Wk6eDmiegVrMPjKj7wpA==",
"requires": {
"execa": "0.7.0",
"p-finally": "1.0.0",
"pify": "3.0.0",
"rimraf": "2.6.2",
"tempfile": "2.0.0"
},
"dependencies": {
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
},
"tempfile": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/tempfile/-/tempfile-2.0.0.tgz",
"integrity": "sha1-awRGhWqbERTRhW/8vlCczLCXcmU=",
"requires": {
"temp-dir": "1.0.0",
"uuid": "3.1.0"
}
}
}
},
"exec-series": { "exec-series": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/exec-series/-/exec-series-1.0.3.tgz", "resolved": "https://registry.npmjs.org/exec-series/-/exec-series-1.0.3.tgz",
@@ -4180,7 +4208,6 @@
"version": "0.7.0", "version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
"integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
"dev": true,
"requires": { "requires": {
"cross-spawn": "5.1.0", "cross-spawn": "5.1.0",
"get-stream": "3.0.0", "get-stream": "3.0.0",
@@ -5461,6 +5488,16 @@
} }
} }
}, },
"imagemin-pngquant": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/imagemin-pngquant/-/imagemin-pngquant-5.0.1.tgz",
"integrity": "sha1-2KMp2lU6+iJrEc5i3r4Lfje0OeY=",
"requires": {
"exec-buffer": "3.2.0",
"is-png": "1.1.0",
"pngquant-bin": "3.1.1"
}
},
"import-lazy": { "import-lazy": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz",
@@ -5826,6 +5863,11 @@
"integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
"dev": true "dev": true
}, },
"is-png": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-png/-/is-png-1.1.0.tgz",
"integrity": "sha1-1XSxK/J1wDUEVVcLDltXqwYgd84="
},
"is-posix-bracket": { "is-posix-bracket": {
"version": "0.1.1", "version": "0.1.1",
"resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz", "resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz",
@@ -7826,6 +7868,16 @@
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.3.1.tgz", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.3.1.tgz",
"integrity": "sha512-ggXCTsqHRIsGMkHlCEhbHhUmNTA2r1lpkE0NL4Q9S8spkXbm4vE9TVmPso2AGYn90Gltdz8W5CyzhcIGg2Gejg==" "integrity": "sha512-ggXCTsqHRIsGMkHlCEhbHhUmNTA2r1lpkE0NL4Q9S8spkXbm4vE9TVmPso2AGYn90Gltdz8W5CyzhcIGg2Gejg=="
}, },
"pngquant-bin": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/pngquant-bin/-/pngquant-bin-3.1.1.tgz",
"integrity": "sha1-0STZinWpSH9AwWQLTb/Lsr1aH9E=",
"requires": {
"bin-build": "2.2.0",
"bin-wrapper": "3.0.2",
"logalot": "2.1.0"
}
},
"postcss": { "postcss": {
"version": "5.2.18", "version": "5.2.18",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz", "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
@@ -10309,6 +10361,11 @@
} }
} }
}, },
"temp-dir": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz",
"integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0="
},
"tempfile": { "tempfile": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz", "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz",

1
package.json
View File

@@ -35,6 +35,7 @@
"html": "^1.0.0", "html": "^1.0.0",
"imagemin": "^5.3.1", "imagemin": "^5.3.1",
"imagemin-mozjpeg": "^7.0.0", "imagemin-mozjpeg": "^7.0.0",
"imagemin-pngquant": "^5.0.1",
"ini": "^1.3.4", "ini": "^1.3.4",
"jimp": "^0.2.28", "jimp": "^0.2.28",
"multer": "^1.3.0", "multer": "^1.3.0",

26
routes/api/image.js
View File

@@ -5,6 +5,7 @@ const router = express.Router();
const sql = require('../../services/sql'); const sql = require('../../services/sql');
const auth = require('../../services/auth'); const auth = require('../../services/auth');
const utils = require('../../services/utils'); const utils = require('../../services/utils');
const sync_table = require('../../services/sync_table');
const multer = require('multer')(); const multer = require('multer')();
const imagemin = require('imagemin'); const imagemin = require('imagemin');
const imageminMozJpeg = require('imagemin-mozjpeg'); const imageminMozJpeg = require('imagemin-mozjpeg');
@@ -24,6 +25,7 @@ router.get('/:imageId/:filename', auth.checkApiAuth, async (req, res, next) => {
}); });
router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, res, next) => { router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, res, next) => {
const sourceId = req.headers.source_id;
const file = req.file; const file = req.file;
const imageId = utils.newNoteId(); const imageId = utils.newNoteId();
@@ -37,15 +39,19 @@ router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, r
const resizedImage = await resize(file.buffer); const resizedImage = await resize(file.buffer);
const optimizedImage = await optimize(resizedImage); const optimizedImage = await optimize(resizedImage);
await sql.insert("images", { await sql.doInTransaction(async () => {
image_id: imageId, await sql.insert("images", {
format: file.mimetype.substr(6), image_id: imageId,
name: file.originalname, format: file.mimetype.substr(6),
checksum: utils.hash(optimizedImage), name: file.originalname,
data: optimizedImage, checksum: utils.hash(optimizedImage),
is_deleted: 0, data: optimizedImage,
date_modified: now, is_deleted: 0,
date_created: now date_modified: now,
date_created: now
});
await sync_table.addImageSync(imageId, sourceId);
}); });
res.send({ res.send({
@@ -60,8 +66,6 @@ const MAX_BYTE_SIZE = 200000; // images should have under 100 KBs
async function resize(buffer) { async function resize(buffer) {
const image = await jimp.read(buffer); const image = await jimp.read(buffer);
console.log("Size: ", buffer.byteLength);
if (image.bitmap.width > image.bitmap.height && image.bitmap.width > MAX_SIZE) { if (image.bitmap.width > image.bitmap.height && image.bitmap.width > MAX_SIZE) {
image.resize(MAX_SIZE, jimp.AUTO); image.resize(MAX_SIZE, jimp.AUTO);
} }

17
routes/api/sync.js
View File

@@ -122,6 +122,17 @@ router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, async (req, res, next
res.send(await sql.getFirst("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId])); res.send(await sql.getFirst("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId]));
}); });
router.get('/images/:imageId', auth.checkApiAuth, async (req, res, next) => {
const imageId = req.params.imageId;
const entity = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [imageId]);
if (entity && entity.data !== null) {
entity.data = entity.data.toString('base64');
}
res.send(entity);
});
router.put('/notes', auth.checkApiAuth, async (req, res, next) => { router.put('/notes', auth.checkApiAuth, async (req, res, next) => {
await syncUpdate.updateNote(req.body.entity, req.body.sourceId); await syncUpdate.updateNote(req.body.entity, req.body.sourceId);
@@ -158,4 +169,10 @@ router.put('/recent_notes', auth.checkApiAuth, async (req, res, next) => {
res.send({}); res.send({});
}); });
router.put('/images', auth.checkApiAuth, async (req, res, next) => {
await syncUpdate.updateImage(req.body.entity, req.body.sourceId);
res.send({});
});
module.exports = router; module.exports = router;

5
services/consistency_checks.js
View File

@@ -4,8 +4,11 @@ const sql = require('./sql');
const log = require('./log'); const log = require('./log');
const messaging = require('./messaging'); const messaging = require('./messaging');
const sync_mutex = require('./sync_mutex'); const sync_mutex = require('./sync_mutex');
const utils = require('./utils');
async function runCheck(query, errorText, errorList) { async function runCheck(query, errorText, errorList) {
utils.assertArguments(query, errorText, errorList);
const result = await sql.getFirstColumn(query); const result = await sql.getFirstColumn(query);
if (result.length > 0) { if (result.length > 0) {
@@ -138,7 +141,7 @@ async function runAllChecks() {
WHERE WHERE
(SELECT COUNT(*) FROM notes_tree WHERE notes.note_id = notes_tree.note_id AND notes_tree.is_deleted = 0) = 0 (SELECT COUNT(*) FROM notes_tree WHERE notes.note_id = notes_tree.note_id AND notes_tree.is_deleted = 0) = 0
AND notes.is_deleted = 0 AND notes.is_deleted = 0
`,); `, 'No undeleted note trees for note IDs', errorList);
await runCheck(` await runCheck(`
SELECT SELECT

101
services/content_hash.js
View File

@@ -19,51 +19,70 @@ async function getHashes() {
const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(','); const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(',');
const hashes = { const hashes = {
notes: getHash(await sql.getAll(`SELECT notes: getHash(await sql.getAll(`
note_id, SELECT
note_title, note_id,
note_text, note_title,
date_modified, note_text,
is_protected, date_modified,
is_deleted is_protected,
FROM notes is_deleted
ORDER BY note_id`)), FROM notes
ORDER BY note_id`)),
notes_tree: getHash(await sql.getAll(`SELECT notes_tree: getHash(await sql.getAll(`
note_tree_id, SELECT
note_id, note_tree_id,
parent_note_id, note_id,
note_position, parent_note_id,
date_modified, note_position,
is_deleted, date_modified,
prefix is_deleted,
FROM notes_tree prefix
ORDER BY note_tree_id`)), FROM notes_tree
ORDER BY note_tree_id`)),
notes_history: getHash(await sql.getAll(`SELECT notes_history: getHash(await sql.getAll(`
note_history_id, SELECT
note_id, note_history_id,
note_title, note_id,
note_text, note_title,
date_modified_from, note_text,
date_modified_to date_modified_from,
FROM notes_history date_modified_to
ORDER BY note_history_id`)), FROM notes_history
ORDER BY note_history_id`)),
recent_notes: getHash(await sql.getAll(`SELECT recent_notes: getHash(await sql.getAll(`
note_tree_id, SELECT
note_path, note_tree_id,
date_accessed, note_path,
is_deleted date_accessed,
FROM recent_notes is_deleted
ORDER BY note_path`)), FROM recent_notes
ORDER BY note_path`)),
options: getHash(await sql.getAll(`SELECT options: getHash(await sql.getAll(`
opt_name, SELECT
opt_value opt_name,
FROM options opt_value
WHERE opt_name IN (${optionsQuestionMarks}) FROM options
ORDER BY opt_name`, options.SYNCED_OPTIONS)) WHERE opt_name IN (${optionsQuestionMarks})
ORDER BY opt_name`, options.SYNCED_OPTIONS)),
// we don't include image data on purpose because they are quite large, checksum is good enough
// to represent the data anyway
images: getHash(await sql.getAll(`
SELECT
image_id,
format,
checksum,
name,
is_deleted,
date_modified,
date_created
FROM images
ORDER BY image_id`))
}; };
const elapseTimeMs = new Date().getTime() - startTime.getTime(); const elapseTimeMs = new Date().getTime() - startTime.getTime();

10
services/sync.js
View File

@@ -143,6 +143,9 @@ async function pullSync(syncContext) {
else if (sync.entity_name === 'recent_notes') { else if (sync.entity_name === 'recent_notes') {
await syncUpdate.updateRecentNotes(resp, syncContext.sourceId); await syncUpdate.updateRecentNotes(resp, syncContext.sourceId);
} }
else if (sync.entity_name === 'images') {
await syncUpdate.updateImage(resp, syncContext.sourceId);
}
else { else {
throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`); throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
} }
@@ -214,6 +217,13 @@ async function pushEntity(sync, syncContext) {
else if (sync.entity_name === 'recent_notes') { else if (sync.entity_name === 'recent_notes') {
entity = await sql.getFirst('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]); entity = await sql.getFirst('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]);
} }
else if (sync.entity_name === 'images') {
entity = await sql.getFirst('SELECT * FROM images WHERE image_id = ?', [sync.entity_id]);
if (entity.data !== null) {
entity.data = entity.data.toString('base64');
}
}
else { else {
throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`); throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
} }

6
services/sync_table.js
View File

@@ -28,6 +28,10 @@ async function addRecentNoteSync(noteTreeId, sourceId) {
await addEntitySync("recent_notes", noteTreeId, sourceId); await addEntitySync("recent_notes", noteTreeId, sourceId);
} }
async function addImageSync(imageId, sourceId) {
await addEntitySync("images", imageId, sourceId);
}
async function addEntitySync(entityName, entityId, sourceId) { async function addEntitySync(entityName, entityId, sourceId) {
await sql.replace("sync", { await sql.replace("sync", {
entity_name: entityName, entity_name: entityName,
@@ -78,6 +82,7 @@ async function fillAllSyncRows() {
await fillSyncRows("notes_tree", "note_tree_id"); await fillSyncRows("notes_tree", "note_tree_id");
await fillSyncRows("notes_history", "note_history_id"); await fillSyncRows("notes_history", "note_history_id");
await fillSyncRows("recent_notes", "note_tree_id"); await fillSyncRows("recent_notes", "note_tree_id");
await fillSyncRows("images", "image_id");
} }
module.exports = { module.exports = {
@@ -87,6 +92,7 @@ module.exports = {
addNoteHistorySync, addNoteHistorySync,
addOptionsSync, addOptionsSync,
addRecentNoteSync, addRecentNoteSync,
addImageSync,
cleanupSyncRowsForMissingEntities, cleanupSyncRowsForMissingEntities,
fillAllSyncRows fillAllSyncRows
}; };

21
services/sync_update.js
View File

@@ -92,11 +92,30 @@ async function updateRecentNotes(entity, sourceId) {
} }
} }
async function updateImage(entity, sourceId) {
if (entity.data !== null) {
entity.data = Buffer.from(entity.data, 'base64');
}
const origImage = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [entity.image_id]);
if (!origImage || origImage.date_modified <= entity.date_modified) {
await sql.doInTransaction(async () => {
await sql.replace("images", entity);
await sync_table.addImageSync(entity.image_id, sourceId);
});
log.info("Update/sync image " + entity.image_id);
}
}
module.exports = { module.exports = {
updateNote, updateNote,
updateNoteTree, updateNoteTree,
updateNoteHistory, updateNoteHistory,
updateNoteReordering, updateNoteReordering,
updateOptions, updateOptions,
updateRecentNotes updateRecentNotes,
updateImage
}; };

11
services/utils.js
View File

@@ -79,6 +79,14 @@ function sanitizeSql(str) {
return str.replace(/'/g, "\\'"); return str.replace(/'/g, "\\'");
} }
function assertArguments() {
for (const i in arguments) {
if (!arguments[i]) {
throw new Error(`Argument idx#${i} should not be falsy: ${arguments[i]}`);
}
}
}
module.exports = { module.exports = {
randomSecureToken, randomSecureToken,
randomString, randomString,
@@ -95,5 +103,6 @@ module.exports = {
hash, hash,
isEmptyOrWhitespace, isEmptyOrWhitespace,
getDateTimeForFile, getDateTimeForFile,
sanitizeSql sanitizeSql,
assertArguments
}; };