feat: pruning of stale notes older than 30 days with no engagement

src/activitypub/index.js

@@ -3,6 +3,7 @@
 const nconf = require('nconf');
 const winston = require('winston');
 const { createHash, createSign, createVerify, getHashes } = require('crypto');
+const { CronJob } = require('cron');
 
 const request = require('../request');
 const db = require('../database');
@@ -41,6 +42,12 @@ ActivityPub.mocks = require('./mocks');
 ActivityPub.notes = require('./notes');
 ActivityPub.actors = require('./actors');
 
+ActivityPub.startJobs = () => {
+	winston.verbose('[activitypub/jobs] Registering jobs.');
+	new CronJob('0 0 * * *', ActivityPub.notes.prune, null, true);
+	// new CronJob(new Date(Date.now() + 2000), ActivityPub.notes.prune, null, true);
+};
+
 ActivityPub.resolveId = async (uid, id) => {
 	try {
 		const query = new URL(id);

src/activitypub/notes.js

@@ -4,6 +4,7 @@ const winston = require('winston');
 const nconf = require('nconf');
 
 const db = require('../database');
+const batch = require('../batch');
 const meta = require('../meta');
 const privileges = require('../privileges');
 const categories = require('../categories');
@@ -372,3 +373,50 @@ Notes.delete = async (pids) => {
 
 	await db.deleteAll([...recipientSets, ...announcerSets]);
 };
+
+Notes.prune = async () => {
+	/**
+	 * Prune topics in cid -1 that have received no engagement.
+	 * Engagement is defined as:
+	 *   - Replied to (contains a local reply)
+	 *   - Post within is liked
+	 */
+	winston.verbose('[notes/prune] Starting scheduled pruning of topics');
+	const start = 0;
+	const stop = Date.now() - (1000 * 60 * 60 * 24 * 30); // 30 days; todo: make configurable?
+	let tids = await db.getSortedSetRangeByScore('cid:-1:tids', 0, -1, start, stop);
+
+	winston.verbose(`[notes/prune] Found ${tids.length} topics older than 30 days (since last activity).`);
+
+	const posters = await db.getSortedSetsMembers(tids.map(tid => `tid:${tid}:posters`));
+	const hasLocalVoter = await Promise.all(tids.map(async (tid) => {
+		const mainPid = await db.getObjectField(`topic:${tid}`, 'mainPid');
+		const pids = await db.getSortedSetMembers(`tid:${tid}:posts`);
+		pids.unshift(mainPid);
+
+		// Check voters of each pid for a local uid
+		const voters = new Set();
+		await Promise.all(pids.map(async (pid) => {
+			const [upvoters, downvoters] = await db.getSetsMembers([`pid:${pid}:upvote`, `pid:${pid}:downvote`]);
+			upvoters.forEach(uid => voters.add(uid));
+			downvoters.forEach(uid => voters.add(uid));
+		}));
+
+		return Array.from(voters).some(uid => utils.isNumber(uid));
+	}));
+
+	tids = tids.filter((_, idx) => {
+		const localPoster = posters[idx].some(uid => utils.isNumber(uid));
+		const localVoter = hasLocalVoter[idx];
+
+		return !localPoster && !localVoter;
+	});
+
+	winston.verbose(`[notes/prune] ${tids.length} topics eligible for pruning`);
+
+	await batch.processArray(tids, async (tids) => {
+		await Promise.all(tids.map(async (tid) => {
+			topics.purgePostsAndTopic(tid, 0);
+		}));
+	}, { batch: 100 });
+};

src/posts/votes.js

@@ -278,7 +278,7 @@ module.exports = function (Posts) {
 			}
 		}
 
-		if (parseInt(topicData.mainPid, 10) !== parseInt(postData.pid, 10)) {
+		if (String(topicData.mainPid) !== String(postData.pid)) {
 			return await db.sortedSetAdd(`tid:${postData.tid}:posts:votes`, postData.votes, postData.pid);
 		}
 		const promises = [

src/start.js

@@ -39,6 +39,7 @@ start.start = async function () {
 			require('./user').startJobs();
 			require('./plugins').startJobs();
 			require('./topics').scheduled.startJobs();
+			require('./activitypub').startJobs();
 			await db.delete('locks');
 		}