refactor: replace JIT actor retrieval with actor assertion and storage logic

src/activitypub/actors.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const winston = require('winston');
+
+const db = require('../database');
+const utils = require('../utils');
+
+const activitypub = module.parent.exports;
+
+const Actors = module.exports;
+
+Actors.assert = async (ids) => {
+	// Handle single values
+	if (!Array.isArray(ids)) {
+		ids = [ids];
+	}
+
+	// Filter out uids if passed in
+	ids = ids.filter(id => !utils.isNumber(id));
+
+	// Filter out existing
+	const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', ids);
+	ids = ids.filter((id, idx) => !exists[idx]);
+
+	if (!ids.length) {
+		return true;
+	}
+
+	const actors = await Promise.all(ids.map(async (id) => {
+		try {
+			const actor = await activitypub.get(0, id);
+
+			// Follow counts
+			const [followers, following] = await Promise.all([
+				actor.followers ? activitypub.get(0, actor.followers) : { totalItems: 0 },
+				actor.following ? activitypub.get(0, actor.following) : { totalItems: 0 },
+			]);
+			actor.followerCount = followers.totalItems;
+			actor.followingCount = following.totalItems;
+
+			// Post count
+			const outbox = actor.outbox ? await activitypub.get(0, actor.outbox) : { totalItems: 0 };
+			actor.postcount = outbox.totalItems;
+
+			return actor;
+		} catch (e) {
+			return null;
+		}
+	}));
+
+	// Build userData object for storage
+	const profiles = await activitypub.mocks.profile(actors);
+	const now = Date.now();
+
+	await Promise.all([
+		db.setObjectBulk(profiles.map((profile, idx) => {
+			if (!profile) {
+				return null;
+			}
+			const key = `userRemote:${ids[idx]}`;
+			return [key, profile];
+		}).filter(Boolean)),
+		db.sortedSetAdd('usersRemote:lastCrawled', ids.map((id, idx) => (profiles[idx] ? now : null)).filter(Boolean), ids.filter((id, idx) => profiles[idx])),
+	]);
+
+	return actors.every(Boolean);
+};

src/activitypub/inbox.js

@@ -42,24 +42,24 @@ inbox.follow = async (req) => {
 		throw new Error('[[error:invalid-uid]]');
 	}
 
-	const from = await activitypub.getActor(localUid, req.body.actor);
+	const assertion = await activitypub.actors.assert(req.body.actor);
-	if (!from) {
+	if (!assertion) {
-		throw new Error('[[error:invalid-uid]]'); // should probably be AP specific
+		throw new Error('[[error:activitypub.invalid-id]]');
 	}
 
-	const isFollowed = await inbox.isFollowed(from.id, localUid);
+	const isFollowed = await inbox.isFollowed(req.body.actor, localUid);
 	if (isFollowed) {
 		// No additional parsing required
 		return;
 	}
 
 	const now = Date.now();
-	await db.sortedSetAdd(`followersRemote:${localUid}`, now, from.id);
+	await db.sortedSetAdd(`followersRemote:${localUid}`, now, req.body.actor);
-	await activitypub.send(localUid, from.id, {
+	await activitypub.send(localUid, req.body.actor, {
 		type: 'Accept',
 		object: {
 			type: 'Follow',
-			actor: from.id,
+			actor: req.body.actor,
 		},
 	});
 
@@ -75,7 +75,7 @@ inbox.isFollowed = async (actorId, uid) => {
 };
 
 inbox.accept = async (req) => {
-	let { actor, object } = req.body;
+	const { actor, object } = req.body;
 	const { type } = object;
 
 	const uid = await helpers.resolveLocalUid(object.actor);
@@ -83,19 +83,22 @@ inbox.accept = async (req) => {
 		throw new Error('[[error:invalid-uid]]');
 	}
 
-	actor = await activitypub.getActor(uid, actor);
+	const assertion = await activitypub.actors.assert(actor);
+	if (!assertion) {
+		throw new Error('[[error:activitypub.invalid-id]]');
+	}
 
 	if (type === 'Follow') {
 		const now = Date.now();
 		await Promise.all([
-			db.sortedSetAdd(`followingRemote:${uid}`, now, actor.id),
+			db.sortedSetAdd(`followingRemote:${uid}`, now, actor),
 			db.incrObjectField(`user:${uid}`, 'followingRemoteCount'),
 		]);
 	}
 };
 
 inbox.undo = async (req) => {
-	let { actor, object } = req.body;
+	const { actor, object } = req.body;
 	const { type } = object;
 
 	const uid = await helpers.resolveLocalUid(object.object);
@@ -103,11 +106,14 @@ inbox.undo = async (req) => {
 		throw new Error('[[error:invalid-uid]]');
 	}
 
-	actor = await activitypub.getActor(uid, actor);
+	const assertion = await activitypub.actors.assert(actor);
+	if (!assertion) {
+		throw new Error('[[error:activitypub.invalid-id]]');
+	}
 
 	if (type === 'Follow') {
 		await Promise.all([
-			db.sortedSetRemove(`followingRemote:${uid}`, actor.id),
+			db.sortedSetRemove(`followingRemote:${uid}`, actor),
 			db.decrObjectField(`user:${uid}`, 'followingRemoteCount'),
 		]);
 	}

src/activitypub/index.js

@@ -11,7 +11,6 @@ const utils = require('../utils');
 const ttl = require('../cache/ttl');
 
 const requestCache = ttl({ ttl: 1000 * 60 * 5 }); // 5 minutes
-const actorCache = ttl({ ttl: 1000 * 60 * 60 * 24 }); // 24 hours
 const ActivityPub = module.exports;
 
 ActivityPub._constants = Object.freeze({
@@ -22,55 +21,15 @@ ActivityPub.helpers = require('./helpers');
 ActivityPub.inbox = require('./inbox');
 ActivityPub.mocks = require('./mocks');
 ActivityPub.notes = require('./notes');
+ActivityPub.actors = require('./actors');
 
-ActivityPub.getActor = async (uid, input) => {
+ActivityPub.resolveInboxes = async (ids) => {
-	// Can be a webfinger id, uri, or object, handle as appropriate
-	let uri;
-	if (ActivityPub.helpers.isUri(input)) {
-		uri = input;
-	} else if (input.indexOf('@') !== -1) { // Webfinger
-		({ actorUri: uri } = await ActivityPub.helpers.query(input));
-	} else {
-		throw new Error('[[error:invalid-data]]');
-	}
-
-	if (!uri) {
-		throw new Error('[[error:invalid-uid]]');
-	}
-
-	if (actorCache.has(uri)) {
-		return actorCache.get(uri);
-	}
-
-	try {
-		const actor = await ActivityPub.get(uid, uri);
-
-		// Follow counts
-		const [followers, following] = await Promise.all([
-			actor.followers ? ActivityPub.get(uid, actor.followers) : { totalItems: 0 },
-			actor.following ? ActivityPub.get(uid, actor.following) : { totalItems: 0 },
-		]);
-		actor.followerCount = followers.totalItems;
-		actor.followingCount = following.totalItems;
-
-		actor.hostname = new URL(uri).hostname;
-
-		actorCache.set(uri, actor);
-		return actor;
-	} catch (e) {
-		winston.warn(`[activitypub/getActor] Unable to retrieve actor "${uri}", error: ${e.message}`);
-		return null;
-	}
-};
-
-ActivityPub.resolveInboxes = async (uid, ids) => {
 	const inboxes = new Set();
 
 	await Promise.all(ids.map(async (id) => {
-		const actor = await ActivityPub.getActor(uid, id);
+		const { inbox, sharedInbox } = await user.getUserFields(id, ['inbox', 'sharedInbox']);
-		const inbox = actor.sharedInbox || actor.inbox;
+		if (sharedInbox || inbox) {
-		if (inbox) {
+			inboxes.add(sharedInbox || inbox);
-			inboxes.add(inbox);
 		}
 	}));
 

src/activitypub/mocks.js

@@ -3,35 +3,27 @@
 const nconf = require('nconf');
 const mime = require('mime');
 
-const db = require('../database');
 const user = require('../user');
 const posts = require('../posts');
-const topics = require('../topics');
 
 const activitypub = module.parent.exports;
 const Mocks = module.exports;
 
-Mocks.profile = async (actors, callerUid = 0) => {
+Mocks.profile = async (actors) => {
-	// Accepts an array containing actor objects (the output of getActor()), or uris
+	// Should only ever be called by activitypub.actors.assert
-	let single = false;
-	if (!Array.isArray(actors)) {
-		single = true;
-		actors = [actors];
-	}
-
 	const profiles = (await Promise.all(actors.map(async (actor) => {
-		// convert uri to actor object
-		if (typeof actor === 'string' && activitypub.helpers.isUri(actor)) {
-			actor = await activitypub.getActor(callerUid, actor);
-		}
-
 		if (!actor) {
 			return null;
 		}
 
 		const uid = actor.id;
-		const { preferredUsername, published, icon, image, name, summary, hostname, followerCount, followingCount } = actor;
+		const {
-		const isFollowing = await db.isSortedSetMember(`followingRemote:${callerUid}`, uid);
+			preferredUsername, published, icon, image,
+			name, summary, followerCount, followingCount,
+			postcount, inbox, endpoints,
+		} = actor;
+		const { hostname } = new URL(actor.id);
+		// const isFollowing = await db.isSortedSetMember(`followingRemote:${callerUid}`, uid);
 
 		let picture;
 		if (icon) {
@@ -56,19 +48,18 @@ Mocks.profile = async (actors, callerUid = 0) => {
 			'cover:url': !image || typeof image === 'string' ? image : image.url,
 			'cover:position': '50% 50%',
 			aboutme: summary,
-			aboutmeParsed: summary,
+			postcount,
+			followerCount,
+			followingCount,
 
-			isFollowing,
+			inbox,
-			counts: {
+			sharedInbox: endpoints.sharedInbox,
-				following: followingCount,
-				followers: followerCount,
-			},
 		};
 
 		return payload;
-	}))).filter(Boolean);
+	})));
 
-	return single ? profiles.pop() : profiles;
+	return profiles;
 };
 
 Mocks.post = async (objects) => {

src/api/activitypub.js

@@ -17,36 +17,36 @@ const posts = require('../posts');
 
 const activitypubApi = module.exports;
 
-activitypubApi.follow = async (caller, { uid: actorId } = {}) => {
+activitypubApi.follow = async (caller, { uid } = {}) => {
-	const object = await activitypub.getActor(caller.uid, actorId);
+	const result = await activitypub.helpers.query(uid);
-	if (!object) {
+	if (!result) {
 		throw new Error('[[error:activitypub.invalid-id]]');
 	}
 
-	await activitypub.send(caller.uid, actorId, {
+	await activitypub.send(caller.uid, uid, {
 		type: 'Follow',
-		object: object.id,
+		object: result.actorUri,
 	});
 };
 
-activitypubApi.unfollow = async (caller, { uid: actorId }) => {
+activitypubApi.unfollow = async (caller, { uid }) => {
-	const object = await activitypub.getActor(caller.uid, actorId);
 	const userslug = await user.getUserField(caller.uid, 'userslug');
-	if (!object) {
+	const result = await activitypub.helpers.query(uid);
+	if (!result) {
 		throw new Error('[[error:activitypub.invalid-id]]');
 	}
 
-	await activitypub.send(caller.uid, actorId, {
+	await activitypub.send(caller.uid, uid, {
 		type: 'Undo',
 		object: {
 			type: 'Follow',
 			actor: `${nconf.get('url')}/user/${userslug}`,
-			object: object.id,
+			object: result.actorUri,
 		},
 	});
 
 	await Promise.all([
-		db.sortedSetRemove(`followingRemote:${caller.uid}`, object.id),
+		db.sortedSetRemove(`followingRemote:${caller.uid}`, result.actorUri),
 		db.decrObjectField(`user:${caller.uid}`, 'followingRemoteCount'),
 	]);
 };

src/controllers/accounts/follow.js

@@ -4,8 +4,6 @@ const user = require('../../user');
 const helpers = require('../helpers');
 const pagination = require('../../pagination');
 
-const activitypubController = require('../activitypub');
-
 const followController = module.exports;
 
 followController.getFollowing = async function (req, res, next) {
@@ -17,10 +15,6 @@ followController.getFollowers = async function (req, res, next) {
 };
 
 async function getFollow(tpl, name, req, res) {
-	if (res.locals.uid === -2) {
-		return activitypubController.profiles.getFollow(tpl, name, req, res);
-	}
-
 	const {
 		username, userslug, followerCount, followingCount,
 	} = await user.getUserFields(res.locals.uid, [

src/controllers/accounts/helpers.js

@@ -13,6 +13,8 @@ const privileges = require('../../privileges');
 const translator = require('../../translator');
 const messaging = require('../../messaging');
 const categories = require('../../categories');
+const posts = require('../../posts');
+const activitypub = require('../../activitypub');
 
 const relative_path = nconf.get('relative_path');
 
@@ -177,6 +179,7 @@ async function canChat(callerUID, uid) {
 
 async function getCounts(userData, callerUID) {
 	const { uid } = userData;
+	const isRemote = activitypub.helpers.isUri(uid);
 	const cids = await categories.getCidsByPrivilege('categories:cid', callerUID, 'topics:read');
 	const promises = {
 		posts: db.sortedSetsCardSum(cids.map(c => `cid:${c}:uid:${uid}:pids`)),
@@ -196,6 +199,7 @@ async function getCounts(userData, callerUID) {
 		promises.blocks = user.getUserField(userData.uid, 'blocksCount');
 	}
 	const counts = await utils.promiseParallel(promises);
+	counts.posts = isRemote ? userData.postcount : counts.posts;
 	counts.best = counts.best.reduce((sum, count) => sum + count, 0);
 	counts.controversial = counts.controversial.reduce((sum, count) => sum + count, 0);
 	counts.categoriesWatched = counts.categoriesWatched && counts.categoriesWatched.length;
@@ -271,7 +275,12 @@ async function parseAboutMe(userData) {
 		userData.aboutme = '';
 		userData.aboutmeParsed = '';
 		return;
+	} else if (activitypub.helpers.isUri(userData.uid)) {
+		userData.aboutme = posts.sanitize(userData.aboutme);
+		userData.aboutmeParsed = userData.aboutme;
+		return;
 	}
+
 	userData.aboutme = validator.escape(String(userData.aboutme || ''));
 	const parsed = await plugins.hooks.fire('filter:parse.aboutme', userData.aboutme);
 	userData.aboutme = translator.escape(userData.aboutme);

src/controllers/accounts/profile.js

@@ -13,15 +13,9 @@ const accountHelpers = require('./helpers');
 const helpers = require('../helpers');
 const utils = require('../../utils');
 
-const activitypubController = require('../activitypub');
-
 const profileController = module.exports;
 
 profileController.get = async function (req, res, next) {
-	if (res.locals.uid === -2) {
-		return activitypubController.profiles.get(req, res, next);
-	}
-
 	const lowercaseSlug = req.params.userslug.toLowerCase();
 
 	if (req.params.userslug !== lowercaseSlug) {

src/controllers/activitypub/index.js

@@ -8,7 +8,6 @@ const activitypub = require('../../activitypub');
 const Controller = module.exports;
 
 Controller.actors = require('./actors');
-Controller.profiles = require('./profiles');
 Controller.topics = require('./topics');
 
 Controller.getFollowing = async (req, res) => {

src/controllers/activitypub/profiles.js

@@ -1,52 +0,0 @@
-'use strict';
-
-const { getActor, mocks, get } = require('../../activitypub');
-const helpers = require('../helpers');
-const pagination = require('../../pagination');
-
-const controller = module.exports;
-
-controller.get = async function (req, res, next) {
-	if (req.uid === -1) {
-		return helpers.notAllowed(req, res);
-	}
-
-	const { userslug: uid } = req.params;
-	const actor = await getActor(req.uid, uid);
-	if (!actor) {
-		return next();
-	}
-
-	const payload = await mocks.profile(actor, req.uid);
-	res.render('account/profile', payload);
-};
-
-controller.getFollow = async function (tpl, name, req, res) {
-	if (req.uid === -1) {
-		return helpers.notAllowed(req, res);
-	}
-
-	const actor = await getActor(req.uid, req.params.userslug);
-
-	const { userslug } = req.params;
-	const { preferredUsername: username, followerCount, followingCount } = actor;
-
-	const page = parseInt(req.query.page, 10) || 1;
-
-	const payload = {
-		...await mocks.profile(actor, req.uid),
-	};
-	payload.title = `[[pages:${tpl}, ${username}]]`;
-
-	const collection = await get(req.uid, `${actor[name]}?page=${page}`);
-	const resultsPerPage = collection.orderedItems.length;
-	payload.users = await mocks.profile(collection.orderedItems, req.uid);
-
-	const count = name === 'following' ? followingCount : followerCount;
-	const pageCount = Math.ceil(count / resultsPerPage);
-	payload.pagination = pagination.create(page, pageCount);
-
-	payload.breadcrumbs = helpers.buildBreadcrumbs([{ text: username, url: `/user/${userslug}` }, { text: `[[user:${name}]]` }]);
-
-	res.render(tpl, payload);
-};

src/user/data.js

@@ -78,9 +78,10 @@ module.exports = function (User) {
 			fields = fields.filter(value => value !== 'password');
 		}
 
+		await activitypub.actors.assert(remoteIds);
 		const users = [
 			...await db.getObjectsFields(uniqueUids.map(uid => `user:${uid}`), fields),
-			...await activitypub.mocks.profile(remoteIds, 0, fields),
+			...await db.getObjectsFields(remoteIds.map(id => `userRemote:${id}`), fields),
 		];
 		const result = await plugins.hooks.fire('filter:user.getFields', {
 			uids: uniqueUids,

src/user/index.js

@@ -7,6 +7,7 @@ const plugins = require('../plugins');
 const db = require('../database');
 const privileges = require('../privileges');
 const categories = require('../categories');
+const activitypub = require('../activitypub');
 const meta = require('../meta');
 const utils = require('../utils');
 
@@ -109,6 +110,12 @@ User.getUidByUserslug = async function (userslug) {
 	if (!userslug) {
 		return 0;
 	}
+
+	if (userslug.includes('@')) {
+		const { actorUri } = await activitypub.helpers.query(userslug);
+		return actorUri;
+	}
+
 	return await db.sortedSetScore('userslug:uid', userslug);
 };