refactor: ability to browse to remote categories, group actor assertion logic, etc. -- no logic to assign topics to remote categories yet

public/src/client/category.js

@@ -118,7 +118,7 @@ define('forum/category', [
 	};
 
 	Category.toBottom = async () => {
-		const { count } = await api.get(`/categories/${ajaxify.data.category.cid}/count`);
+		const { count } = await api.get(`/categories/${encodeURIComponent(ajaxify.data.category.cid)}/count`);
 		navigator.scrollBottom(count - 1);
 	};
 
@@ -127,7 +127,7 @@ define('forum/category', [
 
 		hooks.fire('action:topics.loading');
 		const params = utils.params();
-		infinitescroll.loadMore(`/categories/${ajaxify.data.cid}/topics`, {
+		infinitescroll.loadMore(`/categories/${encodeURIComponent(ajaxify.data.cid)}/topics`, {
 			after: after,
 			direction: direction,
 			query: params,

src/activitypub/actors.js

@@ -7,6 +7,7 @@ const _ = require('lodash');
 const db = require('../database');
 const meta = require('../meta');
 const batch = require('../batch');
+const categories = require('../categories');
 const user = require('../user');
 const utils = require('../utils');
 const TTLCache = require('../cache/ttl');
@@ -20,15 +21,12 @@ const activitypub = module.parent.exports;
 
 const Actors = module.exports;
 
-Actors.assert = async (ids, options = {}) => {
+Actors.qualify = async (ids, options = {}) => {
 	/**
-	 * Ensures that the passed in ids or webfinger handles are stored in database.
+	 * Sanity-checks, cache handling, webfinger translations, so that only
-	 * Options:
+	 * an array of actor uris are handled by assert/assertGroup.
-	 *   - update: boolean, forces re-fetch/process of the resolved id
+	 *
-	 * Return one of:
+	 * This method is only called by assert/assertGroup (at least in core.)
-	 *   - An array of newly processed ids
-	 *   - false: if input incorrect (or webfinger handle cannot resolve)
-	 *   - true: no new IDs processed; all passed-in IDs present.
 	 */
 
 	// Handle single values
@@ -85,6 +83,21 @@ Actors.assert = async (ids, options = {}) => {
 		});
 	}
 
+	return ids;
+};
+
+Actors.assert = async (ids, options = {}) => {
+	/**
+	 * Ensures that the passed in ids or webfinger handles are stored in database.
+	 * Options:
+	 *   - update: boolean, forces re-fetch/process of the resolved id
+	 * Return one of:
+	 *   - An array of newly processed ids
+	 *   - false: if input incorrect (or webfinger handle cannot resolve)
+	 *   - true: no new IDs processed; all passed-in IDs present.
+	 */
+
+	ids = await Actors.qualify(ids, options);
 	if (!ids.length) {
 		return true;
 	}
@@ -96,6 +109,7 @@ Actors.assert = async (ids, options = {}) => {
 	const urlMap = new Map();
 	const followersUrlMap = new Map();
 	const pubKeysMap = new Map();
+	const categories = new Set();
 	let actors = await Promise.all(ids.map(async (id) => {
 		try {
 			activitypub.helpers.log(`[activitypub/actors] Processing ${id}`);
@@ -104,8 +118,14 @@ Actors.assert = async (ids, options = {}) => {
 			let typeOk = false;
 			if (Array.isArray(actor.type)) {
 				typeOk = actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type));
+				if (!typeOk && actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type))) {
+					categories.add(actor.id);
+				}
 			} else {
 				typeOk = activitypub._constants.acceptableActorTypes.has(actor.type);
+				if (!typeOk && activitypub._constants.acceptableGroupTypes.has(actor.type)) {
+					categories.add(actor.id);
+				}
 			}
 
 			if (
@@ -220,6 +240,142 @@ Actors.assert = async (ids, options = {}) => {
 	return actors;
 };
 
+Actors.assertGroup = async (ids, options = {}) => {
+	/**
+	 * Ensures that the passed in ids or webfinger handles are stored in database.
+	 * Options:
+	 *   - update: boolean, forces re-fetch/process of the resolved id
+	 * Return one of:
+	 *   - An array of newly processed ids
+	 *   - false: if input incorrect (or webfinger handle cannot resolve)
+	 *   - true: no new IDs processed; all passed-in IDs present.
+	 */
+
+	ids = await Actors.qualify(ids, options);
+	if (!ids.length) {
+		return true;
+	}
+
+	activitypub.helpers.log(`[activitypub/actors] Asserting ${ids.length} group(s)`);
+
+	// NOTE: MAKE SURE EVERY DB ADDITION HAS A CORRESPONDING REMOVAL IN ACTORS.REMOVE!
+
+	const urlMap = new Map();
+	const followersUrlMap = new Map();
+	const pubKeysMap = new Map();
+	const users = new Set();
+	let groups = await Promise.all(ids.map(async (id) => {
+		try {
+			activitypub.helpers.log(`[activitypub/actors] Processing group ${id}`);
+			const actor = (typeof id === 'object' && id.hasOwnProperty('id')) ? id : await activitypub.get('uid', 0, id, { cache: process.env.CI === 'true' });
+
+			let typeOk = false;
+			if (Array.isArray(actor.type)) {
+				typeOk = actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type));
+				if (!typeOk && actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type))) {
+					users.add(actor.id);
+				}
+			} else {
+				typeOk = activitypub._constants.acceptableGroupTypes.has(actor.type);
+				if (!typeOk && activitypub._constants.acceptableActorTypes.has(actor.type)) {
+					users.add(actor.id);
+				}
+			}
+
+			if (
+				!typeOk ||
+				!activitypub._constants.requiredActorProps.every(prop => actor.hasOwnProperty(prop))
+			) {
+				return null;
+			}
+
+			// Save url for backreference
+			const url = Array.isArray(actor.url) ? actor.url.shift() : actor.url;
+			if (url && url !== actor.id) {
+				urlMap.set(url, actor.id);
+			}
+
+			// Save followers url for backreference
+			if (actor.hasOwnProperty('followers') && activitypub.helpers.isUri(actor.followers)) {
+				followersUrlMap.set(actor.followers, actor.id);
+			}
+
+			// Public keys
+			pubKeysMap.set(actor.id, actor.publicKey);
+
+			return actor;
+		} catch (e) {
+			if (e.code === 'ap_get_410') {
+				// const exists = await user.exists(id);
+				// if (exists) {
+				// 	await user.deleteAccount(id);
+				// }
+			}
+
+			return null;
+		}
+	}));
+	groups = groups.filter(Boolean); // remove unresolvable actors
+
+	// Build userData object for storage
+	const categoryObjs = (await activitypub.mocks.category(groups)).filter(Boolean);
+	const now = Date.now();
+
+	const bulkSet = categoryObjs.reduce((memo, category) => {
+		const key = `categoryRemote:${category.cid}`;
+		memo.push([key, category], [`${key}:keys`, pubKeysMap.get(category.cid)]);
+		return memo;
+	}, []);
+	if (urlMap.size) {
+		bulkSet.push(['remoteUrl:cid', Object.fromEntries(urlMap)]);
+	}
+	if (followersUrlMap.size) {
+		bulkSet.push(['followersUrl:cid', Object.fromEntries(followersUrlMap)]);
+	}
+
+	const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', categoryObjs.map(p => p.cid));
+	const cidsForCurrent = categoryObjs.map((p, idx) => (exists[idx] ? p.cid : 0));
+	const current = await categories.getCategoriesFields(cidsForCurrent, ['slug']);
+	const queries = categoryObjs.reduce((memo, profile, idx) => {
+		const { slug } = current[idx];
+
+		if (options.update || slug !== profile.slug) {
+			if (cidsForCurrent[idx] !== 0 && slug) {
+				// memo.searchRemove.push(['ap.preferredUsername:sorted', `${slug.toLowerCase()}:${profile.uid}`]);
+				memo.handleRemove.push(slug.toLowerCase());
+			}
+
+			// memo.searchAdd.push(['ap.preferredUsername:sorted', 0, `${profile.slug.toLowerCase()}:${profile.uid}`]);
+			memo.handleAdd[profile.slug.toLowerCase()] = profile.cid;
+		}
+
+		// if (options.update || (profile.fullname && fullname !== profile.fullname)) {
+		// 	if (fullname && cidsForCurrent[idx] !== 0) {
+		// 		memo.searchRemove.push(['ap.name:sorted', `${fullname.toLowerCase()}:${profile.uid}`]);
+		// 	}
+
+		// 	memo.searchAdd.push(['ap.name:sorted', 0, `${profile.fullname.toLowerCase()}:${profile.uid}`]);
+		// }
+
+		return memo;
+	}, { /* searchRemove: [], searchAdd: [], */ handleRemove: [], handleAdd: {} });
+
+	// Removals
+	await Promise.all([
+		// db.sortedSetRemoveBulk(queries.searchRemove),
+		db.deleteObjectFields('handle:cid', queries.handleRemove),
+	]);
+
+	await Promise.all([
+		db.setObjectBulk(bulkSet),
+		db.sortedSetAdd('usersRemote:lastCrawled', groups.map(() => now), groups.map(p => p.uid)),
+		// db.sortedSetAddBulk(queries.searchAdd),
+		db.setObject('handle:cid', queries.handleAdd),
+	]);
+
+	return categoryObjs;
+};
+
 Actors.getLocalFollowers = async (id) => {
 	const response = {
 		uids: new Set(),

src/activitypub/index.js

@@ -39,7 +39,8 @@ ActivityPub._constants = Object.freeze({
 	acceptedPostTypes: [
 		'Note', 'Page', 'Article', 'Question', 'Video',
 	],
-	acceptableActorTypes: new Set(['Application', 'Group', 'Organization', 'Person', 'Service']),
+	acceptableActorTypes: new Set(['Application', 'Organization', 'Person', 'Service']),
+	acceptableGroupTypes: new Set(['Group']),
 	requiredActorProps: ['inbox', 'outbox'],
 	acceptedProtocols: ['https', ...(process.env.CI === 'true' ? ['http'] : [])],
 	acceptable: {

src/activitypub/mocks.js

@@ -216,7 +216,7 @@ Mocks.profile = async (actors) => {
 			uploadedpicture: undefined,
 			'cover:url': !image || typeof image === 'string' ? image : image.url,
 			'cover:position': '50% 50%',
-			aboutme: summary,
+			aboutme: posts.sanitize(summary),
 			followerCount,
 			followingCount,
 
@@ -233,6 +233,73 @@ Mocks.profile = async (actors) => {
 	return profiles;
 };
 
+Mocks.category = async (actors) => {
+	const categories = await Promise.all(actors.map(async (actor) => {
+		if (!actor) {
+			return null;
+		}
+
+		const cid = actor.id;
+		let hostname;
+		let {
+			url, preferredUsername, /* icon, */ image,
+			name, summary, followers, inbox, endpoints, tag,
+		} = actor;
+		preferredUsername = slugify(preferredUsername || name);
+		// const { followers: followerCount, following: followingCount } = await activitypub.actors.getLocalFollowCounts(uid);
+
+		try {
+			({ hostname } = new URL(actor.id));
+		} catch (e) {
+			return null;
+		}
+
+		// No support for category avatars yet ;(
+		// let picture;
+		// if (icon) {
+		// 	picture = typeof icon === 'string' ? icon : icon.url;
+		// }
+		const iconBackgrounds = await user.getIconBackgrounds();
+		let bgColor = Array.prototype.reduce.call(preferredUsername, (cur, next) => cur + next.charCodeAt(), 0);
+		bgColor = iconBackgrounds[bgColor % iconBackgrounds.length];
+
+		// Replace emoji in summary
+		if (tag && Array.isArray(tag)) {
+			tag
+				.filter(tag => tag.type === 'Emoji' &&
+					isEmojiShortcode.test(tag.name) &&
+					tag.icon && tag.icon.mediaType && tag.icon.mediaType.startsWith('image/'))
+				.forEach((tag) => {
+					summary = summary.replace(new RegExp(tag.name, 'g'), `<img class="not-responsive emoji" src="${tag.icon.url}" title="${tag.name}" />`);
+				});
+		}
+
+		const payload = {
+			cid,
+			name,
+			handle: preferredUsername,
+			slug: `${preferredUsername}@${hostname}`,
+			description: summary,
+			descriptionParsed: posts.sanitize(summary),
+			icon: 'fa-comments',
+			color: '#fff',
+			bgColor,
+			backgroundImage: !image || typeof image === 'string' ? image : image.url,
+			// followerCount,
+			// followingCount,
+
+			url,
+			inbox,
+			sharedInbox: endpoints ? endpoints.sharedInbox : null,
+			followersUrl: followers,
+		};
+
+		return payload;
+	}));
+
+	return categories;
+};
+
 Mocks.post = async (objects) => {
 	let single = false;
 	if (!Array.isArray(objects)) {

src/categories/data.js

@@ -36,8 +36,8 @@ module.exports = function (Categories) {
 			return [];
 		}
 
-		cids = cids.map(cid => parseInt(cid, 10));
+		cids = cids.map(cid => (utils.isNumber(cid) ? parseInt(cid, 10) : cid));
-		const keys = cids.map(cid => `category:${cid}`);
+		const keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`));
 		const categories = await db.getObjects(keys, fields);
 
 		// Handle cid -1

src/categories/index.js

@@ -10,6 +10,7 @@ const plugins = require('../plugins');
 const privileges = require('../privileges');
 const cache = require('../cache');
 const meta = require('../meta');
+const utils = require('../utils');
 
 const Categories = module.exports;
 
@@ -26,9 +27,14 @@ require('./search')(Categories);
 Categories.icons = require('./icon');
 
 Categories.exists = async function (cids) {
-	return await db.exists(
+	let keys;
-		Array.isArray(cids) ? cids.map(cid => `category:${cid}`) : `category:${cids}`
+	if (Array.isArray(cids)) {
-	);
+		keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`));
+	} else {
+		keys = utils.isNumber(cids) ? `category:${cids}` : `categoryRemote:${cids}`;
+	}
+
+	return await db.exists(keys);
 };
 
 Categories.existsByHandle = async function (handle) {

src/categories/topics.js

@@ -31,6 +31,7 @@ module.exports = function (Categories) {
 			Categories.getPinnedTids({ ...data, start: 0, stop: -1 }),
 			Categories.buildTopicsSortedSet(data),
 		]);
+		console.log(set);
 
 		const totalPinnedCount = pinnedTids.length;
 		const pinnedTidsOnPage = pinnedTids.slice(data.start, data.stop !== -1 ? data.stop + 1 : undefined);

src/controllers/category.js

@@ -26,14 +26,22 @@ const validSorts = [
 ];
 
 categoryController.get = async function (req, res, next) {
-	const cid = req.params.category_id;
+	let cid = req.params.category_id;
 	if (cid === '-1') {
 		return helpers.redirect(res, `${res.locals.isAPI ? '/api' : ''}/world?${qs.stringify(req.query)}`);
 	}
 
+	if (!utils.isNumber(cid)) {
+		const assertion = await activitypub.actors.assertGroup([cid]);
+		cid = await db.getObjectField('handle:cid', cid);
+		if (!assertion || !cid) {
+			return next();
+		}
+	}
+
 	let currentPage = parseInt(req.query.page, 10) || 1;
 	let topicIndex = utils.isNumber(req.params.topic_index) ? parseInt(req.params.topic_index, 10) - 1 : 0;
-	if ((req.params.topic_index && !utils.isNumber(req.params.topic_index)) || !utils.isNumber(cid)) {
+	if ((req.params.topic_index && !utils.isNumber(req.params.topic_index))) {
 		return next();
 	}
 
@@ -58,7 +66,7 @@ categoryController.get = async function (req, res, next) {
 		return helpers.notAllowed(req, res);
 	}
 
-	if (!res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) {
+	if (utils.isNumber(cid) && !res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) {
 		return helpers.redirect(res, `/category/${categoryFields.slug}?${qs.stringify(req.query)}`, true);
 	}