Categories refactor (#9233)

* feat: wip categories pagination

* feat: add subCategoriesPerPage setting

* feat: add load more sub categories button to category page

* fix: openapi spec

* feat: show sub categories left on category page

hide button when no more categories left

* breaking: rename categories to allCategories on /search

categories contains the search results

* fix: spec

* refactor: remove cidsPerPage

* fix: tests

* feat: use component for subcategories

* fix: prevent negative subCategoriesLeft

install/data/defaults.json

@@ -101,6 +101,7 @@
     "maxPostsPerPage": 20,
     "topicsPerPage": 20,
     "postsPerPage": 20,
+    "categoriesPerPage": 50,
     "userSearchResultsPerPage": 50,
     "maximumGroupNameLength": 255,
     "maximumGroupTitleLength": 40,

public/language/en-GB/admin/manage/categories.json

@@ -10,6 +10,7 @@
 	"custom-class": "Custom Class",
 	"num-recent-replies": "# of Recent Replies",
 	"ext-link": "External Link",
+	"subcategories-per-page": "Subcategories per page",
 	"is-section": "Treat this category as a section",
 	"post-queue": "Post queue",
 	"tag-whitelist": "Tag Whitelist",

public/language/en-GB/admin/settings/pagination.json

@@ -1,10 +1,12 @@
 {
 	"pagination": "Pagination Settings",
 	"enable": "Paginate topics and posts instead of using infinite scroll.",
+	"posts": "Post Pagination",
 	"topics": "Topic Pagination",
 	"posts-per-page": "Posts per Page",
 	"max-posts-per-page": "Maximum posts per page",
 	"categories": "Category Pagination",
 	"topics-per-page": "Topics per Page",
-	"max-topics-per-page": "Maximum topics per page"
+	"max-topics-per-page": "Maximum topics per page",
+	"categories-per-page": "Categories per page"
 }

public/language/en-GB/category.json

@@ -23,5 +23,6 @@
 	"notwatching.message": "You are not watching updates from this category and all subcategories",
 	"ignoring.message": "You are now ignoring updates from this category and all subcategories",
 
-	"watched-categories": "Watched categories"
+	"watched-categories": "Watched categories",
+	"x-more-categories": "%1 more categories"
 }

public/openapi/components/schemas/CategoryObject.yaml

@@ -72,6 +72,9 @@ CategoryObject:
         totalTopicCount:
           type: number
           description: The number of topics in the category
+        subCategoriesPerPage:
+          type: number
+          description: The number of subcategories to display on the categories and category page
     - type: object
       description: Optional properties that may or may not be present (except for `cid`, which is always present, and is only here as a hack to pass validation)
       properties:

public/openapi/read/categories.yaml

@@ -207,5 +207,6 @@ get:
                                       type: string
                             imageClass:
                               type: string
+              - $ref: ../components/schemas/Pagination.yaml#/Pagination
               - $ref: ../components/schemas/Breadcrumbs.yaml#/Breadcrumbs
               - $ref: ../components/schemas/CommonProps.yaml#/CommonProps

public/openapi/read/index.yaml

@@ -205,4 +205,5 @@ get:
                                       type: string
                             imageClass:
                               type: string
+              - $ref: ../components/schemas/Pagination.yaml#/Pagination
               - $ref: ../components/schemas/CommonProps.yaml#/CommonProps

public/openapi/read/search.yaml

@@ -25,7 +25,7 @@ get:
                     type: string
                   term:
                     type: string
-                  categories:
+                  allCategories:
                     type: array
                     items:
                       type: object
@@ -36,7 +36,7 @@ get:
                             - type: number
                         text:
                           type: string
-                  categoriesCount:
+                  allCategoriesCount:
                     type: number
                   expandSearch:
                     type: boolean
@@ -64,8 +64,8 @@ get:
                   - time
                   - multiplePages
                   - search_query
-                  - categories
+                  - allCategories
-                  - categoriesCount
+                  - allCategoriesCount
                   - expandSearch
                   - showAsPosts
                   - showAsTopics

public/src/client/category.js

@@ -36,6 +36,8 @@ define('forum/category', [
 
 		handleIgnoreWatch(cid);
 
+		handleLoadMoreSubcategories();
+
 		$(window).trigger('action:topics.loaded', { topics: ajaxify.data.topics });
 		$(window).trigger('action:category.loaded', { cid: ajaxify.data.cid });
 	};
@@ -74,6 +76,34 @@ define('forum/category', [
 		});
 	}
 
+	function handleLoadMoreSubcategories() {
+		$('[component="category/load-more-subcategories"]').on('click', function () {
+			var btn = $(this);
+			socket.emit('categories.loadMoreSubCategories', {
+				cid: ajaxify.data.cid,
+				start: ajaxify.data.nextSubCategoryStart,
+			}, function (err, data) {
+				if (err) {
+					return app.alertError(err);
+				}
+				btn.toggleClass('hidden', !data.length || data.length < ajaxify.data.subCategoriesPerPage);
+				if (!data.length) {
+					return;
+				}
+				app.parseAndTranslate('category', 'children', { children: data }, function (html) {
+					html.find('.timeago').timeago();
+					$('[component="category/subcategory/container"]').append(html);
+					utils.makeNumbersHumanReadable(html.find('.human-readable-number'));
+					app.createUserTooltips(html);
+					ajaxify.data.nextSubCategoryStart += ajaxify.data.subCategoriesPerPage;
+					ajaxify.data.subCategoriesLeft -= data.length;
+					btn.translateText('[[category:x-more-categories, ' + ajaxify.data.subCategoriesLeft + ']]');
+				});
+			});
+			return false;
+		});
+	}
+
 	Category.toTop = function () {
 		navigator.scrollTop(0);
 	};

src/categories/create.js

@@ -39,6 +39,7 @@ module.exports = function (Categories) {
 			class: (data.class ? data.class : 'col-md-3 col-xs-6'),
 			imageClass: 'cover',
 			isSection: 0,
+			subCategoriesPerPage: 10,
 		};
 
 		if (data.backgroundImage) {

src/categories/data.js

@@ -10,7 +10,7 @@ const utils = require('../utils');
 const intFields = [
 	'cid', 'parentCid', 'disabled', 'isSection', 'order',
 	'topic_count', 'post_count', 'numRecentReplies',
-	'minTags', 'maxTags', 'postQueue',
+	'minTags', 'maxTags', 'postQueue', 'subCategoriesPerPage',
 ];
 
 module.exports = function (Categories) {

src/controllers/categories.js

@@ -1,10 +1,13 @@
 'use strict';
 
 const nconf = require('nconf');
+const _ = require('lodash');
 
 const categories = require('../categories');
 const meta = require('../meta');
+const pagination = require('../pagination');
 const helpers = require('./helpers');
+const privileges = require('../privileges');
 
 const categoriesController = module.exports;
 
@@ -17,15 +20,45 @@ categoriesController.list = async function (req, res) {
 		content: 'website',
 	}];
 
-	const categoryData = await categories.getCategoriesByPrivilege('categories:cid', req.uid, 'find');
+	const allRootCids = await categories.getAllCidsFromSet('cid:0:children');
+	const rootCids = await privileges.categories.filterCids('find', allRootCids, req.uid);
+	const pageCount = Math.max(1, Math.ceil(rootCids.length / meta.config.categoriesPerPage));
+	const page = Math.min(parseInt(req.query.page, 10) || 1, pageCount);
+	const start = Math.max(0, (page - 1) * meta.config.categoriesPerPage);
+	const stop = start + meta.config.categoriesPerPage - 1;
+	const pageCids = rootCids.slice(start, stop + 1);
+
+	const allChildCids = _.flatten(await Promise.all(pageCids.map(cid => categories.getChildrenCids(cid))));
+	const childCids = await privileges.categories.filterCids('find', allChildCids, req.uid);
+	const categoryData = await categories.getCategories(pageCids.concat(childCids), req.uid);
 	const tree = categories.getTree(categoryData, 0);
 	await categories.getRecentTopicReplies(categoryData, req.uid, req.query);
 
 	const data = {
 		title: meta.config.homePageTitle || '[[pages:home]]',
 		categories: tree,
+		pagination: pagination.create(page, pageCount, req.query),
 	};
 
+	data.categories.forEach(function (category) {
+		if (category) {
+			if (Array.isArray(category.children)) {
+				category.children = category.children.slice(0, category.subCategoriesPerPage);
+				category.children.forEach(function (child) {
+					child.children = undefined;
+				});
+			}
+			if (Array.isArray(category.posts) && category.posts.length && category.posts[0]) {
+				category.teaser = {
+					url: nconf.get('relative_path') + '/post/' + category.posts[0].pid,
+					timestampISO: category.posts[0].timestampISO,
+					pid: category.posts[0].pid,
+					topic: category.posts[0].topic,
+				};
+			}
+		}
+	});
+
 	if (req.originalUrl.startsWith(nconf.get('relative_path') + '/api/categories') || req.originalUrl.startsWith(nconf.get('relative_path') + '/categories')) {
 		data.title = '[[pages:categories]]';
 		data.breadcrumbs = helpers.buildBreadcrumbs([{ text: data.title }]);
@@ -35,16 +68,5 @@ categoriesController.list = async function (req, res) {
 		});
 	}
 
-	data.categories.forEach(function (category) {
-		if (category && Array.isArray(category.posts) && category.posts.length && category.posts[0]) {
-			category.teaser = {
-				url: nconf.get('relative_path') + '/post/' + category.posts[0].pid,
-				timestampISO: category.posts[0].timestampISO,
-				pid: category.posts[0].pid,
-				topic: category.posts[0].topic,
-			};
-		}
-	});
-
 	res.render('categories', data);
 };

src/controllers/category.js

@@ -100,6 +100,13 @@ categoryController.get = async function (req, res, next) {
 		const allCategories = [];
 		categories.flattenCategories(allCategories, categoryData.children);
 		await categories.getRecentTopicReplies(allCategories, req.uid, req.query);
+		categoryData.subCategoriesLeft = Math.max(0, categoryData.children.length - categoryData.subCategoriesPerPage);
+		categoryData.hasMoreSubCategories = categoryData.children.length > categoryData.subCategoriesPerPage;
+		categoryData.nextSubCategoryStart = categoryData.subCategoriesPerPage;
+		categoryData.children = categoryData.children.slice(0, categoryData.subCategoriesPerPage);
+		categoryData.children.forEach(function (child) {
+			child.children = undefined;
+		});
 	}
 
 	categoryData.title = translator.escape(categoryData.name);

src/controllers/search.js

@@ -78,10 +78,8 @@ searchController.search = async function (req, res, next) {
 		return res.json(searchData);
 	}
 
-	if (['titles', 'titlesposts', 'posts'].includes(req.query.in)) {
+	searchData.allCategories = categoriesData;
-		searchData.categories = categoriesData;
+	searchData.allCategoriesCount = Math.max(10, Math.min(20, categoriesData.length));
-		searchData.categoriesCount = Math.max(10, Math.min(20, categoriesData.length));
-	}
 
 	searchData.breadcrumbs = helpers.buildBreadcrumbs([{ text: '[[global:search]]' }]);
 	searchData.expandSearch = !req.query.term;

src/socket.io/categories.js

@@ -153,4 +153,21 @@ SocketCategories.getCategory = async function (socket, cid) {
 	// return await apiController.getCategoryData(cid, socket.uid);
 };
 
+SocketCategories.loadMoreSubCategories = async function (socket, data) {
+	if (!data || !data.cid || !(parseInt(data.start, 10) > 0)) {
+		throw new Error('[[error:invalid-data]]');
+	}
+	const allowed = await privileges.categories.can('read', data.cid, socket.uid);
+	if (!allowed) {
+		throw new Error('[[error:no-privileges]]');
+	}
+	const category = await categories.getCategoryData(data.cid);
+	await categories.getChildrenTree(category, socket.uid);
+	const allCategories = [];
+	categories.flattenCategories(allCategories, category.children);
+	await categories.getRecentTopicReplies(allCategories, socket.uid);
+	const start = parseInt(data.start, 10);
+	return category.children.slice(start, start + category.subCategoriesPerPage);
+};
+
 require('../promisify')(SocketCategories);

src/upgrades/1.17.0/subcategories_per_page.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const db = require('../../database');
+const batch = require('../../batch');
+
+module.exports = {
+	name: 'Create subCategoriesPerPage property for categories',
+	timestamp: Date.UTC(2021, 0, 31),
+	method: async function () {
+		const progress = this.progress;
+
+		await batch.processSortedSet('categories:cid', async function (cids) {
+			const keys = cids.map(cid => 'category:' + cid);
+			await db.setObject(keys, {
+				subCategoriesPerPage: 10,
+			});
+			progress.incr(cids.length);
+		}, {
+			batch: 500,
+			progress: progress,
+		});
+	},
+};

src/views/admin/manage/category.tpl

@@ -82,7 +82,15 @@
 					</div>
 				</fieldset>
 				<fieldset class="row">
-					<div class="col-sm-6 col-xs-12">
+					<div class="col-sm-4 col-xs-12">
+						<div class="form-group">
+							<label for="cid-subcategories-per-page">
+								[[admin/manage/categories:subcategories-per-page]]
+							</label>
+							<input id="cid-subcategories-per-page" type="text" class="form-control" data-name="subCategoriesPerPage" value="{category.subCategoriesPerPage}" />
+						</div>
+					</div>
+					<div class="col-sm-4 col-xs-12">
 						<div class="form-group">
 							<label for="cid-min-tags">
 								[[admin/settings/tags:min-per-topic]]
@@ -90,7 +98,7 @@
 							<input id="cid-min-tags" type="text" class="form-control" data-name="minTags" value="{category.minTags}" />
 						</div>
 					</div>
-					<div class="col-sm-6 col-xs-12">
+					<div class="col-sm-4 col-xs-12">
 						<div class="form-group">
 							<label for="cid-max-tags">
 								[[admin/settings/tags:max-per-topic]]

src/views/admin/settings/pagination.tpl

@@ -15,7 +15,7 @@
 </div>
 
 <div class="row">
-	<div class="col-sm-2 col-xs-12 settings-header">[[admin/settings/pagination:topics]]</div>
+	<div class="col-sm-2 col-xs-12 settings-header">[[admin/settings/pagination:posts]]</div>
 	<div class="col-sm-10 col-xs-12">
 		<form>
 			<strong>[[admin/settings/pagination:posts-per-page]]</strong><br /> <input type="text" class="form-control" value="20" data-field="postsPerPage"><br/>
@@ -25,7 +25,7 @@
 </div>
 
 <div class="row">
-	<div class="col-sm-2 col-xs-12 settings-header">[[admin/settings/pagination:categories]]</div>
+	<div class="col-sm-2 col-xs-12 settings-header">[[admin/settings/pagination:topics]]</div>
 	<div class="col-sm-10 col-xs-12">
 		<form>
 			<strong>[[admin/settings/pagination:topics-per-page]]</strong><br /> <input type="text" class="form-control" value="20" data-field="topicsPerPage"><br />
@@ -34,4 +34,13 @@
 	</div>
 </div>
 
+<div class="row">
+	<div class="col-sm-2 col-xs-12 settings-header">[[admin/settings/pagination:categories]]</div>
+	<div class="col-sm-10 col-xs-12">
+		<form>
+			<strong>[[admin/settings/pagination:categories-per-page]]</strong><br /> <input type="text" class="form-control" value="50" data-field="categoriesPerPage"><br />
+		</form>
+	</div>
+</div>
+
 <!-- IMPORT admin/partials/settings/footer.tpl -->