Merge pull request #979 from miksago/ondemand-feeds

On-demand feeds

feeds/categories/.gitignore

@@ -1 +0,0 @@
-*.rss

feeds/topics/.gitignore

@@ -1 +0,0 @@
-*.rss

src/feed.js

@@ -12,30 +12,13 @@
 		async = require('async');
 
 	Feed.defaults = {
-		ttl: 60,
-		basePath: path.join(__dirname, '../', 'feeds'),
-		baseUrl: nconf.get('url') + '/feeds'
+		ttl: 60
 	};
 
-	Feed.saveFeed = function (location, feed, callback) {
-		var savePath = path.join(__dirname, '../', location);
-
-		fs.writeFile(savePath, feed.xml(), function (err) {
-			if (err) return winston.err(err);
-
-			if (callback) callback(err);
-		});
-	}
-
-	Feed.updateTopic = function (tid, callback) {
-		topics.getTopicWithPosts(tid, 0, 0, -1, true, function (err, topicData) {
+	Feed.forTopic = function (tid, callback) {
+		topics.getTopicWithPosts(tid, 0, 0, 25, true, function (err, topicData) {
 			if (err) {
-				if(callback) {
 				return callback(new Error('topic-invalid'));
-				} else {
-					winston.error(err.message);
-					return;
-				}
 			}
 
 			var description = topicData.posts.length ? topicData.posts[0].content : '';
@@ -45,7 +28,7 @@
 			var feed = new rss({
 					title: topicData.topic_name,
 					description: description,
-					feed_url: Feed.defaults.baseUrl + '/topics/' + tid + '.rss',
+					feed_url: nconf.get('url') + '/topic/' + tid + '.rss',
 					site_url: nconf.get('url') + '/topic/' + topicData.slug,
 					image_url: image_url,
 					author: author,
@@ -58,7 +41,7 @@
 				feed.pubDate = new Date(parseInt(topicData.posts[0].timestamp, 10)).toUTCString();
 			}
 
-			async.each(topicData.posts, function(postData, next) {
+			topicData.posts.forEach(function(postData) {
 				if (parseInt(postData.deleted, 10) === 0) {
 					dateStamp = new Date(parseInt(parseInt(postData.edited, 10) === 0 ? postData.timestamp : postData.edited, 10)).toUTCString();
 
@@ -70,31 +53,23 @@
 						date: dateStamp
 					});
 				}
-
-				next();
-			}, function() {
-				Feed.saveFeed('feeds/topics/' + tid + '.rss', feed, function (err) {
-					if (process.env.NODE_ENV === 'development') {
-						winston.info('[rss] Re-generated RSS Feed for tid ' + tid + '.');
-					}
-
-					if (callback) {
-						callback();
-					}
-				});
 			});
+
+			callback(null, feed.xml());
 		});
 
 	};
 
-	Feed.updateCategory = function (cid, callback) {
+	Feed.forCategory = function (cid, callback) {
 		categories.getCategoryById(cid, 0, 25, 0, function (err, categoryData) {
-			if (err) return callback(new Error('category-invalid'));
+			if (err) {
+				return callback(new Error('category-invalid'));
+			}
 
 			var feed = new rss({
 					title: categoryData.category_name,
 					description: categoryData.category_description,
-					feed_url: Feed.defaults.baseUrl + '/categories/' + cid + '.rss',
+					feed_url: nconf.get('url') + '/category/' + cid + '.rss',
 					site_url: nconf.get('url') + '/category/' + categoryData.category_id,
 					ttl: Feed.defaults.ttl
 				});
@@ -102,35 +77,29 @@
 			// Add pubDate if category has topics
 			if (categoryData.topics.length > 0) feed.pubDate = new Date(parseInt(categoryData.topics[0].lastposttime, 10)).toUTCString();
 
-			async.eachSeries(categoryData.topics, function(topicData, next) {
+			categoryData.topics.forEach(function(topicData) {
 				feed.item({
 					title: topicData.title,
 					url: nconf.get('url') + '/topic/' + topicData.slug,
 					author: topicData.username,
 					date: new Date(parseInt(topicData.lastposttime, 10)).toUTCString()
 				});
-
-				next();
-			}, function() {
-				Feed.saveFeed('feeds/categories/' + cid + '.rss', feed, function (err) {
-					if (process.env.NODE_ENV === 'development') {
-						winston.info('[rss] Re-generated RSS Feed for cid ' + cid + '.');
-					}
-
-					if (callback) {
-						callback();
-					}
-				});
 			});
+
+			callback(null, feed.xml());
 		});
 	};
 
-	Feed.updateRecent = function(callback) {
+	Feed.forRecent = function(callback) {
 		topics.getLatestTopics(0, 0, 19, undefined, function (err, recentData) {
+			if(err){
+				return callback(err);
+			}
+
 			var	feed = new rss({
 					title: 'Recently Active Topics',
 					description: 'A list of topics that have been active within the past 24 hours',
-					feed_url: Feed.defaults.baseUrl + '/recent.rss',
+					feed_url: nconf.get('url') + '/recent.rss',
 					site_url: nconf.get('url') + '/recent',
 					ttl: Feed.defaults.ttl
 				});
@@ -140,32 +109,29 @@
 				feed.pubDate = new Date(parseInt(recentData.topics[0].lastposttime, 10)).toUTCString();
 			}
 
-			async.eachSeries(recentData.topics, function(topicData, next) {
+			recentData.topics.forEach(function(topicData) {
 				feed.item({
 					title: topicData.title,
 					url: nconf.get('url') + '/topic/' + topicData.slug,
 					author: topicData.username,
 					date: new Date(parseInt(topicData.lastposttime, 10)).toUTCString()
 				});
-				next();
-			}, function() {
-				Feed.saveFeed('feeds/recent.rss', feed, function (err) {
-					if (process.env.NODE_ENV === 'development') {
-						winston.info('[rss] Re-generated "recent posts" RSS Feed.');
-					}
+			});
 
-					if (callback) callback();
-				});
-			});
+			callback(null, feed.xml());
 		});
 	};
 
-	Feed.updatePopular = function(callback) {
+	Feed.forPopular = function(callback) {
 		topics.getTopicsFromSet(0, 'topics:posts', 0, 19, function (err, popularData) {
+			if(err){
+				return callback(err);
+			}
+
 			var	feed = new rss({
 					title: 'Popular Topics',
 					description: 'A list of topics that are sorted by post count',
-					feed_url: Feed.defaults.baseUrl + '/popular.rss',
+					feed_url: nconf.get('url') + '/popular.rss',
 					site_url: nconf.get('url') + '/popular',
 					ttl: Feed.defaults.ttl
 				});
@@ -175,33 +141,16 @@
 				feed.pubDate = new Date(parseInt(popularData.topics[0].lastposttime, 10)).toUTCString();
 			}
 
-			async.eachSeries(popularData.topics, function(topicData, next) {
+			popularData.topics.forEach(function(topicData, next) {
 				feed.item({
 					title: topicData.title,
 					url: nconf.get('url') + '/topic/' + topicData.slug,
 					author: topicData.username,
 					date: new Date(parseInt(topicData.lastposttime, 10)).toUTCString()
 				});
-				next();
-			}, function() {
-				Feed.saveFeed('feeds/popular.rss', feed, function (err) {
-					if (process.env.NODE_ENV === 'development') {
-						winston.info('[rss] Re-generated "popular posts" RSS Feed.');
-					}
+			});
 
-					if (callback) callback();
-				});
-			});
-		});
-	};
-
-	Feed.loadFeed = function(rssPath, res) {
-		fs.readFile(rssPath, function (err, data) {
-			if (err) {
-				res.type('text').send(404, "Unable to locate an rss feed at this location.");
-			} else {
-				res.type('xml').set('Content-Length', data.length).send(data);
-			}
+			callback(null, feed.xml());
 		});
 	};
 }(exports));

src/postTools.js

@@ -11,8 +11,7 @@ var winston = require('winston'),
 	utils = require('../public/src/utils'),
 	plugins = require('./plugins'),
 	events = require('./events'),
-	meta = require('./meta'),
-	Feed = require('./feed');
+	meta = require('./meta');
 
 (function(PostTools) {
 	PostTools.isMain = function(pid, tid, callback) {
@@ -162,8 +161,6 @@ var winston = require('winston'),
 					}
 				});
 
-				Feed.updateTopic(postData.tid);
-				Feed.updateRecent();
 
 				callback(null);
 			});
@@ -207,8 +204,6 @@ var winston = require('winston'),
 					});
 				});
 
-				Feed.updateTopic(postData.tid);
-				Feed.updateRecent();
 
 				db.searchIndex('post', postData.content, pid);
 

src/posts.js

@@ -7,7 +7,6 @@ var db = require('./database'),
 	threadTools = require('./threadTools'),
 	postTools = require('./postTools'),
 	categories = require('./categories'),
-	feed = require('./feed'),
 	plugins = require('./plugins'),
 	meta = require('./meta'),
 

src/topics.js

@@ -15,7 +15,6 @@ var async = require('async'),
 	threadTools = require('./threadTools'),
 	postTools = require('./postTools'),
 	notifications = require('./notifications'),
-	feed = require('./feed'),
 	favourites = require('./favourites'),
 	meta = require('./meta');
 
@@ -57,8 +56,6 @@ var async = require('async'),
 				db.incrObjectField('category:' + cid, 'topic_count');
 				db.incrObjectField('global', 'topicCount');
 
-				feed.updateCategory(cid);
-
 				callback(null, tid);
 			});
 		});
@@ -155,15 +152,6 @@ var async = require('async'),
 				posts.create(uid, tid, content, next);
 			},
 			function(postData, next) {
-				db.getObjectField('tid:lastFeedUpdate', tid, function(err, lastFeedUpdate) {
-					var now = Date.now();
-					if(!lastFeedUpdate || parseInt(lastFeedUpdate, 10) < now - 3600000) {
-						feed.updateTopic(tid);
-						db.setObjectField('tid:lastFeedUpdate', tid, now);
-					}
-				});
-
-				feed.updateRecent();
 				threadTools.notifyFollowers(tid, postData.pid, uid);
 				user.sendPostNotificationToFollowers(uid, tid, postData.pid);
 
@@ -1185,7 +1173,6 @@ var async = require('async'),
 		db.sortedSetRemove('topics:views', tid);
 
 		Topics.getTopicField(tid, 'cid', function(err, cid) {
-			feed.updateCategory(cid);
 			db.incrObjectFieldBy('category:' + cid, 'topic_count', -1);
 		});
 	}
@@ -1199,7 +1186,6 @@ var async = require('async'),
 		});
 
 		Topics.getTopicField(tid, 'cid', function(err, cid) {
-			feed.updateCategory(cid);
 			db.incrObjectFieldBy('category:' + cid, 'topic_count', 1);
 		});
 	}

src/upgrade.js

@@ -560,6 +560,25 @@ Upgrade.upgrade = function(callback) {
 				winston.info('[2014/2/7] Updating category recent replies -- skipped');
 				next();
 			}
+		},
+		function(next) {
+			thisSchemaDate = new Date(2014, 1, 9, 20, 50).getTime();
+			if (schemaDate < thisSchemaDate) {
+				updatesMade = true;
+
+				db.delete('tid:lastFeedUpdate', function(err, uids) {
+					if(err) {
+						winston.err('Error upgrading '+ err.message);
+						process.exit();
+					} else {
+						winston.info('[2014/2/9] Remove Topic LastFeedUpdate value, as feeds are now on-demand');
+						next();
+					}
+				});
+			} else {
+				winston.info('[2014/2/9] Remove Topic LastFeedUpdate value, as feeds are now on-demand - skipped');
+				next();
+			}
 		}
 		// Add new schema updates here
 		// IMPORTANT: REMEMBER TO UPDATE VALUE OF latestSchema IN LINE 17!!!

src/webserver.js

@@ -435,6 +435,74 @@ module.exports.server = server;
 		userRoute.createRoutes(app);
 		apiRoute.createRoutes(app);
 
+		// RSS Feeds:
+		app.get('/topic/:topic_id.rss', function(req, res, next) {
+			var tid = req.params.topic_id;
+			var uid = req.user ? req.user.uid || 0 : 0;
+
+			ThreadTools.privileges(tid, uid, function(err, privileges) {
+				if(err) {
+					return next(err);
+				}
+
+				if(!privileges.read) {
+					return res.redirect('403');
+				}
+
+				feed.forTopic(tid, function(err, feedData){
+					if(err) {
+						return next(err);
+					}
+
+					res.type('xml').set('Content-Length', Buffer.byteLength(feedData)).send(feedData);
+				});
+			});
+		});
+
+		app.get('/category/:category_id.rss', function(req, res, next){
+			var cid = req.params.category_id;
+			var uid = req.user ? req.user.uid || 0 : 0;
+
+			CategoryTools.privileges(cid, uid, function(err, privileges) {
+				if(err) {
+					return next(err);
+				}
+
+				if(!privileges.read) {
+					return res.redirect('403');
+				}
+
+				feed.forCategory(cid, function(err, feedData){
+					if(err) {
+						return next(err);
+					}
+
+					res.type('xml').set('Content-Length', Buffer.byteLength(feedData)).send(feedData);
+				})
+			});
+		});
+
+		app.get('/recent.rss', function(req, res) {
+			feed.forRecent(function(err, feedData){
+				if(err) {
+					return next(err);
+				}
+
+				res.type('xml').set('Content-Length', Buffer.byteLength(feedData)).send(feedData);
+			});
+		});
+
+		app.get('/popular.rss', function(req, res) {
+			feed.forPopular(function(err, feedData){
+				if(err) {
+					return next(err);
+				}
+
+				res.type('xml').set('Content-Length', Buffer.byteLength(feedData)).send(feedData);
+			});
+		});
+
+
 		// Basic Routes (entirely client-side parsed, goal is to move the rest of the crap in this file into this one section)
 		(function () {
 			var routes = ['login', 'register', 'account', 'recent', 'popular', '403', '404', '500'],
@@ -517,45 +585,6 @@ module.exports.server = server;
 		app.get('/topic/:topic_id/:slug?', function (req, res, next) {
 			var tid = req.params.topic_id;
 
-			if (tid.match(/^\d+\.rss$/)) {
-				tid = tid.slice(0, -4);
-				var rssPath = path.join(__dirname, '../', 'feeds/topics', tid + '.rss'),
-					loadFeed = function () {
-						fs.readFile(rssPath, function (err, data) {
-							if (err) {
-								res.type('text').send(404, "Unable to locate an rss feed at this location.");
-							} else {
-								res.type('xml').set('Content-Length', data.length).send(data);
-							}
-						});
-
-					};
-
-				ThreadTools.privileges(tid, ((req.user) ? req.user.uid || 0 : 0), function(err, privileges) {
-					if(err) {
-						return next(err);
-					}
-
-					if(!privileges.read) {
-						return res.redirect('403');
-					}
-
-					if (!fs.existsSync(rssPath)) {
-						feed.updateTopic(tid, function (err) {
-							if (err) {
-								res.redirect('/404');
-							} else {
-								loadFeed();
-							}
-						});
-					} else {
-						loadFeed();
-					}
-				});
-
-				return;
-			}
-
 			async.waterfall([
 				function(next) {
 					ThreadTools.privileges(tid, ((req.user) ? req.user.uid || 0 : 0), function(err, privileges) {
@@ -699,45 +728,6 @@ module.exports.server = server;
 		app.get('/category/:category_id/:slug?', function (req, res, next) {
 			var cid = req.params.category_id;
 
-			if (cid.match(/^\d+\.rss$/)) {
-				cid = cid.slice(0, -4);
-				var rssPath = path.join(__dirname, '../', 'feeds/categories', cid + '.rss'),
-					loadFeed = function () {
-						fs.readFile(rssPath, function (err, data) {
-							if (err) {
-								res.type('text').send(404, "Unable to locate an rss feed at this location.");
-							} else {
-								res.type('xml').set('Content-Length', data.length).send(data);
-							}
-						});
-
-					};
-
-				CategoryTools.privileges(cid, ((req.user) ? req.user.uid || 0 : 0), function(err, privileges) {
-					if(err) {
-						return next(err);
-					}
-
-					if(!privileges.read) {
-						return res.redirect('403');
-					}
-
-					if (!fs.existsSync(rssPath)) {
-						feed.updateCategory(cid, function (err) {
-							if (err) {
-								res.redirect('/404');
-							} else {
-								loadFeed();
-							}
-						});
-					} else {
-						loadFeed();
-					}
-				});
-
-				return;
-			}
-
 			async.waterfall([
 				function(next) {
 					CategoryTools.privileges(cid, ((req.user) ? req.user.uid || 0 : 0), function(err, privileges) {
@@ -861,34 +851,6 @@ module.exports.server = server;
 			}
 		});
 
-		app.get('/recent.rss', function(req, res) {
-			var rssPath = path.join(__dirname, '../', 'feeds/recent.rss');
-
-			if (!fs.existsSync(rssPath)) {
-				feed.updateRecent(function (err) {
-					if (err) {
-						res.redirect('/404');
-					} else {
-						feed.loadFeed(rssPath, res);
-					}
-				});
-			} else {
-				feed.loadFeed(rssPath, res);
-			}
-		});
-
-		app.get('/popular.rss', function(req, res) {
-			var rssPath = path.join(__dirname, '../', 'feeds/popular.rss');
-
-			feed.updatePopular(function (err) {
-				if (err) {
-					res.redirect('/404');
-				} else {
-					feed.loadFeed(rssPath, res);
-				}
-			});
-		});
-
 		app.get('/recent/:term?', function (req, res) {
 			// TODO consolidate with /recent route as well -> that can be combined into this area. See "Basic Routes" near top.
 			app.build_header({