Nemcio/NodeBB
1
0
Fork 0
mirror of https://github.com/NodeBB/NodeBB.git synced 2025-11-07 14:35:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Node redis (#13500)

Browse Source 
* refactor: start migrating to node-redis

* few more zset fixes

* fix: db.scan

* fix: list methods

* fix set methods

* fix: hash methods

* use hasOwn, remove cloning

* sorted set fixes

* fix: so data is converted to strings before saving

otherwise node-redis throws below error
TypeError: "arguments[2]" must be of type "string | Buffer", got number instead.

* chore: remove comments

* fix: zrank string param

* use new close

* chore: up dbsearch

* test: add log

* test: more log

* test: log failing test

* test: catch errors in formatApiResponse

add await so exception goes to catch

* tetst: add log

* fix: dont set null/undefined values

* test: more fixes
This commit is contained in:
Barış Uşaklı
2025-06-18 13:04:57 -04:00
committed by GitHub
parent 3f7d415744
commit 14043ab0fd
21 changed files with 307 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
install/package.json
View File

@@ -98,7 +98,7 @@
"nconf": "0.13.0",
"nodebb-plugin-2factor": "7.5.10",
"nodebb-plugin-composer-default": "10.2.51",
"nodebb-plugin-dbsearch": "6.2.19",
"nodebb-plugin-dbsearch": "6.3.0",
"nodebb-plugin-emoji": "6.0.3",
"nodebb-plugin-emoji-android": "4.1.1",
"nodebb-plugin-markdown": "13.2.1",
@@ -122,7 +122,7 @@
"postcss-clean": "1.2.0",
"progress-webpack-plugin": "1.0.16",
"prompt": "1.3.0",
"ioredis": "5.6.1",
"redis": "5.5.6",
"rimraf": "6.0.1",
"rss": "1.2.2",
"rtlcss": "4.3.0",

1
src/activitypub/index.js
View File

@@ -132,7 +132,6 @@ ActivityPub.resolveInboxes = async (ids) => {
}, [[], []]);
const categoryData = await categories.getCategoriesFields(cids, ['inbox', 'sharedInbox']);
const userData = await user.getUsersFields(uids, ['inbox', 'sharedInbox']);
currentIds.forEach((id) => {
if (cids.includes(id)) {
const data = categoryData[cids.indexOf(id)];

5
src/controllers/activitypub/index.js
View File

@@ -145,14 +145,15 @@ Controller.postInbox = async (req, res) => {
const method = String(req.body.type).toLowerCase();
if (!activitypub.inbox.hasOwnProperty(method)) {
winston.warn(`[activitypub/inbox] Received Activity of type ${method} but unable to handle. Ignoring.`);
console.log('[activitypub/inbox] method not found', method, req.body);
return res.sendStatus(200);
}
try {
await activitypub.inbox[method](req);
await activitypub.record(req.body);
helpers.formatApiResponse(202, res);
await helpers.formatApiResponse(202, res);
} catch (e) {
helpers.formatApiResponse(500, res, e);
helpers.formatApiResponse(500, res, e).catch(err => winston.error(err.stack));
}
};

4
src/controllers/helpers.js
View File

@@ -448,6 +448,10 @@ helpers.getHomePageRoutes = async function (uid) {
};
helpers.formatApiResponse = async (statusCode, res, payload) => {
if (!res.hasOwnProperty('req')) {
console.log('formatApiResponse', statusCode, payload);
}
if (res.req.method === 'HEAD') {
return res.sendStatus(statusCode);
}

2
src/database/redis.js
View File

@@ -61,7 +61,7 @@ redisModule.checkCompatibilityVersion = function (version, callback) {
};
redisModule.close = async function () {
await redisModule.client.quit();
await redisModule.client.close();
if (redisModule.objectCache) {
redisModule.objectCache.reset();
}

39
src/database/redis/connection.js
View File

@@ -1,7 +1,7 @@
'use strict';
const nconf = require('nconf');
const Redis = require('ioredis');
const { createClient, createCluster, createSentinel } = require('redis');
const winston = require('winston');
const connection = module.exports;
@@ -13,28 +13,40 @@ connection.connect = async function (options) {
let cxn;
if (options.cluster) {
cxn = new Redis.Cluster(options.cluster, options.options);
} else if (options.sentinels) {
cxn = new Redis({
sentinels: options.sentinels,
const rootNodes = options.cluster.map(node => ({ url : `redis://${node.host}:${node.port}` }));
cxn = createCluster({
...options.options,
rootNodes: rootNodes,
});
} else if (options.sentinels) {
const sentinelRootNodes = options.sentinels.map(sentinel => ({ host: sentinel.host, port: sentinel.port }));
cxn = createSentinel({
...options.options,
name: 'sentinel-db',
sentinelRootNodes,
});
} else if (redis_socket_or_host && String(redis_socket_or_host).indexOf('/') >= 0) {
// If redis.host contains a path name character, use the unix dom sock connection. ie, /tmp/redis.sock
cxn = new Redis({
cxn = createClient({
...options.options,
path: redis_socket_or_host,
password: options.password,
db: options.database,
database: options.database,
socket: {
path: redis_socket_or_host,
reconnectStrategy: 3000,
},
});
} else {
// Else, connect over tcp/ip
cxn = new Redis({
cxn = createClient({
...options.options,
host: redis_socket_or_host,
port: options.port,
password: options.password,
db: options.database,
database: options.database,
socket: {
reconnectStrategy: 3000,
},
});
}
@@ -49,9 +61,14 @@ connection.connect = async function (options) {
});
cxn.on('ready', () => {
// back-compat with node_redis
cxn.batch = cxn.pipeline;
cxn.batch = cxn.multi;
resolve(cxn);
});
cxn.connect().then(() => {
winston.info('Connected to Redis successfully');
}).catch((err) => {
winston.error('Error connecting to Redis:', err);
});
if (options.password) {
cxn.auth(options.password);

50
src/database/redis/hash.js
View File

@@ -25,12 +25,13 @@ module.exports = function (module) {
if (!Object.keys(data).length) {
return;
}
const strObj = helpers.objectFieldsToString(data);
if (Array.isArray(key)) {
const batch = module.client.batch();
key.forEach(k => batch.hmset(k, data));
key.forEach(k => batch.hSet(k, strObj));
await helpers.execBatch(batch);
} else {
await module.client.hmset(key, data);
await module.client.hSet(key, strObj);
}
cache.del(key);
@@ -49,10 +50,16 @@ module.exports = function (module) {
const batch = module.client.batch();
data.forEach((item) => {
if (Object.keys(item[1]).length) {
batch.hmset(item[0], item[1]);
Object.keys(item[1]).forEach((key) => {
if (item[1][key] === undefined || item[1][key] === null) {
delete item[1][key];
}
});
if (Object.keys(item[1]).length) {
batch.hSet(item[0], helpers.objectFieldsToString(item[1]));
}
});
await helpers.execBatch(batch);
cache.del(data.map(item => item[0]));
};
@@ -61,12 +68,15 @@ module.exports = function (module) {
if (!field) {
return;
}
if (value === null || value === undefined) {
return;
}
if (Array.isArray(key)) {
const batch = module.client.batch();
key.forEach(k => batch.hset(k, field, value));
key.forEach(k => batch.hSet(k, field, String(value)));
await helpers.execBatch(batch);
} else {
await module.client.hset(key, field, value);
await module.client.hSet(key, field, String(value));
}
cache.del(key);
@@ -92,9 +102,9 @@ module.exports = function (module) {
const cachedData = {};
cache.getUnCachedKeys([key], cachedData);
if (cachedData[key]) {
return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null;
return Object.hasOwn(cachedData[key], field) ? cachedData[key][field] : null;
}
return await module.client.hget(key, String(field));
return await module.client.hGet(key, String(field));
};
module.getObjectFields = async function (key, fields) {
@@ -116,10 +126,10 @@ module.exports = function (module) {
let data = [];
if (unCachedKeys.length > 1) {
const batch = module.client.batch();
unCachedKeys.forEach(k => batch.hgetall(k));
unCachedKeys.forEach(k => batch.hGetAll(k));
data = await helpers.execBatch(batch);
} else if (unCachedKeys.length === 1) {
data = [await module.client.hgetall(unCachedKeys[0])];
data = [await module.client.hGetAll(unCachedKeys[0])];
}
// convert empty objects into null for back-compat with node_redis
@@ -149,21 +159,21 @@ module.exports = function (module) {
};
module.getObjectKeys = async function (key) {
return await module.client.hkeys(key);
return await module.client.hKeys(key);
};
module.getObjectValues = async function (key) {
return await module.client.hvals(key);
return await module.client.hVals(key);
};
module.isObjectField = async function (key, field) {
const exists = await module.client.hexists(key, field);
const exists = await module.client.hExists(key, String(field));
return exists === 1;
};
module.isObjectFields = async function (key, fields) {
const batch = module.client.batch();
fields.forEach(f => batch.hexists(String(key), String(f)));
fields.forEach(f => batch.hExists(String(key), String(f)));
const results = await helpers.execBatch(batch);
return Array.isArray(results) ? helpers.resultsToBool(results) : null;
};
@@ -174,7 +184,7 @@ module.exports = function (module) {
}
field = field.toString();
if (field) {
await module.client.hdel(key, field);
await module.client.hDel(key, field);
cache.del(key);
}
};
@@ -189,10 +199,10 @@ module.exports = function (module) {
}
if (Array.isArray(key)) {
const batch = module.client.batch();
key.forEach(k => batch.hdel(k, fields));
key.forEach(k => batch.hDel(k, fields));
await helpers.execBatch(batch);
} else {
await module.client.hdel(key, fields);
await module.client.hDel(key, fields);
}
cache.del(key);
@@ -214,10 +224,10 @@ module.exports = function (module) {
let result;
if (Array.isArray(key)) {
const batch = module.client.batch();
key.forEach(k => batch.hincrby(k, field, value));
key.forEach(k => batch.hIncrBy(k, field, value));
result = await helpers.execBatch(batch);
} else {
result = await module.client.hincrby(key, field, value);
result = await module.client.hIncrBy(key, field, value);
}
cache.del(key);
return Array.isArray(result) ? result.map(value => parseInt(value, 10)) : parseInt(result, 10);
@@ -231,7 +241,7 @@ module.exports = function (module) {
const batch = module.client.batch();
data.forEach((item) => {
for (const [field, value] of Object.entries(item[1])) {
batch.hincrby(item[0], field, value);
batch.hIncrBy(item[0], field, value);
}
});
await helpers.execBatch(batch);

40
src/database/redis/helpers.js
View File

@@ -5,13 +5,8 @@ const helpers = module.exports;
helpers.noop = function () {};
helpers.execBatch = async function (batch) {
const results = await batch.exec();
return results.map(([err, res]) => {
if (err) {
throw err;
}
return res;
});
const results = await batch.execAsPipeline();
return results;
};
helpers.resultsToBool = function (results) {
@@ -21,10 +16,29 @@ helpers.resultsToBool = function (results) {
return results;
};
helpers.zsetToObjectArray = function (data) {
const objects = new Array(data.length / 2);
for (let i = 0, k = 0; i < objects.length; i += 1, k += 2) {
objects[i] = { value: data[k], score: parseFloat(data[k + 1]) };
}
return objects;
helpers.objectFieldsToString = function (obj) {
const stringified = Object.fromEntries(
Object.entries(obj).map(([key, value]) => [key, String(value)])
);
return stringified;
};
helpers.normalizeLexRange = function (min, max, reverse) {
let minmin;
let maxmax;
if (reverse) {
minmin = '+';
maxmax = '-';
} else {
minmin = '-';
maxmax = '+';
}
if (min !== minmin && !min.match(/^[[(]/)) {
min = `[${min}`;
}
if (max !== maxmax && !max.match(/^[[(]/)) {
max = `[${max}`;
}
return { lmin: min, lmax: max };
};

22
src/database/redis/list.js
View File

@@ -1,27 +1,25 @@
'use strict';
module.exports = function (module) {
const helpers = require('./helpers');
module.listPrepend = async function (key, value) {
if (!key) {
return;
}
await module.client.lpush(key, value);
await module.client.lPush(key, Array.isArray(value) ? value.map(String) : String(value));
};
module.listAppend = async function (key, value) {
if (!key) {
return;
}
await module.client.rpush(key, value);
await module.client.rPush(key, Array.isArray(value) ? value.map(String) : String(value));
};
module.listRemoveLast = async function (key) {
if (!key) {
return;
}
return await module.client.rpop(key);
return await module.client.rPop(key);
};
module.listRemoveAll = async function (key, value) {
@@ -29,11 +27,11 @@ module.exports = function (module) {
return;
}
if (Array.isArray(value)) {
const batch = module.client.batch();
value.forEach(value => batch.lrem(key, 0, value));
await helpers.execBatch(batch);
const batch = module.client.multi();
value.forEach(value => batch.lRem(key, 0, value));
await batch.execAsPipeline();
} else {
await module.client.lrem(key, 0, value);
await module.client.lRem(key, 0, value);
}
};
@@ -41,17 +39,17 @@ module.exports = function (module) {
if (!key) {
return;
}
await module.client.ltrim(key, start, stop);
await module.client.lTrim(key, start, stop);
};
module.getListRange = async function (key, start, stop) {
if (!key) {
return;
}
return await module.client.lrange(key, start, stop);
return await module.client.lRange(key, start, stop);
};
module.listLength = async function (key) {
return await module.client.llen(key);
return await module.client.lLen(key);
};
};

22
src/database/redis/main.js
View File

@@ -4,7 +4,7 @@ module.exports = function (module) {
const helpers = require('./helpers');
module.flushdb = async function () {
await module.client.send_command('flushdb', []);
await module.client.sendCommand(['FLUSHDB']);
};
module.emptydb = async function () {
@@ -32,9 +32,9 @@ module.exports = function (module) {
const seen = Object.create(null);
do {
/* eslint-disable no-await-in-loop */
const res = await module.client.scan(cursor, 'MATCH', params.match, 'COUNT', 10000);
cursor = res[0];
const values = res[1].filter((value) => {
const res = await module.client.scan(cursor, { MATCH: params.match, COUNT: 10000 });
cursor = res.cursor;
const values = res.keys.filter((value) => {
const isSeen = !!seen[value];
if (!isSeen) {
seen[value] = 1;
@@ -67,7 +67,7 @@ module.exports = function (module) {
if (!keys || !Array.isArray(keys) || !keys.length) {
return [];
}
return await module.client.mget(keys);
return await module.client.mGet(keys);
};
module.set = async function (key, value) {
@@ -96,26 +96,26 @@ module.exports = function (module) {
};
module.expire = async function (key, seconds) {
await module.client.expire(key, seconds);
await module.client.EXPIRE(key, seconds);
};
module.expireAt = async function (key, timestamp) {
await module.client.expireat(key, timestamp);
await module.client.EXPIREAT(key, timestamp);
};
module.pexpire = async function (key, ms) {
await module.client.pexpire(key, ms);
await module.client.PEXPIRE(key, ms);
};
module.pexpireAt = async function (key, timestamp) {
await module.client.pexpireat(key, timestamp);
await module.client.PEXPIREAT(key, timestamp);
};
module.ttl = async function (key) {
return await module.client.ttl(key);
return await module.client.TTL(key);
};
module.pttl = async function (key) {
return await module.client.pttl(key);
return await module.client.PTTL(key);
};
};

24
src/database/redis/sets.js
View File

@@ -10,7 +10,7 @@ module.exports = function (module) {
if (!value.length) {
return;
}
await module.client.sadd(key, value);
await module.client.sAdd(key, value.map(String));
};
module.setsAdd = async function (keys, value) {
@@ -18,7 +18,7 @@ module.exports = function (module) {
return;
}
const batch = module.client.batch();
keys.forEach(k => batch.sadd(String(k), String(value)));
keys.forEach(k => batch.sAdd(String(k), String(value)));
await helpers.execBatch(batch);
};
@@ -34,57 +34,57 @@ module.exports = function (module) {
}
const batch = module.client.batch();
key.forEach(k => batch.srem(String(k), value));
key.forEach(k => batch.sRem(String(k), value.map(String)));
await helpers.execBatch(batch);
};
module.setsRemove = async function (keys, value) {
const batch = module.client.batch();
keys.forEach(k => batch.srem(String(k), value));
keys.forEach(k => batch.sRem(String(k), String(value)));
await helpers.execBatch(batch);
};
module.isSetMember = async function (key, value) {
const result = await module.client.sismember(key, value);
const result = await module.client.sIsMember(key, String(value));
return result === 1;
};
module.isSetMembers = async function (key, values) {
const batch = module.client.batch();
values.forEach(v => batch.sismember(String(key), String(v)));
values.forEach(v => batch.sIsMember(String(key), String(v)));
const results = await helpers.execBatch(batch);
return results ? helpers.resultsToBool(results) : null;
};
module.isMemberOfSets = async function (sets, value) {
const batch = module.client.batch();
sets.forEach(s => batch.sismember(String(s), String(value)));
sets.forEach(s => batch.sIsMember(String(s), String(value)));
const results = await helpers.execBatch(batch);
return results ? helpers.resultsToBool(results) : null;
};
module.getSetMembers = async function (key) {
return await module.client.smembers(key);
return await module.client.sMembers(key);
};
module.getSetsMembers = async function (keys) {
const batch = module.client.batch();
keys.forEach(k => batch.smembers(String(k)));
keys.forEach(k => batch.sMembers(String(k)));
return await helpers.execBatch(batch);
};
module.setCount = async function (key) {
return await module.client.scard(key);
return await module.client.sCard(key);
};
module.setsCount = async function (keys) {
const batch = module.client.batch();
keys.forEach(k => batch.scard(String(k)));
keys.forEach(k => batch.sCard(String(k)));
return await helpers.execBatch(batch);
};
module.setRemoveRandom = async function (key) {
return await module.client.spop(key);
return await module.client.sPop(key);
};
return module;

239
src/database/redis/sorted.js
View File

@@ -11,34 +11,74 @@ module.exports = function (module) {
require('./sorted/intersect')(module);
module.getSortedSetRange = async function (key, start, stop) {
return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', false);
return await sortedSetRange(key, start, stop, '-inf', '+inf', false, false, false);
};
module.getSortedSetRevRange = async function (key, start, stop) {
return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', false);
return await sortedSetRange(key, start, stop, '-inf', '+inf', false, true, false);
};
module.getSortedSetRangeWithScores = async function (key, start, stop) {
return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', true);
return await sortedSetRange(key, start, stop, '-inf', '+inf', true, false, false);
};
module.getSortedSetRevRangeWithScores = async function (key, start, stop) {
return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', true);
return await sortedSetRange(key, start, stop, '-inf', '+inf', true, true, false);
};
async function sortedSetRange(method, key, start, stop, min, max, withScores) {
module.getSortedSetRangeByScore = async function (key, start, count, min, max) {
return await sortedSetRangeByScore(key, start, count, min, max, false, false);
};
module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) {
return await sortedSetRangeByScore(key, start, count, max, min, false, true);
};
module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) {
return await sortedSetRangeByScore(key, start, count, min, max, true, false);
};
module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) {
return await sortedSetRangeByScore(key, start, count, max, min, true, true);
};
async function sortedSetRangeByScore(key, start, count, min, max, withScores, rev) {
if (parseInt(count, 10) === 0) {
return [];
}
const stop = (parseInt(count, 10) === -1) ? -1 : (start + count - 1);
return await sortedSetRange(key, start, stop, min, max, withScores, rev, true);
}
async function sortedSetRange(key, start, stop, min, max, withScores, rev, byScore) {
const opts = {};
const cmd = withScores ? 'zRangeWithScores' : 'zRange';
if (byScore) {
opts.BY = 'SCORE';
opts.LIMIT = { offset: start, count: stop !== -1 ? stop + 1 : stop };
}
if (rev) {
opts.REV = true;
}
if (Array.isArray(key)) {
if (!key.length) {
return [];
}
const batch = module.client.batch();
key.forEach(key => batch[method](genParams(method, key, 0, stop, min, max, true)));
if (byScore) {
key.forEach(key => batch.zRangeWithScores(key, min, max, {
...opts,
LIMIT: { offset: 0, count: stop !== -1 ? stop + 1 : stop },
}));
} else {
key.forEach(key => batch.zRangeWithScores(key, 0, stop, { ...opts }));
}
const data = await helpers.execBatch(batch);
const batchData = data.map(setData => helpers.zsetToObjectArray(setData));
let objects = dbHelpers.mergeBatch(batchData, 0, stop, method === 'zrange' ? 1 : -1);
const batchData = data;
let objects = dbHelpers.mergeBatch(batchData, 0, stop, rev ? -1 : 1);
if (start > 0) {
objects = objects.slice(start, stop !== -1 ? stop + 1 : undefined);
}
@@ -48,63 +88,25 @@ module.exports = function (module) {
return objects;
}
const params = genParams(method, key, start, stop, min, max, withScores);
const data = await module.client[method](params);
let data;
if (byScore) {
data = await module.client[cmd](key, min, max, opts);
} else {
data = await module.client[cmd](key, start, stop, opts);
}
if (!withScores) {
return data;
}
const objects = helpers.zsetToObjectArray(data);
return objects;
}
function genParams(method, key, start, stop, min, max, withScores) {
const params = {
zrevrange: [key, start, stop],
zrange: [key, start, stop],
zrangebyscore: [key, min, max],
zrevrangebyscore: [key, max, min],
};
if (withScores) {
params[method].push('WITHSCORES');
}
if (method === 'zrangebyscore' || method === 'zrevrangebyscore') {
const count = stop !== -1 ? stop - start + 1 : stop;
params[method].push('LIMIT', start, count);
}
return params[method];
}
module.getSortedSetRangeByScore = async function (key, start, count, min, max) {
return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, false);
};
module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) {
return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, false);
};
module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) {
return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, true);
};
module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) {
return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, true);
};
async function sortedSetRangeByScore(method, key, start, count, min, max, withScores) {
if (parseInt(count, 10) === 0) {
return [];
}
const stop = (parseInt(count, 10) === -1) ? -1 : (start + count - 1);
return await sortedSetRange(method, key, start, stop, min, max, withScores);
return data;
}
module.sortedSetCount = async function (key, min, max) {
return await module.client.zcount(key, min, max);
return await module.client.zCount(key, min, max);
};
module.sortedSetCard = async function (key) {
return await module.client.zcard(key);
return await module.client.zCard(key);
};
module.sortedSetsCard = async function (keys) {
@@ -112,7 +114,7 @@ module.exports = function (module) {
return [];
}
const batch = module.client.batch();
keys.forEach(k => batch.zcard(String(k)));
keys.forEach(k => batch.zCard(String(k)));
return await helpers.execBatch(batch);
};
@@ -125,26 +127,26 @@ module.exports = function (module) {
}
const batch = module.client.batch();
if (min !== '-inf' || max !== '+inf') {
keys.forEach(k => batch.zcount(String(k), min, max));
keys.forEach(k => batch.zCount(String(k), min, max));
} else {
keys.forEach(k => batch.zcard(String(k)));
keys.forEach(k => batch.zCard(String(k)));
}
const counts = await helpers.execBatch(batch);
return counts.reduce((acc, val) => acc + val, 0);
};
module.sortedSetRank = async function (key, value) {
return await module.client.zrank(key, value);
return await module.client.zRank(key, String(value));
};
module.sortedSetRevRank = async function (key, value) {
return await module.client.zrevrank(key, value);
return await module.client.zRevRank(key, String(value));
};
module.sortedSetsRanks = async function (keys, values) {
const batch = module.client.batch();
for (let i = 0; i < values.length; i += 1) {
batch.zrank(keys[i], String(values[i]));
batch.zRank(keys[i], String(values[i]));
}
return await helpers.execBatch(batch);
};
@@ -152,7 +154,7 @@ module.exports = function (module) {
module.sortedSetsRevRanks = async function (keys, values) {
const batch = module.client.batch();
for (let i = 0; i < values.length; i += 1) {
batch.zrevrank(keys[i], String(values[i]));
batch.zRevRank(keys[i], String(values[i]));
}
return await helpers.execBatch(batch);
};
@@ -160,7 +162,7 @@ module.exports = function (module) {
module.sortedSetRanks = async function (key, values) {
const batch = module.client.batch();
for (let i = 0; i < values.length; i += 1) {
batch.zrank(key, String(values[i]));
batch.zRank(key, String(values[i]));
}
return await helpers.execBatch(batch);
};
@@ -168,7 +170,7 @@ module.exports = function (module) {
module.sortedSetRevRanks = async function (key, values) {
const batch = module.client.batch();
for (let i = 0; i < values.length; i += 1) {
batch.zrevrank(key, String(values[i]));
batch.zRevRank(key, String(values[i]));
}
return await helpers.execBatch(batch);
};
@@ -177,8 +179,7 @@ module.exports = function (module) {
if (!key || value === undefined) {
return null;
}
const score = await module.client.zscore(key, value);
const score = await module.client.zScore(key, String(value));
return score === null ? score : parseFloat(score);
};
@@ -187,7 +188,7 @@ module.exports = function (module) {
return [];
}
const batch = module.client.batch();
keys.forEach(key => batch.zscore(String(key), String(value)));
keys.forEach(key => batch.zScore(String(key), String(value)));
const scores = await helpers.execBatch(batch);
return scores.map(d => (d === null ? d : parseFloat(d)));
};
@@ -197,7 +198,7 @@ module.exports = function (module) {
return [];
}
const batch = module.client.batch();
values.forEach(value => batch.zscore(String(key), String(value)));
values.forEach(value => batch.zScore(String(key), String(value)));
const scores = await helpers.execBatch(batch);
return scores.map(d => (d === null ? d : parseFloat(d)));
};
@@ -211,9 +212,9 @@ module.exports = function (module) {
if (!values.length) {
return [];
}
const batch = module.client.batch();
values.forEach(v => batch.zscore(key, String(v)));
const results = await helpers.execBatch(batch);
const batch = module.client.multi();
values.forEach(v => batch.zScore(key, String(v)));
const results = await batch.execAsPipeline();
return results.map(utils.isNumber);
};
@@ -221,20 +222,18 @@ module.exports = function (module) {
if (!Array.isArray(keys) || !keys.length) {
return [];
}
const batch = module.client.batch();
keys.forEach(k => batch.zscore(k, String(value)));
const results = await helpers.execBatch(batch);
const batch = module.client.multi();
keys.forEach(k => batch.zScore(k, String(value)));
const results = await batch.execAsPipeline();
return results.map(utils.isNumber);
};
module.getSortedSetMembers = async function (key) {
return await module.client.zrange(key, 0, -1);
return await module.client.zRange(key, 0, -1);
};
module.getSortedSetMembersWithScores = async function (key) {
return helpers.zsetToObjectArray(
await module.client.zrange(key, 0, -1, 'WITHSCORES')
);
return await module.client.zRangeWithScores(key, 0, -1);
};
module.getSortedSetsMembers = async function (keys) {
@@ -242,7 +241,7 @@ module.exports = function (module) {
return [];
}
const batch = module.client.batch();
keys.forEach(k => batch.zrange(k, 0, -1));
keys.forEach(k => batch.zRange(k, 0, -1));
return await helpers.execBatch(batch);
};
@@ -251,65 +250,52 @@ module.exports = function (module) {
return [];
}
const batch = module.client.batch();
keys.forEach(k => batch.zrange(k, 0, -1, 'WITHSCORES'));
keys.forEach(k => batch.zRangeWithScores(k, 0, -1));
const res = await helpers.execBatch(batch);
return res.map(helpers.zsetToObjectArray);
return res;
};
module.sortedSetIncrBy = async function (key, increment, value) {
const newValue = await module.client.zincrby(key, increment, value);
const newValue = await module.client.zIncrBy(key, increment, String(value));
return parseFloat(newValue);
};
module.sortedSetIncrByBulk = async function (data) {
const multi = module.client.multi();
data.forEach((item) => {
multi.zincrby(item[0], item[1], item[2]);
multi.zIncrBy(item[0], item[1], String(item[2]));
});
const result = await multi.exec();
return result.map(item => item && parseFloat(item[1]));
return result;
};
module.getSortedSetRangeByLex = async function (key, min, max, start, count) {
return await sortedSetLex('zrangebylex', false, key, min, max, start, count);
module.getSortedSetRangeByLex = async function (key, min, max, start = 0, count = -1) {
const { lmin, lmax } = helpers.normalizeLexRange(min, max, false);
return await module.client.zRange(key, lmin, lmax, {
BY: 'LEX',
LIMIT: { offset: start, count: count },
});
};
module.getSortedSetRevRangeByLex = async function (key, max, min, start, count) {
return await sortedSetLex('zrevrangebylex', true, key, max, min, start, count);
module.getSortedSetRevRangeByLex = async function (key, max, min, start = 0, count = -1) {
const { lmin, lmax } = helpers.normalizeLexRange(max, min, true);
return await module.client.zRange(key, lmin, lmax, {
REV: true,
BY: 'LEX',
LIMIT: { offset: start, count: count },
});
};
module.sortedSetRemoveRangeByLex = async function (key, min, max) {
await sortedSetLex('zremrangebylex', false, key, min, max);
const { lmin, lmax } = helpers.normalizeLexRange(min, max, false);
await module.client.zRemRangeByLex(key, lmin, lmax);
};
module.sortedSetLexCount = async function (key, min, max) {
return await sortedSetLex('zlexcount', false, key, min, max);
const { lmin, lmax } = helpers.normalizeLexRange(min, max, false);
return await module.client.zLexCount(key, lmin, lmax);
};
async function sortedSetLex(method, reverse, key, min, max, start, count) {
let minmin;
let maxmax;
if (reverse) {
minmin = '+';
maxmax = '-';
} else {
minmin = '-';
maxmax = '+';
}
if (min !== minmin && !min.match(/^[[(]/)) {
min = `[${min}`;
}
if (max !== maxmax && !max.match(/^[[(]/)) {
max = `[${max}`;
}
const args = [key, min, max];
if (count) {
args.push('LIMIT', start, count);
}
return await module.client[method](args);
}
module.getSortedSetScan = async function (params) {
let cursor = '0';
@@ -318,20 +304,19 @@ module.exports = function (module) {
const seen = Object.create(null);
do {
/* eslint-disable no-await-in-loop */
const res = await module.client.zscan(params.key, cursor, 'MATCH', params.match, 'COUNT', 5000);
cursor = res[0];
const res = await module.client.zScan(params.key, cursor, { MATCH: params.match, COUNT: 5000 });
cursor = res.cursor;
done = cursor === '0';
const data = res[1];
for (let i = 0; i < data.length; i += 2) {
const value = data[i];
if (!seen[value]) {
seen[value] = 1;
for (let i = 0; i < res.members.length; i ++) {
const item = res.members[i];
if (!seen[item.value]) {
seen[item.value] = 1;
if (params.withScores) {
returnData.push({ value: value, score: parseFloat(data[i + 1]) });
returnData.push({ value: item.value, score: parseFloat(item.score) });
} else {
returnData.push(value);
returnData.push(item.value);
}
if (params.limit && returnData.length >= params.limit) {
done = true;

24
src/database/redis/sorted/add.js
View File

@@ -1,7 +1,6 @@
'use strict';
module.exports = function (module) {
const helpers = require('../helpers');
const utils = require('../../../utils');
module.sortedSetAdd = async function (key, score, value) {
@@ -14,7 +13,8 @@ module.exports = function (module) {
if (!utils.isNumber(score)) {
throw new Error(`[[error:invalid-score, ${score}]]`);
}
await module.client.zadd(key, score, String(value));
await module.client.zAdd(key, { score, value: String(value) });
};
async function sortedSetAddMulti(key, scores, values) {
@@ -30,11 +30,8 @@ module.exports = function (module) {
throw new Error(`[[error:invalid-score, ${scores[i]}]]`);
}
}
const args = [key];
for (let i = 0; i < scores.length; i += 1) {
args.push(scores[i], String(values[i]));
}
await module.client.zadd(args);
const members = scores.map((score, i) => ({ score, value: String(values[i])}));
await module.client.zAdd(key, members);
}
module.sortedSetsAdd = async function (keys, scores, value) {
@@ -51,13 +48,16 @@ module.exports = function (module) {
throw new Error('[[error:invalid-data]]');
}
const batch = module.client.batch();
const batch = module.client.multi();
for (let i = 0; i < keys.length; i += 1) {
if (keys[i]) {
batch.zadd(keys[i], isArrayOfScores ? scores[i] : scores, String(value));
batch.zAdd(keys[i], {
score: isArrayOfScores ? scores[i] : scores,
value: String(value),
});
}
}
await helpers.execBatch(batch);
await batch.execAsPipeline();
};
module.sortedSetAddBulk = async function (data) {
@@ -69,8 +69,8 @@ module.exports = function (module) {
if (!utils.isNumber(item[1])) {
throw new Error(`[[error:invalid-score, ${item[1]}]]`);
}
batch.zadd(item[0], item[1], item[2]);
batch.zAdd(item[0], { score: item[1], value: String(item[2]) });
});
await helpers.execBatch(batch);
await batch.execAsPipeline();
};
};

30
src/database/redis/sorted/intersect.js
View File

@@ -8,52 +8,46 @@ module.exports = function (module) {
return 0;
}
const tempSetName = `temp_${Date.now()}`;
const interParams = [tempSetName, keys.length].concat(keys);
const multi = module.client.multi();
multi.zinterstore(interParams);
multi.zcard(tempSetName);
multi.zInterStore(tempSetName, keys);
multi.zCard(tempSetName);
multi.del(tempSetName);
const results = await helpers.execBatch(multi);
return results[1] || 0;
};
module.getSortedSetIntersect = async function (params) {
params.method = 'zrange';
params.reverse = false;
return await getSortedSetRevIntersect(params);
};
module.getSortedSetRevIntersect = async function (params) {
params.method = 'zrevrange';
params.reverse = true;
return await getSortedSetRevIntersect(params);
};
async function getSortedSetRevIntersect(params) {
const { sets } = params;
let { sets } = params;
const start = params.hasOwnProperty('start') ? params.start : 0;
const stop = params.hasOwnProperty('stop') ? params.stop : -1;
const weights = params.weights || [];
const tempSetName = `temp_${Date.now()}`;
let interParams = [tempSetName, sets.length].concat(sets);
const interParams = {};
if (weights.length) {
interParams = interParams.concat(['WEIGHTS'].concat(weights));
sets = sets.map((set, index) => ({ key: set, weight: weights[index] }));
}
if (params.aggregate) {
interParams = interParams.concat(['AGGREGATE', params.aggregate]);
interParams['AGGREGATE'] = params.aggregate.toUpperCase();
}
const rangeParams = [tempSetName, start, stop];
if (params.withScores) {
rangeParams.push('WITHSCORES');
}
const rangeCmd = params.withScores ? 'zRangeWithScores' : 'zRange';
const multi = module.client.multi();
multi.zinterstore(interParams);
multi[params.method](rangeParams);
multi.zInterStore(tempSetName, sets, interParams);
multi[rangeCmd](tempSetName, start, stop, { REV: params.reverse});
multi.del(tempSetName);
let results = await helpers.execBatch(multi);
@@ -61,6 +55,6 @@ module.exports = function (module) {
return results ? results[1] : null;
}
results = results[1] || [];
return helpers.zsetToObjectArray(results);
return results;
}
};

8
src/database/redis/sorted/remove.js
View File

@@ -18,10 +18,10 @@ module.exports = function (module) {
if (Array.isArray(key)) {
const batch = module.client.batch();
key.forEach(k => batch.zrem(k, value));
key.forEach(k => batch.zRem(k, value.map(String)));
await helpers.execBatch(batch);
} else {
await module.client.zrem(key, value);
await module.client.zRem(key, value.map(String));
}
};
@@ -31,7 +31,7 @@ module.exports = function (module) {
module.sortedSetsRemoveRangeByScore = async function (keys, min, max) {
const batch = module.client.batch();
keys.forEach(k => batch.zremrangebyscore(k, min, max));
keys.forEach(k => batch.zRemRangeByScore(k, min, max));
await helpers.execBatch(batch);
};
@@ -40,7 +40,7 @@ module.exports = function (module) {
return;
}
const batch = module.client.batch();
data.forEach(item => batch.zrem(item[0], item[1]));
data.forEach(item => batch.zRem(item[0], String(item[1])));
await helpers.execBatch(batch);
};
};

26
src/database/redis/sorted/union.js
View File

@@ -4,25 +4,20 @@
module.exports = function (module) {
const helpers = require('../helpers');
module.sortedSetUnionCard = async function (keys) {
const tempSetName = `temp_${Date.now()}`;
if (!keys.length) {
return 0;
}
const multi = module.client.multi();
multi.zunionstore([tempSetName, keys.length].concat(keys));
multi.zcard(tempSetName);
multi.del(tempSetName);
const results = await helpers.execBatch(multi);
return Array.isArray(results) && results.length ? results[1] : 0;
const results = await module.client.zUnion(keys);
return results ? results.length : 0;
};
module.getSortedSetUnion = async function (params) {
params.method = 'zrange';
params.reverse = false;
return await module.sortedSetUnion(params);
};
module.getSortedSetRevUnion = async function (params) {
params.method = 'zrevrange';
params.reverse = true;
return await module.sortedSetUnion(params);
};
@@ -32,21 +27,16 @@ module.exports = function (module) {
}
const tempSetName = `temp_${Date.now()}`;
const rangeParams = [tempSetName, params.start, params.stop];
if (params.withScores) {
rangeParams.push('WITHSCORES');
}
const rangeCmd = params.withScores ? 'zRangeWithScores' : 'zRange';
const multi = module.client.multi();
multi.zunionstore([tempSetName, params.sets.length].concat(params.sets));
multi[params.method](rangeParams);
multi.zUnionStore(tempSetName, params.sets);
multi[rangeCmd](tempSetName, params.start, params.stop, { REV: params.reverse });
multi.del(tempSetName);
let results = await helpers.execBatch(multi);
if (!params.withScores) {
return results ? results[1] : null;
}
results = results[1] || [];
return helpers.zsetToObjectArray(results);
return results;
};
};

2
src/topics/posts.js
View File

@@ -442,7 +442,7 @@ module.exports = function (Topics) {
let { content } = postData;
// ignore lines that start with `>`
content = content.split('\n').filter(line => !line.trim().startsWith('>')).join('\n');
content = (content || '').split('\n').filter(line => !line.trim().startsWith('>')).join('\n');
// Scan post content for topic links
const matches = [...content.matchAll(backlinkRegex)];
if (!matches) {

3
test/activitypub/actors.js
View File

@@ -449,7 +449,8 @@ describe('Inbox resolution', () => {
await activitypub.actors.assert(id);
const inboxes = await activitypub.resolveInboxes([id]);
console.log('inboxes', inboxes);
console.log('actor', actor);
assert(inboxes && Array.isArray(inboxes));
assert.strictEqual(inboxes.length, 1);
assert.strictEqual(inboxes[0], actor.inbox);

4
test/activitypub/notes.js
View File

@@ -650,7 +650,7 @@ describe('Notes', () => {
it('should upvote an asserted remote post', async () => {
const { id } = helpers.mocks.note();
await activitypub.notes.assert(0, [id], { skipChecks: true });
await activitypub.notes.assert(0, id, { skipChecks: true });
const { activity: like } = helpers.mocks.like({
object: id,
});
@@ -672,7 +672,7 @@ describe('Notes', () => {
it('should update a note\'s content', async () => {
const { id: actor } = helpers.mocks.person();
const { id, note } = helpers.mocks.note({ attributedTo: actor });
await activitypub.notes.assert(0, [id], { skipChecks: true });
await activitypub.notes.assert(0, id, { skipChecks: true });
note.content = utils.generateUUID();
const { activity: update } = helpers.mocks.update({ object: note });
const { activity } = helpers.mocks.announce({ object: update });

56
test/database/sorted.js
View File

@@ -501,7 +501,9 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea
['byScoreWithScoresKeys1', 1, 'value1'],
['byScoreWithScoresKeys2', 2, 'value2'],
]);
const data = await db.getSortedSetRevRangeByScoreWithScores(['byScoreWithScoresKeys1', 'byScoreWithScoresKeys2'], 0, -1, 5, -5);
const data = await db.getSortedSetRevRangeByScoreWithScores([
'byScoreWithScoresKeys1', 'byScoreWithScoresKeys2',
], 0, -1, 5, -5);
assert.deepStrictEqual(data, [{ value: 'value2', score: 2 }, { value: 'value1', score: 1 }]);
});
});
@@ -1144,23 +1146,17 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea
assert.strictEqual(await db.exists('sorted3'), false);
});
it('should remove multiple values from multiple keys', (done) => {
db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four'], (err) => {
assert.ifError(err);
db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six'], (err) => {
assert.ifError(err);
db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist'], (err) => {
assert.ifError(err);
db.getSortedSetsMembers(['multiTest1', 'multiTest2'], (err, members) => {
assert.ifError(err);
it('should remove multiple values from multiple keys', async () => {
await db.sortedSetAdd('multiTest1', [1, 2, 3, 4], ['one', 'two', 'three', 'four']);
await db.sortedSetAdd('multiTest2', [3, 4, 5, 6], ['three', 'four', 'five', 'six']);
await db.sortedSetRemove(['multiTest1', 'multiTest2'], ['two', 'three', 'four', 'five', 'doesnt exist']);
const members = await db.getSortedSetsMembers(['multiTest1', 'multiTest2']);
assert.equal(members[0].length, 1);
assert.equal(members[1].length, 1);
assert.deepEqual(members, [['one'], ['six']]);
done();
});
});
});
});
});
it('should remove value from multiple keys', async () => {
@@ -1171,24 +1167,15 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea
assert.deepStrictEqual(await db.getSortedSetRange('multiTest4', 0, -1), ['four', 'five', 'six']);
});
it('should remove multiple values from multiple keys', (done) => {
db.sortedSetAdd('multiTest5', [1], ['one'], (err) => {
assert.ifError(err);
db.sortedSetAdd('multiTest6', [2], ['two'], (err) => {
assert.ifError(err);
db.sortedSetAdd('multiTest7', [3], [333], (err) => {
assert.ifError(err);
db.sortedSetRemove(['multiTest5', 'multiTest6', 'multiTest7'], ['one', 'two', 333], (err) => {
assert.ifError(err);
db.getSortedSetsMembers(['multiTest5', 'multiTest6', 'multiTest7'], (err, members) => {
assert.ifError(err);
it('should remove multiple values from multiple keys', async () => {
await db.sortedSetAdd('multiTest5', [1], ['one']);
await db.sortedSetAdd('multiTest6', [2], ['two']);
await db.sortedSetAdd('multiTest7', [3], [333]);
await db.sortedSetRemove(['multiTest5', 'multiTest6', 'multiTest7'], ['one', 'two', 333]);
const members = await db.getSortedSetsMembers(['multiTest5', 'multiTest6', 'multiTest7']);
assert.deepEqual(members, [[], [], []]);
done();
});
});
});
});
});
});
it('should not remove anything if values is empty array', (done) => {
@@ -1379,7 +1366,10 @@ NUMERIC)-- WsPn&query[cid]=-1&parentCid=0&selectedCids[]=-1&privilege=topics:rea
weights: [1, 0.5],
}, (err, data) => {
assert.ifError(err);
assert.deepEqual([{ value: 'value2', score: 4 }, { value: 'value3', score: 5.5 }], data);
assert.deepEqual([
{ value: 'value2', score: 4 },
{ value: 'value3', score: 5.5 },
], data);
done();
});
});

1
test/mocks/databasemock.js
View File

@@ -171,7 +171,6 @@ before(async function () {
require('../../src/user').startJobs();
await webserver.listen();
// Iterate over all of the test suites/contexts
this.test.parent.suites.forEach((suite) => {
// Attach an afterAll listener that resets the defaults