mirror of
https://github.com/NodeBB/NodeBB.git
synced 2025-11-05 05:25:49 +01:00
PostgreSQL database driver (#5861)
* [test/database/list] Fix test list 4 being used in two different tests * [database/postgres] PostgreSQL database driver * [database/postgres] Make transactions work based on continuation scope. * [database/postgres] Implement nested transactions * eslint --fix * Add database changes from earlier this week to the PostgreSQL driver. * Fix typo * Fix postgres.incrObjectFieldBy returning undefined instead of null when given NaN * [database/postgres] Fix sortedSetsCard returning an array of strings. * Update socket.io postgres adapter * Fix PostgreSQL erroring when multiple updates are made to the same sorted set entry in a single operation. Add a test case to catch this error. * Fix lint errors. * Only prune sessions on one instance in a cluster to avoid deadlocks. They're caught and handled by the database server, but they spam the logs. * Fix arguments.slice.
This commit is contained in:
Ben Lubar
Julian Lam
108
src/database/postgres/sorted/add.js
Normal file
108
src/database/postgres/sorted/add.js
Normal file
@@ -0,0 +1,108 @@
|
||||
'use strict';
|
||||
|
||||
var async = require('async');
|
||||
|
||||
module.exports = function (db, module) {
|
||||
var helpers = module.helpers.postgres;
|
||||
|
||||
module.sortedSetAdd = function (key, score, value, callback) {
|
||||
callback = callback || helpers.noop;
|
||||
|
||||
if (!key) {
|
||||
return callback();
|
||||
}
|
||||
|
||||
if (Array.isArray(score) && Array.isArray(value)) {
|
||||
return sortedSetAddBulk(key, score, value, callback);
|
||||
}
|
||||
|
||||
value = helpers.valueToString(value);
|
||||
score = parseFloat(score);
|
||||
|
||||
module.transaction(function (tx, done) {
|
||||
var query = tx.client.query.bind(tx.client);
|
||||
|
||||
async.series([
|
||||
async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'),
|
||||
async.apply(query, {
|
||||
name: 'sortedSetAdd',
|
||||
text: `
|
||||
INSERT INTO "legacy_zset" ("_key", "value", "score")
|
||||
VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC)
|
||||
ON CONFLICT ("_key", "value")
|
||||
DO UPDATE SET "score" = $3::NUMERIC`,
|
||||
values: [key, value, score],
|
||||
}),
|
||||
], function (err) {
|
||||
done(err);
|
||||
});
|
||||
}, callback);
|
||||
};
|
||||
|
||||
function sortedSetAddBulk(key, scores, values, callback) {
|
||||
if (!scores.length || !values.length) {
|
||||
return callback();
|
||||
}
|
||||
if (scores.length !== values.length) {
|
||||
return callback(new Error('[[error:invalid-data]]'));
|
||||
}
|
||||
|
||||
values = values.map(helpers.valueToString);
|
||||
scores = scores.map(function (score) {
|
||||
return parseFloat(score);
|
||||
});
|
||||
|
||||
helpers.removeDuplicateValues(values, scores);
|
||||
|
||||
module.transaction(function (tx, done) {
|
||||
var query = tx.client.query.bind(tx.client);
|
||||
|
||||
async.series([
|
||||
async.apply(helpers.ensureLegacyObjectType, tx.client, key, 'zset'),
|
||||
async.apply(query, {
|
||||
name: 'sortedSetAddBulk',
|
||||
text: `
|
||||
INSERT INTO "legacy_zset" ("_key", "value", "score")
|
||||
SELECT $1::TEXT, v, s
|
||||
FROM UNNEST($2::TEXT[], $3::NUMERIC[]) vs(v, s)
|
||||
ON CONFLICT ("_key", "value")
|
||||
DO UPDATE SET "score" = EXCLUDED."score"`,
|
||||
values: [key, values, scores],
|
||||
}),
|
||||
], function (err) {
|
||||
done(err);
|
||||
});
|
||||
}, callback);
|
||||
}
|
||||
|
||||
module.sortedSetsAdd = function (keys, score, value, callback) {
|
||||
callback = callback || helpers.noop;
|
||||
|
||||
if (!Array.isArray(keys) || !keys.length) {
|
||||
return callback();
|
||||
}
|
||||
|
||||
value = helpers.valueToString(value);
|
||||
score = parseFloat(score);
|
||||
|
||||
module.transaction(function (tx, done) {
|
||||
var query = tx.client.query.bind(tx.client);
|
||||
|
||||
async.series([
|
||||
async.apply(helpers.ensureLegacyObjectsType, tx.client, keys, 'zset'),
|
||||
async.apply(query, {
|
||||
name: 'sortedSetsAdd',
|
||||
text: `
|
||||
INSERT INTO "legacy_zset" ("_key", "value", "score")
|
||||
SELECT k, $2::TEXT, $3::NUMERIC
|
||||
FROM UNNEST($1::TEXT[]) k
|
||||
ON CONFLICT ("_key", "value")
|
||||
DO UPDATE SET "score" = $3::NUMERIC`,
|
||||
values: [keys, value, score],
|
||||
}),
|
||||
], function (err) {
|
||||
done(err);
|
||||
});
|
||||
}, callback);
|
||||
};
|
||||
};
|
||||
105
src/database/postgres/sorted/intersect.js
Normal file
105
src/database/postgres/sorted/intersect.js
Normal file
@@ -0,0 +1,105 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = function (db, module) {
|
||||
module.sortedSetIntersectCard = function (keys, callback) {
|
||||
if (!Array.isArray(keys) || !keys.length) {
|
||||
return callback(null, 0);
|
||||
}
|
||||
|
||||
db.query({
|
||||
name: 'sortedSetIntersectCard',
|
||||
text: `
|
||||
WITH A AS (SELECT z."value" v,
|
||||
COUNT(*) c
|
||||
FROM "legacy_object_live" o
|
||||
INNER JOIN "legacy_zset" z
|
||||
ON o."_key" = z."_key"
|
||||
AND o."type" = z."type"
|
||||
WHERE o."_key" = ANY($1::TEXT[])
|
||||
GROUP BY z."value")
|
||||
SELECT COUNT(*) c
|
||||
FROM A
|
||||
WHERE A.c = array_length($1::TEXT[], 1)`,
|
||||
values: [keys],
|
||||
}, function (err, res) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
callback(null, parseInt(res.rows[0].c, 10));
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
module.getSortedSetIntersect = function (params, callback) {
|
||||
params.sort = 1;
|
||||
getSortedSetIntersect(params, callback);
|
||||
};
|
||||
|
||||
module.getSortedSetRevIntersect = function (params, callback) {
|
||||
params.sort = -1;
|
||||
getSortedSetIntersect(params, callback);
|
||||
};
|
||||
|
||||
function getSortedSetIntersect(params, callback) {
|
||||
var sets = params.sets;
|
||||
var start = params.hasOwnProperty('start') ? params.start : 0;
|
||||
var stop = params.hasOwnProperty('stop') ? params.stop : -1;
|
||||
var weights = params.weights || [];
|
||||
var aggregate = params.aggregate || 'SUM';
|
||||
|
||||
if (sets.length < weights.length) {
|
||||
weights = weights.slice(0, sets.length);
|
||||
}
|
||||
while (sets.length > weights.length) {
|
||||
weights.push(1);
|
||||
}
|
||||
|
||||
var limit = stop - start + 1;
|
||||
if (limit <= 0) {
|
||||
limit = null;
|
||||
}
|
||||
|
||||
db.query({
|
||||
name: 'getSortedSetIntersect' + aggregate + (params.sort > 0 ? 'Asc' : 'Desc') + 'WithScores',
|
||||
text: `
|
||||
WITH A AS (SELECT z."value",
|
||||
` + aggregate + `(z."score" * k."weight") "score",
|
||||
COUNT(*) c
|
||||
FROM UNNEST($1::TEXT[], $2::NUMERIC[]) k("_key", "weight")
|
||||
INNER JOIN "legacy_object_live" o
|
||||
ON o."_key" = k."_key"
|
||||
INNER JOIN "legacy_zset" z
|
||||
ON o."_key" = z."_key"
|
||||
AND o."type" = z."type"
|
||||
GROUP BY z."value")
|
||||
SELECT A."value",
|
||||
A."score"
|
||||
FROM A
|
||||
WHERE c = array_length($1::TEXT[], 1)
|
||||
ORDER BY A."score" ` + (params.sort > 0 ? 'ASC' : 'DESC') + `
|
||||
LIMIT $4::INTEGER
|
||||
OFFSET $3::INTEGER`,
|
||||
values: [sets, weights, start, limit],
|
||||
}, function (err, res) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (params.withScores) {
|
||||
res.rows = res.rows.map(function (r) {
|
||||
return {
|
||||
value: r.value,
|
||||
score: parseFloat(r.score),
|
||||
};
|
||||
});
|
||||
} else {
|
||||
res.rows = res.rows.map(function (r) {
|
||||
return r.value;
|
||||
});
|
||||
}
|
||||
|
||||
callback(null, res.rows);
|
||||
});
|
||||
}
|
||||
};
|
||||
83
src/database/postgres/sorted/remove.js
Normal file
83
src/database/postgres/sorted/remove.js
Normal file
@@ -0,0 +1,83 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = function (db, module) {
|
||||
var helpers = module.helpers.postgres;
|
||||
|
||||
module.sortedSetRemove = function (key, value, callback) {
|
||||
function done(err) {
|
||||
if (callback) {
|
||||
callback(err);
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
return done();
|
||||
}
|
||||
|
||||
if (!Array.isArray(key)) {
|
||||
key = [key];
|
||||
}
|
||||
|
||||
if (!Array.isArray(value)) {
|
||||
value = [value];
|
||||
}
|
||||
value = value.map(helpers.valueToString);
|
||||
|
||||
db.query({
|
||||
name: 'sortedSetRemove',
|
||||
text: `
|
||||
DELETE FROM "legacy_zset"
|
||||
WHERE "_key" = ANY($1::TEXT[])
|
||||
AND "value" = ANY($2::TEXT[])`,
|
||||
values: [key, value],
|
||||
}, done);
|
||||
};
|
||||
|
||||
module.sortedSetsRemove = function (keys, value, callback) {
|
||||
callback = callback || helpers.noop;
|
||||
|
||||
if (!Array.isArray(keys) || !keys.length) {
|
||||
return callback();
|
||||
}
|
||||
|
||||
value = helpers.valueToString(value);
|
||||
|
||||
db.query({
|
||||
name: 'sortedSetsRemove',
|
||||
text: `
|
||||
DELETE FROM "legacy_zset"
|
||||
WHERE "_key" = ANY($1::TEXT[])
|
||||
AND "value" = $2::TEXT`,
|
||||
values: [keys, value],
|
||||
}, function (err) {
|
||||
callback(err);
|
||||
});
|
||||
};
|
||||
|
||||
module.sortedSetsRemoveRangeByScore = function (keys, min, max, callback) {
|
||||
callback = callback || helpers.noop;
|
||||
|
||||
if (!Array.isArray(keys) || !keys.length) {
|
||||
return callback();
|
||||
}
|
||||
|
||||
if (min === '-inf') {
|
||||
min = null;
|
||||
}
|
||||
if (max === '+inf') {
|
||||
max = null;
|
||||
}
|
||||
|
||||
db.query({
|
||||
name: 'sortedSetsRemoveRangeByScore',
|
||||
text: `
|
||||
DELETE FROM "legacy_zset"
|
||||
WHERE "_key" = ANY($1::TEXT[])
|
||||
AND ("score" >= $2::NUMERIC OR $2::NUMERIC IS NULL)
|
||||
AND ("score" <= $3::NUMERIC OR $3::NUMERIC IS NULL)`,
|
||||
values: [keys, min, max],
|
||||
}, function (err) {
|
||||
callback(err);
|
||||
});
|
||||
};
|
||||
};
|
||||
97
src/database/postgres/sorted/union.js
Normal file
97
src/database/postgres/sorted/union.js
Normal file
@@ -0,0 +1,97 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = function (db, module) {
|
||||
module.sortedSetUnionCard = function (keys, callback) {
|
||||
if (!Array.isArray(keys) || !keys.length) {
|
||||
return callback(null, 0);
|
||||
}
|
||||
|
||||
db.query({
|
||||
name: 'sortedSetUnionCard',
|
||||
text: `
|
||||
SELECT COUNT(DISTINCT z."value") c
|
||||
FROM "legacy_object_live" o
|
||||
INNER JOIN "legacy_zset" z
|
||||
ON o."_key" = z."_key"
|
||||
AND o."type" = z."type"
|
||||
WHERE o."_key" = ANY($1::TEXT[])`,
|
||||
values: [keys],
|
||||
}, function (err, res) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
callback(null, parseInt(res.rows[0].c, 10));
|
||||
});
|
||||
};
|
||||
|
||||
module.getSortedSetUnion = function (params, callback) {
|
||||
params.sort = 1;
|
||||
getSortedSetUnion(params, callback);
|
||||
};
|
||||
|
||||
module.getSortedSetRevUnion = function (params, callback) {
|
||||
params.sort = -1;
|
||||
getSortedSetUnion(params, callback);
|
||||
};
|
||||
|
||||
function getSortedSetUnion(params, callback) {
|
||||
var sets = params.sets;
|
||||
var start = params.hasOwnProperty('start') ? params.start : 0;
|
||||
var stop = params.hasOwnProperty('stop') ? params.stop : -1;
|
||||
var weights = params.weights || [];
|
||||
var aggregate = params.aggregate || 'SUM';
|
||||
|
||||
if (sets.length < weights.length) {
|
||||
weights = weights.slice(0, sets.length);
|
||||
}
|
||||
while (sets.length > weights.length) {
|
||||
weights.push(1);
|
||||
}
|
||||
|
||||
var limit = stop - start + 1;
|
||||
if (limit <= 0) {
|
||||
limit = null;
|
||||
}
|
||||
|
||||
db.query({
|
||||
name: 'getSortedSetUnion' + aggregate + (params.sort > 0 ? 'Asc' : 'Desc') + 'WithScores',
|
||||
text: `
|
||||
WITH A AS (SELECT z."value",
|
||||
` + aggregate + `(z."score" * k."weight") "score"
|
||||
FROM UNNEST($1::TEXT[], $2::NUMERIC[]) k("_key", "weight")
|
||||
INNER JOIN "legacy_object_live" o
|
||||
ON o."_key" = k."_key"
|
||||
INNER JOIN "legacy_zset" z
|
||||
ON o."_key" = z."_key"
|
||||
AND o."type" = z."type"
|
||||
GROUP BY z."value")
|
||||
SELECT A."value",
|
||||
A."score"
|
||||
FROM A
|
||||
ORDER BY A."score" ` + (params.sort > 0 ? 'ASC' : 'DESC') + `
|
||||
LIMIT $4::INTEGER
|
||||
OFFSET $3::INTEGER`,
|
||||
values: [sets, weights, start, limit],
|
||||
}, function (err, res) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (params.withScores) {
|
||||
res.rows = res.rows.map(function (r) {
|
||||
return {
|
||||
value: r.value,
|
||||
score: parseFloat(r.score),
|
||||
};
|
||||
});
|
||||
} else {
|
||||
res.rows = res.rows.map(function (r) {
|
||||
return r.value;
|
||||
});
|
||||
}
|
||||
|
||||
callback(null, res.rows);
|
||||
});
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user