SunRed
/
haste-server
mirror of https://github.com/SunRed/haste-server.git
1
0
Fork 0
Code Issues Releases Activity

Rewritten postgresql document store 

This is experimental as there are couple issues with attemting to set duplicate keys and connecting on startup, but it's working regardless of those issues.\nMaybe I can look into solving those later if needed
This commit is contained in:
zneix 2020-09-03 19:24:34 +02:00
parent e3e45bee3c
commit 7d476a668e
No known key found for this signature in database
GPG Key ID: 911916E0523B22F6
5 changed files with 93 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/storage.md
View File

@ -87,8 +87,32 @@ Check [documentation](https://mongodb.github.io/node-mongodb-native/3.5/api/Mong
## Postgres
Not rewritten yet, to be filled in
Requires npm package (Tested on v8.3.3):
```bash
npm install pg
```
You will have to create the database and add a table named `entries`. It can be easily done with the following query:
`CREATE TABLE entries (id SERIAL PRIMARY KEY, key VARCHAR(255) NOT NULL, value TEXT NOT NULL, expiration INT, UNIQUE(key));`
Properties in `clientOptions` are optimal defaults and should be sufficient to run haste, however more detailed explanation for them can be found in pg package [documentation](https://node-postgres.com/api/client).
Expiration property in config can be changed to a value in seconds after which entries will not be served.
```json
{
"type": "postgres",
"expire": 0,
"clientOptions": {
"host": "localhost",
"port": 5432,
"user": "username",
"password": "password",
"database": "haste"
}
}
```
## Redis

12
lib/document_stores/mongodb.js
View File

@ -12,7 +12,7 @@ class MongoDocumentStore {
const now = Math.floor(Date.now() / 1000);
const that = this;
if ((await this.safeConnect()).error) return null;
if ((await this.safeConnect()).error) return false;
return await this.MongoClient.db().collection('entries').updateOne(
{
@ -70,13 +70,13 @@ class MongoDocumentStore {
}
return document ? document.value : null;
}
safeConnect(){
async safeConnect(){
//don't try connecting again if already connected
//https://jira.mongodb.org/browse/NODE-1868
if (this.MongoClient.isConnected()) return { error: null };
return this.MongoClient.connect()
.then(client => {
winston.debug('connected to mongodb', { success: true });
return await this.MongoClient.connect()
.then(() => {
winston.info('connected to mongodb');
return { error: null };
})
.catch(err => {
@ -89,4 +89,4 @@ class MongoDocumentStore {
module.exports = MongoDocumentStore;
module.exports = MongoDocumentStore;

131
lib/document_stores/postgres.js
View File

@ -1,80 +1,65 @@
// /*global require,module,process*/
const postgres = require('pg');
const winston = require('winston');
const postgres = require('pg');
// create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key));
// A postgres document store
const PostgresDocumentStore = function (options){
this.expireJS = options.expire;
this.connectionUrl = process.env.DATABASE_URL || options.connectionUrl;
};
PostgresDocumentStore.prototype = {
// Set a given key
set: function (key, data, callback, skipExpire){
const now = Math.floor(Date.now() / 1000);
const that = this;
this.safeConnect(function (err, client, done){
if (err){ return callback(false); }
client.query('INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)', [
key,
data,
that.expireJS && !skipExpire ? that.expireJS + now : null
], function (err){
if (err){
winston.error('error persisting value to postgres', { error: err });
return callback(false);
}
callback(true);
done();
});
});
},
// Get a given key's data
get: function (key, callback, skipExpire){
const now = Math.floor(Date.now() / 1000);
const that = this;
this.safeConnect(function (err, client, done){
if (err){ return callback(false); }
client.query('SELECT id,value,expiration from entries where KEY = $1 and (expiration IS NULL or expiration > $2)', [key, now],
function (err, result){
if (err){
winston.error('error retrieving value from postgres', { error: err });
return callback(false);
}
callback(result.rows.length ? result.rows[0].value : false);
if (result.rows.length && that.expireJS && !skipExpire){
client.query('UPDATE entries SET expiration = $1 WHERE ID = $2', [
that.expireJS + now,
result.rows[0].id
], function (err){
if (!err){
done();
}
});
} else {
done();
}
});
});
},
// A connection wrapper
safeConnect: function (callback){
postgres.connect(this.connectionUrl, function (err, client, done){
if (err){
winston.error('error connecting to postgres', { error: err });
callback(err);
} else {
callback(undefined, client, done);
}
class PostgresDocumentStore {
constructor(options){
this.expire = options.expire;
this.PostgresClient = new postgres.Client(options.clientOptions);
this.safeConnect();
this.PostgresClient.on('end', () => {
winston.debug('disconnected from pg!');
});
}
};
async set(key, data, skipExpire){
const now = Math.floor(Date.now() / 1000);
return await this.PostgresClient.query(
'INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)',
[ key, data, (this.expire && !skipExpire) ? this.expire + now : null ]
)
.then(() => {
return true;
})
.catch(err => {
winston.error('failed to set postgres document', { key: key, error: err });
return false;
});
}
async get(key, skipExpire){
const now = Math.floor(Date.now() / 1000);
return await this.PostgresClient.query(
'SELECT id,value,expiration FROM entries WHERE key = $1 AND (expiration IS NULL OR expiration > $2)',
[ key, now ])
.then(async res => {
if (res.rows.length && this.expire && !skipExpire){
await this.PostgresClient.query(
'UPDATE entries SET expiration = $1 WHERE ID = $2',
[ this.expire + now, res.rows[0].id ]
);
}
return res.rows.length ? res.rows[0].value : null;
})
.catch(err => {
winston.error('error retrieving value from postgres', { error: err });
return null;
});
}
async safeConnect(){
return await this.PostgresClient.connect()
.then(() => {
winston.info('connected to postgres!');
return { error: null };
})
.catch(err => {
winston.error('failed connecting to postgres!', {error: err});
return { error: err };
});
}
}
module.exports = PostgresDocumentStore;

119
package-lock.json generated
View File

@ -780,11 +780,6 @@
"ieee754": "^1.1.4"
}
},
"buffer-writer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
},
"busboy": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz",
@ -1237,12 +1232,6 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"dev": true
},
"semver": {
"version": "7.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
"integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
"dev": true
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
@ -2150,11 +2139,6 @@
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
"dev": true
},
"packet-reader": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
"integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
},
"parent-module": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@ -2197,90 +2181,12 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"pg": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/pg/-/pg-8.3.2.tgz",
"integrity": "sha512-hOoRCTriXS+VWwyXHchRjWb9yv3Koq8irlwwXniqhdgK0AbfWvEnybGS2HIUE+UdCSTuYAM4WGPujFpPg9Vcaw==",
"requires": {
"buffer-writer": "2.0.0",
"packet-reader": "1.0.0",
"pg-connection-string": "^2.3.0",
"pg-pool": "^3.2.1",
"pg-protocol": "^1.2.5",
"pg-types": "^2.1.0",
"pgpass": "1.x",
"semver": "4.3.2"
}
},
"pg-connection-string": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz",
"integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w=="
},
"pg-int8": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
},
"pg-pool": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz",
"integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA=="
},
"pg-protocol": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz",
"integrity": "sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg=="
},
"pg-types": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
"requires": {
"pg-int8": "1.0.1",
"postgres-array": "~2.0.0",
"postgres-bytea": "~1.0.0",
"postgres-date": "~1.0.4",
"postgres-interval": "^1.1.0"
}
},
"pgpass": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz",
"integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=",
"requires": {
"split": "^1.0.0"
}
},
"picomatch": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz",
"integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
"dev": true
},
"postgres-array": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="
},
"postgres-bytea": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU="
},
"postgres-date": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.6.tgz",
"integrity": "sha512-o2a4gxeFcox+CgB3Ig/kNHBP23PiEXHCXx7pcIIsvzoNz4qv+lKTyiSkjOXIMNUl12MO/mOYl2K6wR9X5K6Plg=="
},
"postgres-interval": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
"requires": {
"xtend": "^4.0.0"
}
},
"prelude-ls": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@ -2432,9 +2338,10 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"semver": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz",
"integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c="
"version": "7.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
"integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
"dev": true
},
"send": {
"version": "0.17.1",
@ -2559,14 +2466,6 @@
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
"integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w="
},
"split": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz",
"integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==",
"requires": {
"through": "2"
}
},
"sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@ -2719,11 +2618,6 @@
"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
"dev": true
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
@ -2986,11 +2880,6 @@
"mkdirp": "^0.5.1"
}
},
"xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",

1
package.json
View File

@ -19,7 +19,6 @@
"busboy": "^0.3.1",
"express": "^4.17.1",
"express-rate-limit": "^5.1.3",
"pg": "^8.3.2",
"st": "^2.0.0",
"winston": "^3.3.3"
},