From 7d476a668edb0580f5360136ddd9f6eebec2d30b Mon Sep 17 00:00:00 2001 From: zneix <44851575+zneix@users.noreply.github.com> Date: Thu, 3 Sep 2020 19:24:34 +0200 Subject: [PATCH] Rewritten postgresql document store This is experimental as there are couple issues with attemting to set duplicate keys and connecting on startup, but it's working regardless of those issues.\nMaybe I can look into solving those later if needed --- docs/storage.md | 26 ++++++- lib/document_stores/mongodb.js | 12 +-- lib/document_stores/postgres.js | 131 ++++++++++++++------------------ package-lock.json | 119 +---------------------------- package.json | 1 - 5 files changed, 93 insertions(+), 196 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index d934d3c..a065073 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -87,8 +87,32 @@ Check [documentation](https://mongodb.github.io/node-mongodb-native/3.5/api/Mong ## Postgres -Not rewritten yet, to be filled in +Requires npm package (Tested on v8.3.3): +```bash +npm install pg +``` + +You will have to create the database and add a table named `entries`. It can be easily done with the following query: + +`CREATE TABLE entries (id SERIAL PRIMARY KEY, key VARCHAR(255) NOT NULL, value TEXT NOT NULL, expiration INT, UNIQUE(key));` + +Properties in `clientOptions` are optimal defaults and should be sufficient to run haste, however more detailed explanation for them can be found in pg package [documentation](https://node-postgres.com/api/client). +Expiration property in config can be changed to a value in seconds after which entries will not be served. + +```json +{ + "type": "postgres", + "expire": 0, + "clientOptions": { + "host": "localhost", + "port": 5432, + "user": "username", + "password": "password", + "database": "haste" + } +} +``` ## Redis diff --git a/lib/document_stores/mongodb.js b/lib/document_stores/mongodb.js index 9c64f68..6cbdc42 100644 --- a/lib/document_stores/mongodb.js +++ b/lib/document_stores/mongodb.js @@ -12,7 +12,7 @@ class MongoDocumentStore { const now = Math.floor(Date.now() / 1000); const that = this; - if ((await this.safeConnect()).error) return null; + if ((await this.safeConnect()).error) return false; return await this.MongoClient.db().collection('entries').updateOne( { @@ -70,13 +70,13 @@ class MongoDocumentStore { } return document ? document.value : null; } - safeConnect(){ + async safeConnect(){ //don't try connecting again if already connected //https://jira.mongodb.org/browse/NODE-1868 if (this.MongoClient.isConnected()) return { error: null }; - return this.MongoClient.connect() - .then(client => { - winston.debug('connected to mongodb', { success: true }); + return await this.MongoClient.connect() + .then(() => { + winston.info('connected to mongodb'); return { error: null }; }) .catch(err => { @@ -89,4 +89,4 @@ class MongoDocumentStore { -module.exports = MongoDocumentStore; \ No newline at end of file +module.exports = MongoDocumentStore; diff --git a/lib/document_stores/postgres.js b/lib/document_stores/postgres.js index 247b507..bd29115 100644 --- a/lib/document_stores/postgres.js +++ b/lib/document_stores/postgres.js @@ -1,80 +1,65 @@ -// /*global require,module,process*/ - -const postgres = require('pg'); const winston = require('winston'); +const postgres = require('pg'); -// create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key)); - -// A postgres document store -const PostgresDocumentStore = function (options){ - this.expireJS = options.expire; - this.connectionUrl = process.env.DATABASE_URL || options.connectionUrl; -}; - -PostgresDocumentStore.prototype = { - - // Set a given key - set: function (key, data, callback, skipExpire){ - const now = Math.floor(Date.now() / 1000); - const that = this; - this.safeConnect(function (err, client, done){ - if (err){ return callback(false); } - client.query('INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)', [ - key, - data, - that.expireJS && !skipExpire ? that.expireJS + now : null - ], function (err){ - if (err){ - winston.error('error persisting value to postgres', { error: err }); - return callback(false); - } - callback(true); - done(); - }); - }); - }, - - // Get a given key's data - get: function (key, callback, skipExpire){ - const now = Math.floor(Date.now() / 1000); - const that = this; - this.safeConnect(function (err, client, done){ - if (err){ return callback(false); } - client.query('SELECT id,value,expiration from entries where KEY = $1 and (expiration IS NULL or expiration > $2)', [key, now], - function (err, result){ - if (err){ - winston.error('error retrieving value from postgres', { error: err }); - return callback(false); - } - callback(result.rows.length ? result.rows[0].value : false); - if (result.rows.length && that.expireJS && !skipExpire){ - client.query('UPDATE entries SET expiration = $1 WHERE ID = $2', [ - that.expireJS + now, - result.rows[0].id - ], function (err){ - if (!err){ - done(); - } - }); - } else { - done(); - } - }); - }); - }, - - // A connection wrapper - safeConnect: function (callback){ - postgres.connect(this.connectionUrl, function (err, client, done){ - if (err){ - winston.error('error connecting to postgres', { error: err }); - callback(err); - } else { - callback(undefined, client, done); - } +class PostgresDocumentStore { + constructor(options){ + this.expire = options.expire; + this.PostgresClient = new postgres.Client(options.clientOptions); + this.safeConnect(); + this.PostgresClient.on('end', () => { + winston.debug('disconnected from pg!'); }); } -}; + async set(key, data, skipExpire){ + const now = Math.floor(Date.now() / 1000); + + return await this.PostgresClient.query( + 'INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)', + [ key, data, (this.expire && !skipExpire) ? this.expire + now : null ] + ) + .then(() => { + return true; + }) + .catch(err => { + winston.error('failed to set postgres document', { key: key, error: err }); + return false; + }); + + } + + async get(key, skipExpire){ + const now = Math.floor(Date.now() / 1000); + + return await this.PostgresClient.query( + 'SELECT id,value,expiration FROM entries WHERE key = $1 AND (expiration IS NULL OR expiration > $2)', + [ key, now ]) + .then(async res => { + if (res.rows.length && this.expire && !skipExpire){ + await this.PostgresClient.query( + 'UPDATE entries SET expiration = $1 WHERE ID = $2', + [ this.expire + now, res.rows[0].id ] + ); + } + return res.rows.length ? res.rows[0].value : null; + }) + .catch(err => { + winston.error('error retrieving value from postgres', { error: err }); + return null; + }); + } + + async safeConnect(){ + return await this.PostgresClient.connect() + .then(() => { + winston.info('connected to postgres!'); + return { error: null }; + }) + .catch(err => { + winston.error('failed connecting to postgres!', {error: err}); + return { error: err }; + }); + } +} module.exports = PostgresDocumentStore; diff --git a/package-lock.json b/package-lock.json index 3fe5eaa..b835ac6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -780,11 +780,6 @@ "ieee754": "^1.1.4" } }, - "buffer-writer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", - "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" - }, "busboy": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz", @@ -1237,12 +1232,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "semver": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", - "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", - "dev": true - }, "strip-ansi": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", @@ -2150,11 +2139,6 @@ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, - "packet-reader": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", - "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" - }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -2197,90 +2181,12 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "pg": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.3.2.tgz", - "integrity": "sha512-hOoRCTriXS+VWwyXHchRjWb9yv3Koq8irlwwXniqhdgK0AbfWvEnybGS2HIUE+UdCSTuYAM4WGPujFpPg9Vcaw==", - "requires": { - "buffer-writer": "2.0.0", - "packet-reader": "1.0.0", - "pg-connection-string": "^2.3.0", - "pg-pool": "^3.2.1", - "pg-protocol": "^1.2.5", - "pg-types": "^2.1.0", - "pgpass": "1.x", - "semver": "4.3.2" - } - }, - "pg-connection-string": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.3.0.tgz", - "integrity": "sha512-ukMTJXLI7/hZIwTW7hGMZJ0Lj0S2XQBCJ4Shv4y1zgQ/vqVea+FLhzywvPj0ujSuofu+yA4MYHGZPTsgjBgJ+w==" - }, - "pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" - }, - "pg-pool": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.2.1.tgz", - "integrity": "sha512-BQDPWUeKenVrMMDN9opfns/kZo4lxmSWhIqo+cSAF7+lfi9ZclQbr9vfnlNaPr8wYF3UYjm5X0yPAhbcgqNOdA==" - }, - "pg-protocol": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.2.5.tgz", - "integrity": "sha512-1uYCckkuTfzz/FCefvavRywkowa6M5FohNMF5OjKrqo9PSR8gYc8poVmwwYQaBxhmQdBjhtP514eXy9/Us2xKg==" - }, - "pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "requires": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - } - }, - "pgpass": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", - "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", - "requires": { - "split": "^1.0.0" - } - }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", "dev": true }, - "postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" - }, - "postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" - }, - "postgres-date": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.6.tgz", - "integrity": "sha512-o2a4gxeFcox+CgB3Ig/kNHBP23PiEXHCXx7pcIIsvzoNz4qv+lKTyiSkjOXIMNUl12MO/mOYl2K6wR9X5K6Plg==" - }, - "postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "requires": { - "xtend": "^4.0.0" - } - }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2432,9 +2338,10 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "semver": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", - "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true }, "send": { "version": "0.17.1", @@ -2559,14 +2466,6 @@ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" }, - "split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", - "requires": { - "through": "2" - } - }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -2719,11 +2618,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -2986,11 +2880,6 @@ "mkdirp": "^0.5.1" } }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", diff --git a/package.json b/package.json index 6cc66f7..d57d2f9 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "busboy": "^0.3.1", "express": "^4.17.1", "express-rate-limit": "^5.1.3", - "pg": "^8.3.2", "st": "^2.0.0", "winston": "^3.3.3" },