1
0
Fork 0
mirror of https://github.com/SunRed/haste-server.git synced 2024-11-27 11:20:18 +01:00

Rewrote redis storage handler

also removed unnecessary newline in file storage handler
This commit is contained in:
zneix 2020-09-01 23:43:54 +02:00
parent a4cf027900
commit c165781b18
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG key ID: 911916E0523B22F6
6 changed files with 67 additions and 151 deletions

View file

@ -50,17 +50,17 @@ Not rewritten yet, to be filled in
Requires npm package (Tested on v3.6.0): Requires npm package (Tested on v3.6.0):
```bash ```bash
npm i mongodb@3.6.0 npm i mongodb
``` ```
Stores documents in a specified database in a collection named `entries`. Stores documents in a specified database in a collection named `entries`.
Expiration property in config can be changed to a value in seconds after which entries will not be served to users. Expiration property in config can be changed to a value in seconds after which entries will not be served.
Optimal default config: Optimal default config:
> **NOTE:** Depending on how your MongoDB server is configured, options as connectionUri may vary. > **NOTE:** Depending on how your MongoDB server is configured, options as connectionUri may vary.
If server has no authentication, you can omit the `auth` object. If server has no authentication, you can omit the `auth` object.
Check [documentation](http://mongodb.github.io/node-mongodb-native/3.5/api/MongoClient.html) for more detailed explanation about available `clientOptions` properties. Check [documentation](https://mongodb.github.io/node-mongodb-native/3.5/api/MongoClient.html) for more detailed explanation about available `clientOptions` properties.
```json ```json
{ {
@ -92,8 +92,28 @@ Not rewritten yet, to be filled in
## Redis ## Redis
Not rewritten yet, to be filled in Requires npm package (Tested on v4.17.3):
```bash
npm install ioredis
```
Stores documents in a specified redis database.
Expiration property in config can be changed to a value in seconds after which entries will not be served.
`redisOptions` object below contains default values, but you can adjust those to match your redis-server configuration. Check [documentation](https://github.com/luin/ioredis/blob/master/API.md#new-redisport-host-options) for more information about accepted values.
```json
{
"type": "redis",
"expire": 0,
"redisOptions": {
"host": "127.0.0.1",
"port": 6379,
"db": 1
}
}
```
## RethinkDB ## RethinkDB

View file

@ -2,7 +2,6 @@ const winston = require('winston');
const fs = require('fs'); const fs = require('fs');
const crypto = require('crypto'); const crypto = require('crypto');
class FileDocumentStore { class FileDocumentStore {
constructor(options){ constructor(options){

View file

@ -1,89 +1,49 @@
const redis = require('redis'); const Redis = require('ioredis');
const winston = require('winston'); const winston = require('winston');
// For storing in redis class RedisDocumentStore {
// options[type] = redis
// options[host] - The host to connect to (default localhost)
// options[port] - The port to connect to (default 5379)
// options[db] - The db to use (default 0)
// options[expire] - The time to live for each key set (default never)
var RedisDocumentStore = function(options, client){ constructor(options = {}){
this.expire = options.expire; this.expire = options.expire;
if (client){ const redisClient = new Redis(options.redisOptions);
winston.info('using predefined redis client');
RedisDocumentStore.client = client;
} else if (!RedisDocumentStore.client){
winston.info('configuring redis');
RedisDocumentStore.connect(options);
}
};
// Create a connection according to config redisClient.on('error', err => {
RedisDocumentStore.connect = function(options){ winston.error('redisClient errored', {error: err});
var host = options.host || '127.0.0.1';
var port = options.port || 6379;
var index = options.db || 0;
RedisDocumentStore.client = redis.createClient(port, host);
// authenticate if password is provided
if (options.password){
RedisDocumentStore.client.auth(options.password);
}
RedisDocumentStore.client.on('error', function(err){
winston.error('redis disconnected', err);
});
RedisDocumentStore.client.select(index, function(err){
if (err){
winston.error(
'error connecting to redis index ' + index,
{ error: err }
);
process.exit(1); process.exit(1);
}
else {
winston.info('connected to redis on ' + host + ':' + port + '/' + index);
}
}); });
}; redisClient.on('ready', () => {
winston.info(`connected to redis on ${redisClient.options.host}:${redisClient.options.port}/${redisClient.options.db}`);
});
this.client = redisClient;
winston.info('initialized redis client');
}
// Save file in a key async set(key, data, callback, skipExpire){
RedisDocumentStore.prototype.set = function(key, data, callback, skipExpire){ await this.client.set(key, data).catch(err => {
var _this = this; winston.error('failed to call redisClient.set', {error: err});
RedisDocumentStore.client.set(key, data, function(err){
if (err){
callback(false); callback(false);
} return;
else { });
if (!skipExpire){ if (!skipExpire) this.setExpiration(key);
_this.setExpiration(key);
}
callback(true); callback(true);
} }
});
};
// Expire a key in expire time if set async get(key, callback, skipExpire){
RedisDocumentStore.prototype.setExpiration = function(key){ let data = await this.client.get(key).catch(err => {
if (this.expire){ winston.error('failed to get document from redis', {key: key, error: err});
RedisDocumentStore.client.expire(key, this.expire, function(err){ callback(false);
if (err){ return;
winston.error('failed to set expiry on key: ' + key);
}
}); });
if (!skipExpire) this.setExpiration(key);
callback(data);
} }
};
// Get a file from a key async setExpiration(key){
RedisDocumentStore.prototype.get = function(key, callback, skipExpire){ if (!this.expire) return;
var _this = this; await this.client.expire(key, this.expire).catch(err => {
RedisDocumentStore.client.get(key, function(err, reply){ winston.warn('failed to set expiry on key', {key: key, error: err});
if (!err && !skipExpire){
_this.setExpiration(key);
}
callback(err ? false : reply);
}); });
}; }
}
module.exports = RedisDocumentStore; module.exports = RedisDocumentStore;

44
package-lock.json generated
View file

@ -1,6 +1,6 @@
{ {
"name": "haste", "name": "haste",
"version": "0.2.2", "version": "0.2.3",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
@ -981,11 +981,6 @@
"object-keys": "^1.0.12" "object-keys": "^1.0.12"
} }
}, },
"denque": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
"integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
},
"depd": { "depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@ -2326,43 +2321,6 @@
"picomatch": "^2.2.1" "picomatch": "^2.2.1"
} }
}, },
"redis": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
"requires": {
"denque": "^1.4.1",
"redis-commands": "^1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0"
}
},
"redis-commands": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz",
"integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"requires": {
"redis-errors": "^1.0.0"
}
},
"redis-url": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/redis-url/-/redis-url-1.2.1.tgz",
"integrity": "sha1-GGcAlaOOmJ03k1ndTG5Kv/heLrE=",
"requires": {
"redis": ">= 0.0.1"
}
},
"regexpp": { "regexpp": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz",

View file

@ -1,6 +1,6 @@
{ {
"name": "haste", "name": "haste",
"version": "0.2.2", "version": "0.2.3",
"description": "Private Pastebin Server - continued by zneix", "description": "Private Pastebin Server - continued by zneix",
"keywords": [ "keywords": [
"paste", "paste",
@ -20,8 +20,6 @@
"express": "^4.17.1", "express": "^4.17.1",
"express-rate-limit": "^5.1.3", "express-rate-limit": "^5.1.3",
"pg": "^8.3.2", "pg": "^8.3.2",
"redis": "^3.0.2",
"redis-url": "^1.2.1",
"st": "^2.0.0", "st": "^2.0.0",
"winston": "^3.3.3" "winston": "^3.3.3"
}, },
@ -29,17 +27,6 @@
"eslint": "^7.7.0", "eslint": "^7.7.0",
"mocha": "^8.1.2" "mocha": "^8.1.2"
}, },
"bin": {
"haste-server": "./server.js"
},
"files": [
"server.js",
"lib",
"static"
],
"directories": {
"lib": "./lib"
},
"scripts": { "scripts": {
"start": "node server.js", "start": "node server.js",
"test": "mocha --recursive" "test": "mocha --recursive"

View file

@ -33,16 +33,8 @@ if (!config.storage){
} }
if (!config.storage.type) config.storage.type = 'file'; if (!config.storage.type) config.storage.type = 'file';
let preferredStore;
if (process.env.REDISTOGO_URL && config.storage.type == 'redis'){
let redisClient = require('redis-url').connect(process.env.REDISTOGO_URL);
let Store = require('./lib/document_stores/redis');
preferredStore = new Store(config.storage, redisClient);
}
else {
let Store = require(`./lib/document_stores/${config.storage.type}`); let Store = require(`./lib/document_stores/${config.storage.type}`);
preferredStore = new Store(config.storage); let preferredStore = new Store(config.storage);
}
//compress static javascript assets //compress static javascript assets
if (config.compressStaticAssets){ if (config.compressStaticAssets){