From 171ddfe19ee434f1d684ea0ae660f0e09fe450d6 Mon Sep 17 00:00:00 2001 From: Youri Tolstoy Date: Thu, 28 Jan 2016 10:31:29 +0000 Subject: [PATCH 1/4] Upgrading Mongo driver and cache-manager to make it work with Mongo 3.2 --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index f3c35bb..3900b33 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "homepage": "https://github.com/lammertw/prerender-mongodb-cache", "dependencies": { - "mongodb": "~2.1.0", - "cache-manager": "0.2.0" + "cache-manager": "^1.3.0", + "mongodb": "^2.1.4" } } From a73c6af2d80a4ede8f796e3a93d46ee2c555863d Mon Sep 17 00:00:00 2001 From: Youri Tolstoy Date: Tue, 2 Feb 2016 10:15:36 +0000 Subject: [PATCH 2/4] Using newer cache manager and the mongo adapter of it + reviewed README --- README.md | 8 ++++-- lib/mongoCache.js | 72 +++++++++++++++++++++++++++-------------------- package.json | 2 +- 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 69fa899..2411846 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ prerender-mongodb-cache ======================= -Prerender plugin for MongoDB caching, to be used with the prerender node application from https://github.com/prerender/prerender. +Prerender plugin for MongoDB caching, to be used with the prerender node application from https://github.com/prerender/prerender. It relys on [cache-manager](https://github.com/BryanDonovan/node-cache-manager) and on the [mongo adapter](https://github.com/v4l3r10/node-cache-manager-mongodb) How it works ------------ This plugin will store all prerendered pages into a MongoDB instance. There is currently no expiration functionality, which means that once a page is stored, future requests for prerendering a page will always be served from from the database cache if it's available and the page caches are never updated. -To get a fresh cache, you will have to delete the cache in the MongoDB instance manually or from another process. +To set the cache lifetime use the environment variable when launching prerender: `CACHE_TTL` (in seconds) by default it has been set to one month. How to use ---------- @@ -19,7 +19,9 @@ In your local prerender project run: Then in the server.js that initializes the prerender: - server.use(require('prerender-mongodb-cache')); +```js +server.use(require('prerender-mongodb-cache')); +``` Configuration ------------- diff --git a/lib/mongoCache.js b/lib/mongoCache.js index 7156d72..821eb05 100644 --- a/lib/mongoCache.js +++ b/lib/mongoCache.js @@ -5,27 +5,48 @@ var mongoUri = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || 'mongodb://localhost/prerender'; -var database; +var ttl = process.env.CACHE_TTL || 2592000; -MongoClient.connect(mongoUri, function(err, db) { - database = db; -}); - -var cache_manager = require('cache-manager'); +var cacheManager = require('cache-manager'), + mongoStore = require('cache-manager-mongodb'), + mongoCache; module.exports = { init: function() { - this.cache = cache_manager.caching({ - store: mongo_cache + mongoCache = cacheManager.caching({ + store: mongoStore, + uri: mongoUri, + options: { + collection: "pages", + compression: false, + server: { + poolSize: 5, + auto_reconnect: true + } + }, + ttl: ttl }); }, beforePhantomRequest: function(req, res, next) { - if(req.method !== 'GET') { + + var url = req.url.replace(/%3f\_escaped\_fragment.*/gi, ''); + var parts = req.url.split('/'), + lastPart = parts[parts.length - 1]; + try { + url = url.replace(lastPart, decodeURIComponent(lastPart).replace(/\?\_escaped\_fragment.*/g, '')); + } catch (e) {}; + + if (req.method !== 'GET') { return next(); } - this.cache.get(req.url, function (err, result) { + + mongoCache.get(url, function(err, result) { + if (err) { + throw err; + } + if (!err && result) { res.send(200, result); } else { @@ -35,28 +56,19 @@ module.exports = { }, afterPhantomRequest: function(req, res, next) { - this.cache.set(req.url, req.prerender.documentHTML); - next(); - } -}; + var url = req.url.replace(/%3f\_escaped\_fragment.*/gi, ''); + var parts = req.url.split('/'), + lastPart = parts[parts.length - 1]; + try { + url = url.replace(lastPart, decodeURIComponent(lastPart).replace(/\?\_escaped\_fragment.*/g, '')); + } catch (e) {}; -var mongo_cache = { - get: function(key, callback) { - database.collection('pages', function(err, collection) { - collection.findOne({key: key}, function (err, item) { - var value = item ? item.value : null; - callback(err, value); - }); - }); - }, - set: function(key, value, callback) { - database.collection('pages', function(err, collection) { - var object = {key: key, value: value, created: new Date()}; - collection.update({key: key}, object, { - upsert: true - }, function (err) { - }); + mongoCache.set(url, req.prerender.documentHTML, function(err) { + if (err) { + throw err; + } + next(); }); } }; diff --git a/package.json b/package.json index 3900b33..a632b65 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,6 @@ "homepage": "https://github.com/lammertw/prerender-mongodb-cache", "dependencies": { "cache-manager": "^1.3.0", - "mongodb": "^2.1.4" + "cache-manager-mongodb": "^0.1.6" } } From b982bb8cb8709b7907a682ce09b27421fe9533bd Mon Sep 17 00:00:00 2001 From: Youri Tolstoy Date: Tue, 2 Feb 2016 10:17:42 +0000 Subject: [PATCH 3/4] Contributors section --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 2411846..cd48eed 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,9 @@ Configuration By default it will connect to your MongoDB instance running on localhost and use the *prerender* collection. You can overwrite this by setting the `MONGOLAB_URI` or `MONGOHQ_URL` environment variables to valid MongoDB connection strings. This is done to make it work automatically when deployed on Heroku with the MongoDB add-ons. + +Contributions +------------- + +Creator: [lammertw](https://github.com/lammertw) +Contributor: [YouriT](https://github.com/YouriT) From 3aa1abaab7ea3433e49054783fade6dd027423b9 Mon Sep 17 00:00:00 2001 From: Youri Tolstoy Date: Fri, 12 Feb 2016 17:36:15 +0000 Subject: [PATCH 4/4] More security --- lib/mongoCache.js | 51 ++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/lib/mongoCache.js b/lib/mongoCache.js index 821eb05..ebacd96 100644 --- a/lib/mongoCache.js +++ b/lib/mongoCache.js @@ -30,7 +30,9 @@ module.exports = { beforePhantomRequest: function(req, res, next) { - var url = req.url.replace(/%3f\_escaped\_fragment.*/gi, ''); + var currentTTL = req.headers['cache-ttl'] || ttl; + + var url = req.url.replace(/%(25)+/gi, '%').replace(/%3f\_escaped\_fragment.*/gi, ''); var parts = req.url.split('/'), lastPart = parts[parts.length - 1]; try { @@ -42,33 +44,46 @@ module.exports = { } - mongoCache.get(url, function(err, result) { - if (err) { - throw err; - } - - if (!err && result) { - res.send(200, result); - } else { - next(); - } - }); + try { + mongoCache.get(url, function(err, result) { + if (err) { + throw err; + } + if (!err && result) { + res.send(200, result); + } else { + next(); + } + }); + } catch (e) { + console.log(e.stack || e.message); + next(); + } }, afterPhantomRequest: function(req, res, next) { - var url = req.url.replace(/%3f\_escaped\_fragment.*/gi, ''); + var currentTTL = req.headers['cache-ttl'] || ttl; + + var url = req.url.replace(/%(25)+/gi, '%').replace(/%3f\_escaped\_fragment.*/gi, ''); var parts = req.url.split('/'), lastPart = parts[parts.length - 1]; try { url = url.replace(lastPart, decodeURIComponent(lastPart).replace(/\?\_escaped\_fragment.*/g, '')); } catch (e) {}; - mongoCache.set(url, req.prerender.documentHTML, function(err) { - if (err) { - throw err; - } + try { + mongoCache.set(url, req.prerender.documentHTML, { + ttl: currentTTL + }, function(err) { + if (err) { + throw err; + } + next(); + }); + } catch (e) { + console.log(e.stack || e.message); next(); - }); + } } };