From 43d7c8d48cdb352835755ff2693a1cc3014f4f04 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Mon, 2 Jul 2018 22:32:20 +0100 Subject: [PATCH] Add caching to localfilesystem context --- red/api/admin/context.js | 32 +++-- red/runtime/nodes/context/localfilesystem.js | 134 +++++++++++++++--- .../nodes/context/localfilesystem_spec.js | 6 +- 3 files changed, 140 insertions(+), 32 deletions(-) diff --git a/red/api/admin/context.js b/red/api/admin/context.js index 37c6462cb..66aedcdcb 100644 --- a/red/api/admin/context.js +++ b/red/api/admin/context.js @@ -29,7 +29,6 @@ module.exports = { var scope = req.params.scope; var id = req.params.id; var key = req.params[0]; - var result = {}; var ctx; if (scope === 'global') { ctx = redNodes.getContext('global'); @@ -43,19 +42,28 @@ module.exports = { } if (ctx) { if (key) { - result = util.encodeObject({msg:ctx.get(key)}); + ctx.get(key,function(err, v) { + console.log(key,v); + res.json(util.encodeObject({msg:v})); + }); + return; } else { - var keys = ctx.keys(); - - var i = 0; - var l = keys.length; - while(i < l) { - var k = keys[i]; - result[k] = util.encodeObject({msg:ctx.get(k)}); - i++; - } + ctx.keys(function(err, keys) { + var result = {}; + var c = keys.length; + keys.forEach(function(key) { + ctx.get(key,function(err, v) { + result[key] = util.encodeObject({msg:v}); + c--; + if (c === 0) { + res.json(result); + } + }); + }); + }); } + } else { + res.json({}); } - res.json(result); } } diff --git a/red/runtime/nodes/context/localfilesystem.js b/red/runtime/nodes/context/localfilesystem.js index fed8ae959..5cf37e4a6 100644 --- a/red/runtime/nodes/context/localfilesystem.js +++ b/red/runtime/nodes/context/localfilesystem.js @@ -14,10 +14,39 @@ * limitations under the License. **/ +/** + * Local file-system based context storage + * + * Configuration options: + * { + * base: "contexts", // the base directory to use + * // default: "contexts" + * dir: "/path/to/storage", // the directory to create the base directory in + * // default: settings.userDir + * cache: true // whether to cache contents in memory + * // default: true + * } + * + * + * $HOME/.node-red/contexts + * ├── global + * │ └── global_context.json + * ├── + * │ ├── flow_context.json + * │ ├── .json + * │ └── .json + * └── + * ├── flow_context.json + * ├── .json + * └── .json + */ + var fs = require('fs-extra'); var path = require("path"); var util = require("../../util"); +var MemoryStore = require("./memory"); + function getStoragePath(storageBaseDir, scope) { if(scope.indexOf(":") === -1){ if(scope === "global"){ @@ -76,10 +105,48 @@ function loadFile(storagePath){ function LocalFileSystem(config){ this.config = config; this.storageBaseDir = getBasePath(this.config); + if (config.hasOwnProperty('cache')?config.cache:true) { + this.cache = MemoryStore({}); + } } LocalFileSystem.prototype.open = function(){ - return Promise.resolve(); + var self = this; + if (this.cache) { + var scopes = []; + var promises = []; + var subdirs = []; + var subdirPromises = []; + return fs.readdir(self.storageBaseDir).then(function(dirs){ + dirs.forEach(function(fn) { + var p = getStoragePath(self.storageBaseDir ,fn)+".json"; + scopes.push(fn); + promises.push(loadFile(p)); + subdirs.push(path.join(self.storageBaseDir,fn)); + subdirPromises.push(fs.readdir(path.join(self.storageBaseDir,fn))); + }) + return Promise.all(subdirPromises); + }).then(function(dirs) { + dirs.forEach(function(files,i) { + files.forEach(function(fn) { + if (fn !== 'flow.json' && fn !== 'global.json') { + scopes.push(fn.substring(0,fn.length-5)+":"+scopes[i]); + promises.push(loadFile(path.join(subdirs[i],fn))) + } + }); + }) + return Promise.all(promises); + }).then(function(res) { + scopes.forEach(function(scope,i) { + var data = res[i]?JSON.parse(res[i]):{}; + Object.keys(data).forEach(function(key) { + self.cache.set(scope,key,data[key]); + }) + }); + }) + } else { + return Promise.resolve(); + } } LocalFileSystem.prototype.close = function(){ @@ -87,6 +154,9 @@ LocalFileSystem.prototype.close = function(){ } LocalFileSystem.prototype.get = function(scope, key, callback) { + if (this.cache) { + return this.cache.get(scope,key,callback); + } if(typeof callback !== "function"){ throw new Error("Callback must be a function"); } @@ -102,7 +172,7 @@ LocalFileSystem.prototype.get = function(scope, key, callback) { }); }; -LocalFileSystem.prototype.set =function(scope, key, value, callback) { +LocalFileSystem.prototype._set = function(scope, key, value, callback) { var storagePath = getStoragePath(this.storageBaseDir ,scope); loadFile(storagePath + ".json").then(function(data){ var obj = data ? JSON.parse(data) : {} @@ -117,9 +187,23 @@ LocalFileSystem.prototype.set =function(scope, key, value, callback) { callback(err); } }); +} + +LocalFileSystem.prototype.set = function(scope, key, value, callback) { + if (this.cache) { + this.cache.set(scope,key,value,callback); + this._set(scope,key,value, function(err) { + // TODO: log any errors + }); + } else { + this._set(scope,key,value,callback); + } }; LocalFileSystem.prototype.keys = function(scope, callback){ + if (this.cache) { + return this.cache.keys(scope,callback); + } if(typeof callback !== "function"){ throw new Error("Callback must be a function"); } @@ -136,29 +220,45 @@ LocalFileSystem.prototype.keys = function(scope, callback){ }; LocalFileSystem.prototype.delete = function(scope){ - var storagePath = getStoragePath(this.storageBaseDir ,scope); - return fs.remove(storagePath + ".json"); + var cachePromise; + if (this.cache) { + cachePromise = this.cache.delete(scope); + } else { + cachePromise = Promise.resolve(); + } + var that = this; + return cachePromise.then(function() { + var storagePath = getStoragePath(that.storageBaseDir,scope); + return fs.remove(storagePath + ".json"); + }); } LocalFileSystem.prototype.clean = function(activeNodes){ var self = this; - return fs.readdir(self.storageBaseDir).then(function(dirs){ - return Promise.all(dirs.reduce(function(result, item){ - if(item !== "global" && activeNodes.indexOf(item) === -1){ - result.push(fs.remove(path.join(self.storageBaseDir,item))); + var cachePromise; + if (this.cache) { + cachePromise = this.cache.clean(activeNodes); + } else { + cachePromise = Promise.resolve(); + } + return cachePromise.then(function() { + return fs.readdir(self.storageBaseDir).then(function(dirs){ + return Promise.all(dirs.reduce(function(result, item){ + if(item !== "global" && activeNodes.indexOf(item) === -1){ + result.push(fs.remove(path.join(self.storageBaseDir,item))); + } + return result; + },[])); + }).catch(function(err){ + if(err.code == 'ENOENT') { + return Promise.resolve(); + }else{ + return Promise.reject(err); } - return result; - },[])); - }).catch(function(err){ - if(err.code == 'ENOENT') { - return Promise.resolve(); - }else{ - return Promise.reject(err); - } + }); }); } module.exports = function(config){ return new LocalFileSystem(config); }; - diff --git a/test/red/runtime/nodes/context/localfilesystem_spec.js b/test/red/runtime/nodes/context/localfilesystem_spec.js index a03d5b24a..61d591b07 100644 --- a/test/red/runtime/nodes/context/localfilesystem_spec.js +++ b/test/red/runtime/nodes/context/localfilesystem_spec.js @@ -29,7 +29,7 @@ describe('localfilesystem',function() { }); beforeEach(function() { - context = LocalFileSystem({dir: resourcesDir}); + context = LocalFileSystem({dir: resourcesDir, cache: false}); return context.open(); }); @@ -308,7 +308,7 @@ describe('localfilesystem',function() { done(); }); }); - }); + }).catch(done); }); }); }); @@ -375,4 +375,4 @@ describe('localfilesystem',function() { }); }); }); -}); \ No newline at end of file +});