pull out editor-client and editor-api

This commit is contained in:
Nick O'Leary
2018-08-17 22:10:54 +01:00
parent 6b79c6135f
commit e57d8ba0ef
287 changed files with 245 additions and 294 deletions

View File

@@ -0,0 +1,384 @@
/**
* Copyright JS Foundation and other contributors, http://js.foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
var clone = require("clone");
var log = require("@node-red/util").log; // TODO: separate module
var memory = require("./memory");
var settings;
// A map of scope id to context instance
var contexts = {};
// A map of store name to instance
var stores = {};
var storeList = [];
var defaultStore;
// Whether there context storage has been configured or left as default
var hasConfiguredStore = false;
// Unknown Stores
var unknownStores = {};
function logUnknownStore(name) {
if (name) {
var count = unknownStores[name] || 0;
if (count == 0) {
log.warn(log._("context.unknown-store", {name: name}));
count++;
unknownStores[name] = count;
}
}
}
function init(_settings) {
settings = _settings;
contexts = {};
stores = {};
storeList = [];
hasConfiguredStore = false;
var seed = settings.functionGlobalContext || {};
contexts['global'] = createContext("global",seed);
// create a default memory store - used by the unit tests that skip the full
// `load()` initialisation sequence.
// If the user has any stores configured, this will be disgarded
stores["_"] = new memory();
defaultStore = "memory";
}
function load() {
return new Promise(function(resolve,reject) {
// load & init plugins in settings.contextStorage
var plugins = settings.contextStorage || {};
var defaultIsAlias = false;
var promises = [];
if (plugins && Object.keys(plugins).length > 0) {
var hasDefault = plugins.hasOwnProperty('default');
var defaultName;
for (var pluginName in plugins) {
if (plugins.hasOwnProperty(pluginName)) {
// "_" is a reserved name - do not allow it to be overridden
if (pluginName === "_") {
continue;
}
if (!/^[a-zA-Z0-9_]+$/.test(pluginName)) {
return reject(new Error(log._("context.error-invalid-module-name", {name:pluginName})));
}
// Check if this is setting the 'default' context to be a named plugin
if (pluginName === "default" && typeof plugins[pluginName] === "string") {
// Check the 'default' alias exists before initialising anything
if (!plugins.hasOwnProperty(plugins[pluginName])) {
return reject(new Error(log._("context.error-invalid-default-module", {storage:plugins["default"]})));
}
defaultIsAlias = true;
continue;
}
if (!hasDefault && !defaultName) {
defaultName = pluginName;
}
var plugin;
if (plugins[pluginName].hasOwnProperty("module")) {
// Get the provided config and copy in the 'approved' top-level settings (eg userDir)
var config = plugins[pluginName].config || {};
copySettings(config, settings);
if (typeof plugins[pluginName].module === "string") {
// This config identifies the module by name - assume it is a built-in one
// TODO: check it exists locally, if not, try to require it as-is
try {
plugin = require("./"+plugins[pluginName].module);
} catch(err) {
return reject(new Error(log._("context.error-loading-module", {module:plugins[pluginName].module,message:err.toString()})));
}
} else {
// Assume `module` is an already-required module we can use
plugin = plugins[pluginName].module;
}
try {
// Create a new instance of the plugin by calling its module function
stores[pluginName] = plugin(config);
var moduleInfo = plugins[pluginName].module;
if (typeof moduleInfo !== 'string') {
if (moduleInfo.hasOwnProperty("toString")) {
moduleInfo = moduleInfo.toString();
} else {
moduleInfo = "custom";
}
}
log.info(log._("context.log-store-init", {name:pluginName, info:"module="+moduleInfo}));
} catch(err) {
return reject(new Error(log._("context.error-loading-module",{module:pluginName,message:err.toString()})));
}
} else {
// Plugin does not specify a 'module'
return reject(new Error(log._("context.error-module-not-defined", {storage:pluginName})));
}
}
}
// Open all of the configured contexts
for (var plugin in stores) {
if (stores.hasOwnProperty(plugin)) {
promises.push(stores[plugin].open());
}
}
// There is a 'default' listed in the configuration
if (hasDefault) {
// If 'default' is an alias, point it at the right module - we have already
// checked that it exists. If it isn't an alias, then it will
// already be set to a configured store
if (defaultIsAlias) {
stores["_"] = stores[plugins["default"]];
defaultStore = plugins["default"];
} else {
stores["_"] = stores["default"];
defaultStore = "default";
}
} else if (defaultName) {
// No 'default' listed, so pick first in list as the default
stores["_"] = stores[defaultName];
defaultStore = defaultName;
defaultIsAlias = true;
} else {
// else there were no stores list the config object - fall through
// to below where we default to a memory store
storeList = ["memory"];
defaultStore = "memory";
}
hasConfiguredStore = true;
storeList = Object.keys(stores).filter(n=>!(defaultIsAlias && n==="default") && n!== "_");
} else {
// No configured plugins
log.info(log._("context.log-store-init", {name:"default", info:"module=memory"}));
promises.push(stores["_"].open())
storeList = ["memory"];
defaultStore = "memory";
}
return resolve(Promise.all(promises));
}).catch(function(err) {
throw new Error(log._("context.error-loading-module",{message:err.toString()}));
});
}
function copySettings(config, settings){
var copy = ["userDir"]
config.settings = {};
copy.forEach(function(setting){
config.settings[setting] = clone(settings[setting]);
});
}
function getContextStorage(storage) {
if (stores.hasOwnProperty(storage)) {
// A known context
return stores[storage];
} else if (stores.hasOwnProperty("_")) {
// Not known, but we have a default to fall back to
if (storage !== defaultStore) {
// It isn't the default store either, so log it
logUnknownStore(storage);
}
return stores["_"];
}
}
function createContext(id,seed) {
// Seed is only set for global context - sourced from functionGlobalContext
var scope = id;
var obj = seed || {};
var seedKeys;
var insertSeedValues;
if (seed) {
seedKeys = Object.keys(seed);
insertSeedValues = function(keys,values) {
if (!Array.isArray(keys)) {
if (values[0] === undefined) {
values[0] = seed[keys];
}
} else {
for (var i=0;i<keys.length;i++) {
if (values[i] === undefined) {
values[i] = seed[keys[i]];
}
}
}
}
}
obj.get = function(key, storage, callback) {
var context;
if (!storage && !callback) {
context = stores["_"];
} else {
if (typeof storage === 'function') {
callback = storage;
storage = "_";
}
if (callback && typeof callback !== 'function'){
throw new Error("Callback must be a function");
}
context = getContextStorage(storage);
}
if (callback) {
if (!seed) {
context.get(scope,key,callback);
} else {
context.get(scope,key,function() {
if (arguments[0]) {
callback(arguments[0]);
return;
}
var results = Array.prototype.slice.call(arguments,[1]);
insertSeedValues(key,results);
// Put the err arg back
results.unshift(undefined);
callback.apply(null,results);
});
}
} else {
// No callback, attempt to do this synchronously
var results = context.get(scope,key);
if (seed) {
if (Array.isArray(key)) {
insertSeedValues(key,results);
} else if (results === undefined){
results = seed[key];
}
}
return results;
}
};
obj.set = function(key, value, storage, callback) {
var context;
if (!storage && !callback) {
context = stores["_"];
} else {
if (typeof storage === 'function') {
callback = storage;
storage = "_";
}
if (callback && typeof callback !== 'function') {
throw new Error("Callback must be a function");
}
context = getContextStorage(storage);
}
context.set(scope, key, value, callback);
};
obj.keys = function(storage, callback) {
var context;
if (!storage && !callback) {
context = stores["_"];
} else {
if (typeof storage === 'function') {
callback = storage;
storage = "_";
}
if (callback && typeof callback !== 'function') {
throw new Error("Callback must be a function");
}
context = getContextStorage(storage);
}
if (seed) {
if (callback) {
context.keys(scope, function(err,keys) {
callback(err,Array.from(new Set(seedKeys.concat(keys)).keys()));
});
} else {
var keys = context.keys(scope);
return Array.from(new Set(seedKeys.concat(keys)).keys())
}
} else {
return context.keys(scope, callback);
}
};
return obj;
}
function getContext(localId,flowId) {
var contextId = localId;
if (flowId) {
contextId = localId+":"+flowId;
}
if (contexts.hasOwnProperty(contextId)) {
return contexts[contextId];
}
var newContext = createContext(contextId);
if (flowId) {
newContext.flow = getContext(flowId);
}
newContext.global = contexts['global'];
contexts[contextId] = newContext;
return newContext;
}
function deleteContext(id,flowId) {
if(!hasConfiguredStore){
// only delete context if there's no configured storage.
var contextId = id;
if (flowId) {
contextId = id+":"+flowId;
}
delete contexts[contextId];
return stores["_"].delete(contextId);
}else{
return Promise.resolve();
}
}
function clean(flowConfig) {
var promises = [];
for(var plugin in stores){
if(stores.hasOwnProperty(plugin)){
promises.push(stores[plugin].clean(Object.keys(flowConfig.allNodes)));
}
}
for (var id in contexts) {
if (contexts.hasOwnProperty(id) && id !== "global") {
var idParts = id.split(":");
if (!flowConfig.allNodes.hasOwnProperty(idParts[0])) {
delete contexts[id];
}
}
}
return Promise.all(promises);
}
function close() {
var promises = [];
for(var plugin in stores){
if(stores.hasOwnProperty(plugin)){
promises.push(stores[plugin].close());
}
}
return Promise.all(promises);
}
function listStores() {
return {default:defaultStore,stores:storeList};
}
module.exports = {
init: init,
load: load,
listStores: listStores,
get: getContext,
delete: deleteContext,
clean: clean,
close: close
};

View File

@@ -0,0 +1,377 @@
/**
* Copyright JS Foundation and other contributors, http://js.foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
/**
* Local file-system based context storage
*
* Configuration options:
* {
* base: "context", // the base directory to use
* // default: "context"
* dir: "/path/to/storage", // the directory to create the base directory in
* // default: settings.userDir
* cache: true, // whether to cache contents in memory
* // default: true
* flushInterval: 30 // if cache is enabled, the minimum interval
* // between writes to storage, in seconds. This
* can be used to reduce wear on underlying storage.
* default: 30 seconds
* }
*
*
* $HOME/.node-red/contexts
* ├── global
* │ └── global_context.json
* ├── <id of Flow 1>
* │ ├── flow_context.json
* │ ├── <id of Node a>.json
* │ └── <id of Node b>.json
* └── <id of Flow 2>
* ├── flow_context.json
* ├── <id of Node x>.json
* └── <id of Node y>.json
*/
var fs = require('fs-extra');
var path = require("path");
var util = require("@node-red/util").util;
var log = require("@node-red/util").log;
var safeJSONStringify = require("json-stringify-safe");
var MemoryStore = require("./memory");
function getStoragePath(storageBaseDir, scope) {
if(scope.indexOf(":") === -1){
if(scope === "global"){
return path.join(storageBaseDir,"global",scope);
}else{ // scope:flow
return path.join(storageBaseDir,scope,"flow");
}
}else{ // scope:local
var ids = scope.split(":")
return path.join(storageBaseDir,ids[1],ids[0]);
}
}
function getBasePath(config) {
var base = config.base || "context";
var storageBaseDir;
if (!config.dir) {
if(config.settings && config.settings.userDir){
storageBaseDir = path.join(config.settings.userDir, base);
}else{
try {
fs.statSync(path.join(process.env.NODE_RED_HOME,".config.json"));
storageBaseDir = path.join(process.env.NODE_RED_HOME, base);
} catch(err) {
try {
// Consider compatibility for older versions
if (process.env.HOMEPATH) {
fs.statSync(path.join(process.env.HOMEPATH,".node-red",".config.json"));
storageBaseDir = path.join(process.env.HOMEPATH, ".node-red", base);
}
} catch(err) {
}
if (!storageBaseDir) {
storageBaseDir = path.join(process.env.HOME || process.env.USERPROFILE || process.env.HOMEPATH || process.env.NODE_RED_HOME,".node-red", base);
}
}
}
}else{
storageBaseDir = path.join(config.dir, base);
}
return storageBaseDir;
}
function loadFile(storagePath){
return fs.pathExists(storagePath).then(function(exists){
if(exists === true){
return fs.readFile(storagePath, "utf8");
}else{
return Promise.resolve(undefined);
}
});
}
function listFiles(storagePath) {
var promises = [];
return fs.readdir(storagePath).then(function(files) {
files.forEach(function(file) {
if (!/^\./.test(file)) {
var fullPath = path.join(storagePath,file);
var stats = fs.statSync(fullPath);
if (stats.isDirectory()) {
promises.push(fs.readdir(fullPath).then(function(subdirFiles) {
var result = [];
subdirFiles.forEach(subfile => {
if (/\.json$/.test(subfile)) {
result.push(path.join(file,subfile))
}
});
return result;
}))
}
}
});
return Promise.all(promises);
}).then(dirs => dirs.reduce((acc, val) => acc.concat(val), []));
}
function stringify(value) {
var hasCircular;
var result = safeJSONStringify(value,null,4,function(k,v){hasCircular = true})
return { json: result, circular: hasCircular };
}
function LocalFileSystem(config){
this.config = config;
this.storageBaseDir = getBasePath(this.config);
if (config.hasOwnProperty('cache')?config.cache:true) {
this.cache = MemoryStore({});
}
this.pendingWrites = {};
this.knownCircularRefs = {};
if (config.hasOwnProperty('flushInterval')) {
this.flushInterval = Math.max(0,config.flushInterval) * 1000;
} else {
this.flushInterval = 30000;
}
}
LocalFileSystem.prototype.open = function(){
var self = this;
if (this.cache) {
var scopes = [];
var promises = [];
return listFiles(self.storageBaseDir).then(function(files) {
files.forEach(function(file) {
var parts = file.split(path.sep);
if (parts[0] === 'global') {
scopes.push("global");
} else if (parts[1] === 'flow.json') {
scopes.push(parts[0])
} else {
scopes.push(parts[1].substring(0,parts[1].length-5)+":"+parts[0]);
}
promises.push(loadFile(path.join(self.storageBaseDir,file)));
})
return Promise.all(promises);
}).then(function(res) {
scopes.forEach(function(scope,i) {
var data = res[i]?JSON.parse(res[i]):{};
Object.keys(data).forEach(function(key) {
self.cache.set(scope,key,data[key]);
})
});
}).catch(function(err){
if(err.code == 'ENOENT') {
return fs.ensureDir(self.storageBaseDir);
}else{
throw err;
}
}).then(function() {
self._flushPendingWrites = function() {
var scopes = Object.keys(self.pendingWrites);
self.pendingWrites = {};
var promises = [];
var newContext = self.cache._export();
scopes.forEach(function(scope) {
var storagePath = getStoragePath(self.storageBaseDir,scope);
var context = newContext[scope];
var stringifiedContext = stringify(context);
if (stringifiedContext.circular && !self.knownCircularRefs[scope]) {
log.warn(log._("error-circular",{scope:scope}));
self.knownCircularRefs[scope] = true;
} else {
delete self.knownCircularRefs[scope];
}
log.debug("Flushing localfilesystem context scope "+scope);
promises.push(fs.outputFile(storagePath + ".json", stringifiedContext.json, "utf8"));
});
delete self._pendingWriteTimeout;
return Promise.all(promises);
}
});
} else {
return fs.ensureDir(self.storageBaseDir);
}
}
LocalFileSystem.prototype.close = function(){
if (this.cache && this._flushPendingWrites) {
clearTimeout(this._pendingWriteTimeout);
delete this._pendingWriteTimeout;
return this._flushPendingWrites();
}
return Promise.resolve();
}
LocalFileSystem.prototype.get = function(scope, key, callback) {
if (this.cache) {
return this.cache.get(scope,key,callback);
}
if(typeof callback !== "function"){
throw new Error("Callback must be a function");
}
var storagePath = getStoragePath(this.storageBaseDir ,scope);
loadFile(storagePath + ".json").then(function(data){
if(data){
data = JSON.parse(data);
if (!Array.isArray(key)) {
callback(null, util.getObjectProperty(data,key));
} else {
var results = [undefined];
for (var i=0;i<key.length;i++) {
results.push(util.getObjectProperty(data,key[i]))
}
callback.apply(null,results);
}
}else{
callback(null, undefined);
}
}).catch(function(err){
callback(err);
});
};
LocalFileSystem.prototype.set = function(scope, key, value, callback) {
var self = this;
var storagePath = getStoragePath(this.storageBaseDir ,scope);
if (this.cache) {
this.cache.set(scope,key,value,callback);
this.pendingWrites[scope] = true;
if (this._pendingWriteTimeout) {
// there's a pending write which will handle this
return;
} else {
this._pendingWriteTimeout = setTimeout(function() {
self._flushPendingWrites.call(self).catch(function(err) {
log.error(log._("context.localfilesystem.error-write",{message:err.toString()}))
});
}, this.flushInterval);
}
} else if (callback && typeof callback !== 'function') {
throw new Error("Callback must be a function");
} else {
loadFile(storagePath + ".json").then(function(data){
var obj = data ? JSON.parse(data) : {}
if (!Array.isArray(key)) {
key = [key];
value = [value];
} else if (!Array.isArray(value)) {
// key is an array, but value is not - wrap it as an array
value = [value];
}
for (var i=0;i<key.length;i++) {
var v = null;
if (i<value.length) {
v = value[i];
}
util.setObjectProperty(obj,key[i],v);
}
var stringifiedContext = stringify(obj);
if (stringifiedContext.circular && !self.knownCircularRefs[scope]) {
log.warn(log._("error-circular",{scope:scope}));
self.knownCircularRefs[scope] = true;
} else {
delete self.knownCircularRefs[scope];
}
return fs.outputFile(storagePath + ".json", stringifiedContext.json, "utf8");
}).then(function(){
if(typeof callback === "function"){
callback(null);
}
}).catch(function(err){
if(typeof callback === "function"){
callback(err);
}
});
}
};
LocalFileSystem.prototype.keys = function(scope, callback){
if (this.cache) {
return this.cache.keys(scope,callback);
}
if(typeof callback !== "function"){
throw new Error("Callback must be a function");
}
var storagePath = getStoragePath(this.storageBaseDir ,scope);
loadFile(storagePath + ".json").then(function(data){
if(data){
callback(null, Object.keys(JSON.parse(data)));
}else{
callback(null, []);
}
}).catch(function(err){
callback(err);
});
};
LocalFileSystem.prototype.delete = function(scope){
var cachePromise;
if (this.cache) {
cachePromise = this.cache.delete(scope);
} else {
cachePromise = Promise.resolve();
}
var that = this;
delete this.pendingWrites[scope];
return cachePromise.then(function() {
var storagePath = getStoragePath(that.storageBaseDir,scope);
return fs.remove(storagePath + ".json");
});
}
LocalFileSystem.prototype.clean = function(_activeNodes) {
var activeNodes = {};
_activeNodes.forEach(function(node) { activeNodes[node] = true });
var self = this;
var cachePromise;
if (this.cache) {
cachePromise = this.cache.clean(_activeNodes);
} else {
cachePromise = Promise.resolve();
}
this.knownCircularRefs = {};
return cachePromise.then(() => listFiles(self.storageBaseDir)).then(function(files) {
var promises = [];
files.forEach(function(file) {
var parts = file.split(path.sep);
var removePromise;
if (parts[0] === 'global') {
// never clean global
return;
} else if (!activeNodes[parts[0]]) {
// Flow removed - remove the whole dir
removePromise = fs.remove(path.join(self.storageBaseDir,parts[0]));
} else if (parts[1] !== 'flow.json' && !activeNodes[parts[1].substring(0,parts[1].length-5)]) {
// Node removed - remove the context file
removePromise = fs.remove(path.join(self.storageBaseDir,file));
}
if (removePromise) {
promises.push(removePromise);
}
});
return Promise.all(promises)
})
}
module.exports = function(config){
return new LocalFileSystem(config);
};

View File

@@ -0,0 +1,166 @@
/**
* Copyright JS Foundation and other contributors, http://js.foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
var util = require("@node-red/util").util;
function Memory(config){
this.data = {};
}
Memory.prototype.open = function(){
return Promise.resolve();
};
Memory.prototype.close = function(){
return Promise.resolve();
};
Memory.prototype._getOne = function(scope, key) {
var value;
var error;
if(this.data[scope]){
value = util.getObjectProperty(this.data[scope], key);
}
return value;
}
Memory.prototype.get = function(scope, key, callback) {
var value;
var error;
if (!Array.isArray(key)) {
try {
value = this._getOne(scope,key);
} catch(err) {
if (!callback) {
throw err;
}
error = err;
}
if (callback) {
callback(error,value);
return;
} else {
return value;
}
}
value = [];
for (var i=0; i<key.length; i++) {
try {
value.push(this._getOne(scope,key[i]));
} catch(err) {
if (!callback) {
throw err;
} else {
callback(err);
return;
}
}
}
if (callback) {
callback.apply(null, [undefined].concat(value));
} else {
return value;
}
};
Memory.prototype.set = function(scope, key, value, callback) {
if(!this.data[scope]){
this.data[scope] = {};
}
var error;
if (!Array.isArray(key)) {
key = [key];
value = [value];
} else if (!Array.isArray(value)) {
// key is an array, but value is not - wrap it as an array
value = [value];
}
try {
for (var i=0; i<key.length; i++) {
var v = null;
if (i < value.length) {
v = value[i];
}
util.setObjectProperty(this.data[scope],key[i],v);
}
} catch(err) {
if (callback) {
error = err;
} else {
throw err;
}
}
if(callback){
callback(error);
}
};
Memory.prototype.keys = function(scope, callback){
var values = [];
var error;
try{
if(this.data[scope]){
if (scope !== "global") {
values = Object.keys(this.data[scope]);
} else {
values = Object.keys(this.data[scope]).filter(function (key) {
return key !== "set" && key !== "get" && key !== "keys";
});
}
}
}catch(err){
if(callback){
error = err;
}else{
throw err;
}
}
if(callback){
if(error){
callback(error);
} else {
callback(null, values);
}
} else {
return values;
}
};
Memory.prototype.delete = function(scope){
delete this.data[scope];
return Promise.resolve();
};
Memory.prototype.clean = function(activeNodes){
for(var id in this.data){
if(this.data.hasOwnProperty(id) && id !== "global"){
var idParts = id.split(":");
if(activeNodes.indexOf(idParts[0]) === -1){
delete this.data[id];
}
}
}
return Promise.resolve();
}
Memory.prototype._export = function() {
return this.data;
}
module.exports = function(config){
return new Memory(config);
};