frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Abstract all file-system operations

Browse Source 
Stage 1 of  #62
This commit is contained in:
Nicholas O'Leary 2013-11-10 00:05:58 +00:00
parent 22f46a4317
commit 95bef6b6ca
7 changed files with 480 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -21,6 +21,7 @@
],
"dependencies": {
"express": "3.x",
"when": "~2.6.0",
"mqtt": "~0.3.3",
"ws": "~0.4.31",
"mustache": "~0.7.2",

213
red/library.js
View File

@ -14,12 +14,13 @@
* limitations under the License.
**/
var fs = require("fs");
var fspath = require("path");
var redApp = null;
var storage = null;
function init() {
redApp = require("./server").app;
storage = require("./storage").storage;
// -------- Flow Library --------
redApp.post(new RegExp("/library/flows\/(.*)"), function(req,res) {
var fullBody = '';
@ -27,65 +28,35 @@ function init() {
fullBody += chunk.toString();
});
req.on('end', function() {
var fn = "lib/flows/"+req.params[0]+".json";
var parts = fn.split("/");
for (var i = 3;i<parts.length;i+=1) {
var dirname = parts.slice(0,i).join("/");
if (!fs.existsSync(dirname)) {
fs.mkdirSync(dirname);
}
}
fs.writeFile(fn,fullBody,function(err) {
storage.saveFlow(req.params[0],fullBody).then(function() {
res.writeHead(204, {'Content-Type': 'text/plain'});
res.end();
});
}).otherwise(function(err) {
util.log("[red] Error loading flow '"+req.params[0]+"' : "+err);
res.send(500);
});
});
function listFiles(dir) {
var dirs = {};
var files = [];
var dirCount = 0;
fs.readdirSync(dir).sort().filter(function(fn) {
var stats = fs.lstatSync(dir+"/"+fn);
if (stats.isDirectory()) {
dirCount += 1;
dirs[fn] = listFiles(dir+"/"+fn);
} else {
files.push(fn.split(".")[0]);
}
});
var result = {};
if (dirCount > 0) { result.d = dirs; }
if (files.length > 0) { result.f = files; }
return result;
}
redApp.get("/library/flows",function(req,res) {
var flows = {};
if (fs.existsSync("lib/flows")) {
flows = listFiles("lib/flows");
} else {
fs.mkdirSync("lib/flows");
}
storage.getAllFlows().then(function(flows) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(JSON.stringify(flows));
res.end();
});
});
redApp.get(new RegExp("/library/flows\/(.*)"), function(req,res) {
var fn = "lib/flows/"+req.params[0]+".json";
if (fs.existsSync(fn)) {
fs.readFile(fn,function(err,data) {
storage.getFlow(req.params[0]).then(function(data) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(data);
res.end();
});
} else {
res.send(404);
}).otherwise(function(err) {
if (err) {
util.log("[red] Error loading flow '"+req.params[0]+"' : "+err);
}
res.send(404);
});
});
// ------------------------------
@ -93,53 +64,21 @@ function init() {
function createLibrary(type) {
//TODO: use settings.library as the base root
var root = fspath.join("lib",type)+"/";
fs.exists(root,function(exists) {
if (!exists) {
fs.mkdir(root);
}
});
redApp.get(new RegExp("/library/"+type+"($|\/(.*))"),function(req,res) {
var path = req.params[1]||"";
var rootPath = fspath.join(root,path);
fs.exists(rootPath,function(exists) {
fs.lstat(rootPath,function(err,stats) {
if (stats.isFile()) {
getFileBody(root,path,res);
} else {
if (path.substr(-1) == '/') {
path = path.substr(0,path.length-1);
}
fs.readdir(rootPath,function(err,fns) {
var dirs = [];
var files = [];
fns.sort().filter(function(fn) {
var fullPath = fspath.join(path,fn);
var absoluteFullPath = fspath.join(root,fullPath);
if (fn[0] != ".") {
var stats = fs.lstatSync(absoluteFullPath);
if (stats.isDirectory()) {
dirs.push(fn);
} else {
var meta = getFileMeta(root,fullPath);
meta.fn = fn;
files.push(meta);
}
}
});
storage.getLibraryEntry(type,path).then(function(result) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(JSON.stringify(dirs.concat(files)));
res.end();
});
if (typeof result === "string") {
res.write(result);
} else {
res.write(JSON.stringify(result));
}
});
res.end();
}).otherwise(function(err) {
if (err) {
util.log("[red] Error loading library entry '"+path+"' : "+err);
}
res.send(404);
});
});
@ -150,103 +89,15 @@ function createLibrary(type) {
fullBody += chunk.toString();
});
req.on('end', function() {
writeFile(root,path,req.query,fullBody,res);
storage.saveLibraryEntry(type,path,req.query,fullBody).then(function() {
res.send(204);
}).otherwise(function(err) {
util.log("[red] Error saving library entry '"+path+"' : "+err);
res.send(500);
});;
});
});
}
function writeFile(root,path,meta,body,res) {
var fn = fspath.join(root,path);
var parts = path.split("/");
var dirname = root;
for (var i = 0;i<parts.length-1;i+=1) {
dirname = fspath.join(dirname,parts[i]);
if (!fs.existsSync(dirname)) {
fs.mkdirSync(dirname);
}
}
var headers = "";
for (var i in meta) {
headers += "// "+i+": "+meta[i]+"\n";
}
fs.writeFile(fn,headers+body,function(err) {
//TODO: handle error
res.writeHead(204, {'Content-Type': 'text/plain'});
res.end();
});
}
function getFileMeta(root,path) {
var fn = fspath.join(root,path);
var fd = fs.openSync(fn,"r");
var size = fs.fstatSync(fd).size;
var meta = {};
var scanning = true;
var read = 0;
var length = 10;
var offset = 0;
var remaining = "";
var buffer = Buffer(length);
while(read < size) {
read+=fs.readSync(fd,buffer,0,length);
var data = remaining+buffer.toString();
var parts = data.split("\n");
remaining = parts.splice(-1);
for (var i=0;i<parts.length;i+=1) {
var match = /^\/\/ (\w+): (.*)/.exec(parts[i]);
if (match) {
meta[match[1]] = match[2];
} else {
read = size;
break;
}
}
}
fs.closeSync(fd);
return meta;
}
function getFileBody(root,path,res) {
var fn = fspath.join(root,path);
res.writeHead(200, {'Content-Type': 'text/plain'});
var fd = fs.openSync(fn,"r");
var size = fs.fstatSync(fd).size;
var scanning = true;
var read = 0;
var length = 10;
var offset = 0;
var remaining = "";
var buffer = Buffer(length);
while(read < size) {
var thisRead = fs.readSync(fd,buffer,0,length);
read += thisRead;
if (scanning) {
var data = remaining+buffer.toString();
var parts = data.split("\n");
remaining = parts.splice(-1)[0];
for (var i=0;i<parts.length;i+=1) {
if (! /^\/\/ \w+: /.test(parts[i])) {
scanning = false;
res.write(parts[i]+"\n");
}
}
if (! /^\/\/ \w+: /.test(remaining)) {
scanning = false;
}
if (!scanning) {
res.write(remaining);
}
} else {
res.write(buffer.slice(0,thisRead));
}
}
fs.closeSync(fd);
res.end();
}
module.exports.init = init;
module.exports.register = createLibrary;

35
red/nodes.js
View File

@ -17,6 +17,7 @@ var util = require("util");
var EventEmitter = require("events").EventEmitter;
var fs = require("fs");
var events = require("./events");
var storage = null;
function getCallerFilename(type) {
//if (type == "summary") {
@ -194,29 +195,17 @@ Node.prototype.error = function(msg) {
}
var credentials = {};
var credentialsFile = "credentials.json";
if (fs.existsSync(credentialsFile)) {
credentials = JSON.parse(fs.readFileSync(credentialsFile,'utf8'));
}
function saveCredentialsFile() {
fs.writeFile(credentialsFile, JSON.stringify(credentials), function(err) {
if(err) {
util.log(err);
}
});
}
module.exports.addCredentials = function(id,creds) {
credentials[id] = creds;
saveCredentialsFile();
storage.saveCredentials(credentials);
}
module.exports.getCredentials = function(id) {
return credentials[id];
}
module.exports.deleteCredentials = function(id) {
delete credentials[id];
saveCredentialsFile();
storage.saveCredentials(credentials);
}
module.exports.createNode = function(node,def) {
Node.call(node,def);
@ -253,7 +242,7 @@ module.exports.load = function() {
//events.emit("nodes-loaded");
}
var activeConfig = null;
var activeConfig = [];
var missingTypes = [];
events.on('type-registered',function(type) {
@ -285,7 +274,21 @@ module.exports.stopFlows = stopFlows;
module.exports.setConfig = function(conf) {
stopFlows();
activeConfig = conf;
if (!storage) {
// Do this lazily to ensure the storage provider as been initialised
storage = require("./storage").storage;
}
storage.getCredentials().then(function(creds) {
credentials = creds;
parseConfig();
}).otherwise(function(err) {
util.log("[red] Error loading credentials : "+err);
});
}
module.exports.getConfig = function() {
return activeConfig;
}
var parseConfig = function() {
@ -339,7 +342,7 @@ var parseConfig = function() {
}
}
if (deletedCredentials) {
saveCredentialsFile();
storage.saveCredentials(credentials);
}
events.emit("nodes-started");
}

48
red/server.js
View File

@ -14,31 +14,22 @@
* limitations under the License.
**/
var fs = require('fs');
var util = require('util');
var createUI = require("./ui");
var redNodes = require("./nodes");
//TODO: relocated user dir
var flowfile = '';
var app = null;
var server = null;
function createServer(_server,settings) {
server = _server;
storage = require("./storage").init(settings);
app = createUI(settings);
flowfile = settings.flowFile || 'flows_'+require('os').hostname()+'.json';
//TODO: relocated user dir
fs.exists("lib/",function(exists) {
if (!exists) {
fs.mkdir("lib");
}
});
app.get("/nodes",function(req,res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write(redNodes.getNodeConfigs());
@ -46,15 +37,9 @@ function createServer(_server,settings) {
});
app.get("/flows",function(req,res) {
fs.exists(flowfile, function (exists) {
if (exists) {
res.sendfile(flowfile);
} else {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write("[]");
res.write(JSON.stringify(redNodes.getConfig()));
res.end();
}
});
});
app.post("/flows",function(req,res) {
@ -65,12 +50,11 @@ function createServer(_server,settings) {
req.on('end', function() {
res.writeHead(204, {'Content-Type': 'text/plain'});
res.end();
fs.writeFile(flowfile, fullBody, function(err) {
if(err) {
util.log(err);
} else {
redNodes.setConfig(JSON.parse(fullBody));
}
var flows = JSON.parse(fullBody);
storage.saveFlows(flows).then(function() {
redNodes.setConfig(flows);
}).otherwise(function(err) {
util.log("[red] Error saving flows : "+err);
});
});
});
@ -90,16 +74,12 @@ function start() {
util.log('or any other errors are resolved');
util.log("------------------------------------------");
fs.exists(flowfile, function (exists) {
if (exists) {
util.log("[red] Loading flows : "+flowfile);
fs.readFile(flowfile,'utf8',function(err,data) {
redNodes.setConfig(JSON.parse(data));
});
} else {
util.log("[red] Flows file not found : "+flowfile);
storage.getFlows().then(function(flows) {
if (flows.length > 0) {
redNodes.setConfig(flows);
}
}).otherwise(function(err) {
util.log("[red] Error loading flows : "+err);
});
}

33
red/storage/index.js Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright 2013 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
var settings;
var storage;
module.exports = {
init: function(_settings) {
settings = _settings;
var storageType = settings.storageModule || "localfilesystem";
storage = require("./"+storageType).init(settings);
return storage;
},
};
module.exports.__defineGetter__("storage", function() { return storage; });

360
red/storage/localfilesystem.js Normal file
View File

@ -0,0 +1,360 @@
/**
* Copyright 2013 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
var fs = require('fs');
var when = require('when');
var util = require('util');
var fspath = require("path");
var settings;
var flowsFile;
var credentialsFile;
var userDir;
var libDir;
var libFlowsDir;
// TODO: Make user data directory relocatable
function listFiles(dir) {
var dirs = {};
var files = [];
var dirCount = 0;
fs.readdirSync(dir).sort().filter(function(fn) {
var stats = fs.lstatSync(dir+"/"+fn);
if (stats.isDirectory()) {
dirCount += 1;
dirs[fn] = listFiles(dir+"/"+fn);
} else {
files.push(fn.split(".")[0]);
}
});
var result = {};
if (dirCount > 0) { result.d = dirs; }
if (files.length > 0) { result.f = files; }
return result;
}
function getFileMeta(root,path) {
var fn = fspath.join(root,path);
var fd = fs.openSync(fn,"r");
var size = fs.fstatSync(fd).size;
var meta = {};
var scanning = true;
var read = 0;
var length = 10;
var offset = 0;
var remaining = "";
var buffer = Buffer(length);
while(read < size) {
read+=fs.readSync(fd,buffer,0,length);
var data = remaining+buffer.toString();
var parts = data.split("\n");
remaining = parts.splice(-1);
for (var i=0;i<parts.length;i+=1) {
var match = /^\/\/ (\w+): (.*)/.exec(parts[i]);
if (match) {
meta[match[1]] = match[2];
} else {
read = size;
break;
}
}
}
fs.closeSync(fd);
return meta;
}
function getFileBody(root,path) {
var body = "";
var fn = fspath.join(root,path);
var fd = fs.openSync(fn,"r");
var size = fs.fstatSync(fd).size;
var scanning = true;
var read = 0;
var length = 50;
var offset = 0;
var remaining = "";
var buffer = Buffer(length);
while(read < size) {
var thisRead = fs.readSync(fd,buffer,0,length);
read += thisRead;
if (scanning) {
var data = remaining+buffer.toString();
var parts = data.split("\n");
remaining = parts.splice(-1)[0];
for (var i=0;i<parts.length;i+=1) {
if (! /^\/\/ \w+: /.test(parts[i])) {
scanning = false;
body += parts[i]+"\n";
}
}
if (! /^\/\/ \w+: /.test(remaining)) {
scanning = false;
}
if (!scanning) {
body += remaining;
}
} else {
body += buffer.slice(0,thisRead).toString();
}
}
fs.closeSync(fd);
return body;
}
function writeFile(root,path,meta,body,res) {
var fn = fspath.join(root,path);
var parts = path.split("/");
var dirname = root;
for (var i = 0;i<parts.length-1;i+=1) {
dirname = fspath.join(dirname,parts[i]);
if (!fs.existsSync(dirname)) {
fs.mkdirSync(dirname);
}
}
var headers = "";
for (var i in meta) {
headers += "// "+i+": "+meta[i]+"\n";
}
fs.writeFile(fn,headers+body,function(err) {
//TODO: handle error
res.writeHead(204, {'Content-Type': 'text/plain'});
res.end();
});
}
var localfilesystem = {
init: function(_settings) {
settings = _settings;
userDir = settings.userDir || process.env.NODE_RED_HOME;
if (settings.flowFile) {
flowsFile = settings.flowFile;
flowsFullPath = flowsFile;
} else {
flowsFile = 'flows_'+require('os').hostname()+'.json';
flowsFullPath = fspath.join(userDir,flowsFile);
}
credentialsFile = fspath.join(userDir,"credentials.json");
libDir = fspath.join(userDir,"lib");
libFlowsDir = fspath.join(libDir,"flows");
if (!fs.existsSync(libDir)) {
fs.mkdirSync(libDir);
}
if (!fs.existsSync(libFlowsDir)) {
fs.mkdirSync(libFlowsDir);
}
return this;
},
getFlows: function() {
var defer = when.defer();
fs.exists(flowsFullPath, function(exists) {
if (exists) {
util.log("[red] Loading flows : "+flowsFile);
fs.readFile(flowsFullPath,'utf8',function(err,data) {
if (err) {
defer.reject(err);
} else {
try {
defer.resolve(JSON.parse(data));
} catch(err) {
defer.reject(err);
}
}
});
} else {
util.log("[red] Flows file not found : "+flowsFile );
defer.resolve([]);
}
});
return defer.promise;
},
saveFlows: function(flows) {
var defer = when.defer();
fs.writeFile(flowsFullPath, JSON.stringify(flows), function(err) {
if(err) {
defer.reject(err);
} else {
defer.resolve();
}
});
return defer.promise;
},
getCredentials: function() {
var defer = when.defer();
fs.exists(credentialsFile, function(exists) {
if (exists) {
fs.readFile(credentialsFile,'utf8',function(err,data) {
if (err) {
defer.reject(err);
} else {
try {
defer.resolve(JSON.parse(data));
} catch(err) {
defer.reject(err);
}
}
});
} else {
defer.resolve({});
}
});
return defer.promise;
},
saveCredentials: function(credentials) {
var defer = when.defer();
fs.writeFile(credentialsFile, JSON.stringify(credentials), function(err) {
if(err) {
defer.reject(err);
} else {
defer.resolve();
}
});
return defer.promise;
},
getAllFlows: function() {
var defer = when.defer();
defer.resolve(listFiles(libFlowsDir));
return defer.promise;
},
getFlow: function(fn) {
var defer = when.defer();
var file = fspath.join(libFlowsDir,fn+".json");
fs.exists(file, function(exists) {
if (exists) {
fs.readFile(file,'utf8',function(err,data) {
if (err) {
defer.reject(err);
} else {
defer.resolve(data);
}
});
} else {
defer.reject();
}
});
return defer.promise;
},
saveFlow: function(fn,data) {
var defer = when.defer();
var file = fspath.join(libFlowsDir,fn+".json");
var parts = file.split("/");
for (var i = 0;i<parts.length;i+=1) {
var dirname = parts.slice(0,i).join("/");
if (dirname != "" ){
if (!fs.existsSync(dirname)) {
fs.mkdirSync(dirname);
}
}
}
fs.writeFile(file,data,function(err) {
if (err) {
defer.reject(err);
} else {
defer.resolve();
}
});
return defer.promise;
},
getLibraryEntry: function(type,path) {
var defer = when.defer();
var root = fspath.join(libDir,type);
if (!fs.existsSync(root)) {
fs.mkdirSync(root);
}
var rootPath = fspath.join(libDir,type,path);
fs.lstat(rootPath,function(err,stats) {
if (err) {
console.log(err);
defer.reject(err);
} else if (stats.isFile()) {
defer.resolve(getFileBody(root,path));
} else {
if (path.substr(-1) == '/') {
path = path.substr(0,path.length-1);
}
fs.readdir(rootPath,function(err,fns) {
var dirs = [];
var files = [];
fns.sort().filter(function(fn) {
var fullPath = fspath.join(path,fn);
var absoluteFullPath = fspath.join(root,fullPath);
if (fn[0] != ".") {
var stats = fs.lstatSync(absoluteFullPath);
if (stats.isDirectory()) {
dirs.push(fn);
} else {
var meta = getFileMeta(root,fullPath);
meta.fn = fn;
files.push(meta);
}
}
});
defer.resolve(dirs.concat(files));
});
}
});
return defer.promise;
},
saveLibraryEntry: function(type,path,meta,body) {
var defer = when.defer();
var root = fspath.join(libDir,type);
if (!fs.existsSync(root)) {
fs.mkdirSync(root);
}
var fn = fspath.join(root,path);
var parts = path.split("/");
var dirname = root;
for (var i=0;i<parts.length-1;i+=1) {
dirname = fspath.join(dirname,parts[i]);
if (!fs.existsSync(dirname)) {
fs.mkdirSync(dirname);
}
}
var headers = "";
for (var i in meta) {
headers += "// "+i+": "+meta[i]+"\n";
}
fs.writeFile(fn,headers+body,function(err) {
if (err) {
defer.reject(err);
} else {
defer.resolve();
}
});
return defer.promise;
}
};
module.exports = localfilesystem;

6
settings.js
View File

@ -16,6 +16,8 @@
module.exports = {
uiPort: 1880,
userDir: "/tmp/a",
// By default, the Node-RED UI accepts connections on all IPv4 interfaces.
// The following property can be used to listen on a specific interface. For
// example, the following would only allow connections from the local machine.
@ -28,6 +30,10 @@ module.exports = {
// The file containing the flows. If not set, it defaults to flows_<hostname>.json
//flowFile: 'flows.json',
// By default, all user data is stored in the Node-RED install directory. To
// use a different location, the following property can be used
//userDir: '/home/nol/.node-red/',
// You can protect the user interface with a userid and password by using the following property
// the password must be an md5 hash eg.. 5f4dcc3b5aa765d61d8327deb882cf99 ('password')
//httpAuth: {user:"user",pass:"5f4dcc3b5aa765d61d8327deb882cf99"},