From 95bef6b6ca62f568fa4dc76483532d3d6d6dc395 Mon Sep 17 00:00:00 2001 From: Nicholas O'Leary Date: Sun, 10 Nov 2013 00:05:58 +0000 Subject: [PATCH] Abstract all file-system operations Stage 1 of #62 --- package.json | 1 + red/library.js | 237 ++++------------------ red/nodes.js | 37 ++-- red/server.js | 52 ++--- red/storage/index.js | 33 +++ red/storage/localfilesystem.js | 360 +++++++++++++++++++++++++++++++++ settings.js | 6 + 7 files changed, 480 insertions(+), 246 deletions(-) create mode 100644 red/storage/index.js create mode 100644 red/storage/localfilesystem.js diff --git a/package.json b/package.json index c257405b2..bc53b7092 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ ], "dependencies": { "express": "3.x", + "when": "~2.6.0", "mqtt": "~0.3.3", "ws": "~0.4.31", "mustache": "~0.7.2", diff --git a/red/library.js b/red/library.js index 99b9d5698..9eefa0bce 100644 --- a/red/library.js +++ b/red/library.js @@ -14,78 +14,49 @@ * limitations under the License. **/ -var fs = require("fs"); -var fspath = require("path"); var redApp = null; +var storage = null; function init() { redApp = require("./server").app; + storage = require("./storage").storage; + // -------- Flow Library -------- redApp.post(new RegExp("/library/flows\/(.*)"), function(req,res) { var fullBody = ''; req.on('data', function(chunk) { - fullBody += chunk.toString(); + fullBody += chunk.toString(); }); req.on('end', function() { - var fn = "lib/flows/"+req.params[0]+".json"; - var parts = fn.split("/"); - for (var i = 3;i 0) { result.d = dirs; } - if (files.length > 0) { result.f = files; } - return result; - } - redApp.get("/library/flows",function(req,res) { - var flows = {}; - if (fs.existsSync("lib/flows")) { - flows = listFiles("lib/flows"); - } else { - fs.mkdirSync("lib/flows"); - } - res.writeHead(200, {'Content-Type': 'text/plain'}); - res.write(JSON.stringify(flows)); - res.end(); - + storage.getAllFlows().then(function(flows) { + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.write(JSON.stringify(flows)); + res.end(); + }); }); redApp.get(new RegExp("/library/flows\/(.*)"), function(req,res) { - var fn = "lib/flows/"+req.params[0]+".json"; - if (fs.existsSync(fn)) { - fs.readFile(fn,function(err,data) { - res.writeHead(200, {'Content-Type': 'text/plain'}); - res.write(data); - res.end(); - }); - } else { + storage.getFlow(req.params[0]).then(function(data) { + res.writeHead(200, {'Content-Type': 'text/plain'}); + res.write(data); + res.end(); + }).otherwise(function(err) { + if (err) { + util.log("[red] Error loading flow '"+req.params[0]+"' : "+err); + } res.send(404); - } + }); }); // ------------------------------ @@ -93,53 +64,21 @@ function init() { function createLibrary(type) { - //TODO: use settings.library as the base root - - var root = fspath.join("lib",type)+"/"; - - fs.exists(root,function(exists) { - if (!exists) { - fs.mkdir(root); - } - }); redApp.get(new RegExp("/library/"+type+"($|\/(.*))"),function(req,res) { var path = req.params[1]||""; - var rootPath = fspath.join(root,path); - - fs.exists(rootPath,function(exists) { - fs.lstat(rootPath,function(err,stats) { - if (stats.isFile()) { - getFileBody(root,path,res); - } else { - if (path.substr(-1) == '/') { - path = path.substr(0,path.length-1); - } - fs.readdir(rootPath,function(err,fns) { - var dirs = []; - var files = []; - - fns.sort().filter(function(fn) { - var fullPath = fspath.join(path,fn); - var absoluteFullPath = fspath.join(root,fullPath); - if (fn[0] != ".") { - var stats = fs.lstatSync(absoluteFullPath); - if (stats.isDirectory()) { - dirs.push(fn); - } else { - var meta = getFileMeta(root,fullPath); - meta.fn = fn; - files.push(meta); - } - } - - }); - res.writeHead(200, {'Content-Type': 'text/plain'}); - res.write(JSON.stringify(dirs.concat(files))); - res.end(); - }); - } - - }); + storage.getLibraryEntry(type,path).then(function(result) { + res.writeHead(200, {'Content-Type': 'text/plain'}); + if (typeof result === "string") { + res.write(result); + } else { + res.write(JSON.stringify(result)); + } + res.end(); + }).otherwise(function(err) { + if (err) { + util.log("[red] Error loading library entry '"+path+"' : "+err); + } + res.send(404); }); }); @@ -150,103 +89,15 @@ function createLibrary(type) { fullBody += chunk.toString(); }); req.on('end', function() { - writeFile(root,path,req.query,fullBody,res); + storage.saveLibraryEntry(type,path,req.query,fullBody).then(function() { + res.send(204); + }).otherwise(function(err) { + util.log("[red] Error saving library entry '"+path+"' : "+err); + res.send(500); + });; }); }); } - -function writeFile(root,path,meta,body,res) { - var fn = fspath.join(root,path); - - var parts = path.split("/"); - var dirname = root; - for (var i = 0;i 0) { + redNodes.setConfig(flows); } + }).otherwise(function(err) { + util.log("[red] Error loading flows : "+err); }); } diff --git a/red/storage/index.js b/red/storage/index.js new file mode 100644 index 000000000..e30e61d94 --- /dev/null +++ b/red/storage/index.js @@ -0,0 +1,33 @@ +/** + * Copyright 2013 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + + +var settings; +var storage; + +module.exports = { + init: function(_settings) { + settings = _settings; + + var storageType = settings.storageModule || "localfilesystem"; + + storage = require("./"+storageType).init(settings); + return storage; + }, +}; + +module.exports.__defineGetter__("storage", function() { return storage; }); + diff --git a/red/storage/localfilesystem.js b/red/storage/localfilesystem.js new file mode 100644 index 000000000..6b671be59 --- /dev/null +++ b/red/storage/localfilesystem.js @@ -0,0 +1,360 @@ +/** + * Copyright 2013 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +var fs = require('fs'); +var when = require('when'); +var util = require('util'); +var fspath = require("path"); + +var settings; +var flowsFile; +var credentialsFile; + +var userDir; +var libDir; +var libFlowsDir; + +// TODO: Make user data directory relocatable + +function listFiles(dir) { + var dirs = {}; + var files = []; + var dirCount = 0; + fs.readdirSync(dir).sort().filter(function(fn) { + var stats = fs.lstatSync(dir+"/"+fn); + if (stats.isDirectory()) { + dirCount += 1; + dirs[fn] = listFiles(dir+"/"+fn); + } else { + files.push(fn.split(".")[0]); + } + }); + var result = {}; + if (dirCount > 0) { result.d = dirs; } + if (files.length > 0) { result.f = files; } + return result; +} + +function getFileMeta(root,path) { + var fn = fspath.join(root,path); + + var fd = fs.openSync(fn,"r"); + var size = fs.fstatSync(fd).size; + var meta = {}; + var scanning = true; + var read = 0; + var length = 10; + var offset = 0; + var remaining = ""; + var buffer = Buffer(length); + while(read < size) { + read+=fs.readSync(fd,buffer,0,length); + var data = remaining+buffer.toString(); + var parts = data.split("\n"); + remaining = parts.splice(-1); + for (var i=0;i.json //flowFile: 'flows.json', + // By default, all user data is stored in the Node-RED install directory. To + // use a different location, the following property can be used + //userDir: '/home/nol/.node-red/', + // You can protect the user interface with a userid and password by using the following property // the password must be an md5 hash eg.. 5f4dcc3b5aa765d61d8327deb882cf99 ('password') //httpAuth: {user:"user",pass:"5f4dcc3b5aa765d61d8327deb882cf99"},