Add support for file upload in /nodes api

package.json

@@ -71,6 +71,7 @@
         "raw-body": "2.4.1",
         "request": "2.88.0",
         "semver": "6.3.0",
+        "tar": "6.0.2",
         "uglify-js": "3.10.0",
         "when": "3.7.8",
         "ws": "6.2.1",

packages/node_modules/@node-red/editor-api/lib/admin/index.js

@@ -49,7 +49,14 @@ module.exports = {
 
         // Nodes
         adminApp.get("/nodes",needsPermission("nodes.read"),nodes.getAll,apiUtil.errorHandler);
-        adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);
+
+        if (!settings.editorTheme || !settings.editorTheme.palette || settings.editorTheme.palette.upload !== false) {
+            const multer  = require('multer');
+            const upload = multer({ storage: multer.memoryStorage() });
+            adminApp.post("/nodes",needsPermission("nodes.write"),upload.single("tarball"),nodes.post,apiUtil.errorHandler);
+        } else {
+            adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);
+        }
         adminApp.get(/^\/nodes\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalogs,apiUtil.errorHandler);
         adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+\/[^\/]+)\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalog,apiUtil.errorHandler);
         adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)$/,needsPermission("nodes.read"),nodes.getModule,apiUtil.errorHandler);

packages/node_modules/@node-red/editor-api/lib/admin/nodes.js

@@ -45,8 +45,18 @@ module.exports = {
             module: req.body.module,
             version: req.body.version,
             url: req.body.url,
+            tarball: undefined,
             req: apiUtils.getRequestLogObject(req)
         }
+        if (!runtimeAPI.settings.editorTheme || !runtimeAPI.settings.editorTheme.palette || runtimeAPI.settings.editorTheme.palette.upload !== false) {
+            if (req.file) {
+                opts.tarball = {
+                    name: req.file.originalname,
+                    size: req.file.size,
+                    buffer: req.file.buffer
+                }
+            }
+        }
         runtimeAPI.nodes.addModule(opts).then(function(info) {
             res.json(info);
         }).catch(function(err) {

packages/node_modules/@node-red/editor-api/package.json

@@ -26,6 +26,7 @@
         "express": "4.17.1",
         "memorystore": "1.6.2",
         "mime": "2.4.6",
+        "multer": "1.4.2",
         "mustache": "4.0.1",
         "oauth2orize": "1.11.0",
         "passport-http-bearer": "1.0.1",

packages/node_modules/@node-red/registry/lib/installer.js

@@ -16,7 +16,9 @@
 
 
 var path = require("path");
-var fs = require("fs");
+var os = require("os");
+var fs = require("fs-extra");
+var tar = require("tar");
 
 var registry = require("./registry");
 var library = require("./library");
@@ -30,9 +32,10 @@ var npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm';
 var paletteEditorEnabled = false;
 
 var settings;
-var moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
+const moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
-var slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
+const slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
-var pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
+const pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
+const localtgzRe = /^\/.+tgz$/;
 
 function init(runtime) {
     events = runtime.events;
@@ -45,12 +48,14 @@ var activePromise = Promise.resolve();
 
 function checkModulePath(folder) {
     var moduleName;
+    var moduleVersion;
     var err;
     var fullPath = path.resolve(folder);
     var packageFile = path.join(fullPath,'package.json');
     try {
         var pkg = require(packageFile);
         moduleName = pkg.name;
+        moduleVersion = pkg.version;
         if (!pkg['node-red']) {
             // TODO: nls
             err = new Error("Invalid Node-RED module");
@@ -62,7 +67,10 @@ function checkModulePath(folder) {
         err.code = 404;
         throw err;
     }
-    return moduleName;
+    return {
+        name: moduleName,
+        version: moduleVersion
+    };
 }
 
 function checkExistingModule(module,version) {
@@ -77,7 +85,11 @@ function checkExistingModule(module,version) {
     }
     return false;
 }
+
 function installModule(module,version,url) {
+    if (Buffer.isBuffer(module)) {
+        return installTarball(module)
+    }
     module = module || "";
     activePromise = activePromise.then(() => {
         //TODO: ensure module is 'safe'
@@ -86,7 +98,7 @@ function installModule(module,version,url) {
             var isUpgrade = false;
             try {
                 if (url) {
-                    if (pkgurlRe.test(url)) {
+                    if (pkgurlRe.test(url) || localtgzRe.test(url)) {
                         // Git remote url or Tarball url - check the valid package url
                         installName = url;
                     } else {
@@ -104,7 +116,8 @@ function installModule(module,version,url) {
                 } else if (slashRe.test(module)) {
                     // A path - check if there's a valid package.json
                     installName = module;
-                    module = checkModulePath(module);
+                    let info = checkModulePath(module);
+                    module = info.name;
                 } else {
                     log.warn(log._("server.install.install-failed-name",{name:module}));
                     e = new Error("Invalid module name");
@@ -168,7 +181,6 @@ function installModule(module,version,url) {
     return activePromise;
 }
 
-
 function reportAddedModules(info) {
     //comms.publish("node/added",info.nodes,false);
     if (info.nodes.length > 0) {
@@ -197,6 +209,93 @@ function reportRemovedModules(removedNodes) {
     return removedNodes;
 }
 
+async function getExistingPackageVersion(moduleName) {
+    try {
+        const packageFilename = path.join(settings.userDir || process.env.NODE_RED_HOME || "." , "package.json");
+        const pkg = await fs.readJson(packageFilename);
+        if (pkg.dependencies) {
+            return pkg.dependencies[moduleName];
+        }
+    } catch(err) {
+    }
+    return null;
+}
+
+async function installTarball(tarball) {
+    // Check this tarball contains a valid node-red module.
+    // Get its module name/version
+    const moduleInfo = await getTarballModuleInfo(tarball);
+
+    // Write the tarball to <userDir>/nodes/<filename.tgz>
+    // where the filename is the normalised form based on module name/version
+    let normalisedModuleName = moduleInfo.name[0] === '@'
+          ? moduleInfo.name.substr(1).replace(/\//g, '-')
+          : moduleInfo.name
+    const tarballFile = `${normalisedModuleName}-${moduleInfo.version}.tgz`;
+    let tarballPath = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes", tarballFile));
+
+    // (from fs-extra - move to writeFile with promise once Node 8 dropped)
+    await fs.outputFile(tarballPath, tarball);
+
+    // Next, need to check to see if this module is listed in `<userDir>/package.json`
+    let existingVersion = await getExistingPackageVersion(moduleInfo.name);
+    let existingFile = null;
+    let isUpdate = false;
+
+    // If this is a known module, need to check if there will be an old tarball
+    // to remove after the install of this one
+    if (existingVersion) {
+        // - Known module
+        if (/^file:nodes\//.test(existingVersion)) {
+            existingFile = existingVersion.substring(11);
+            isUpdate = true;
+            if (tarballFile === existingFile) {
+                // Edge case: a tar with the same name has bee uploaded.
+                // Carry on with the install, but don't remove the 'old' file
+                // as it will have been overwritten by the new one
+                existingFile = null;
+            }
+        }
+    }
+
+    // Install the tgz
+    return installModule(moduleInfo.name, moduleInfo.version, tarballPath).then(function(info) {
+        if (existingFile) {
+            // Remove the old file
+            return fs.remove(path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes",existingFile))).then(() => info).catch(() => info)
+        }
+        return info;
+    })
+}
+
+async function getTarballModuleInfo(tarball) {
+    const tarballDir = fs.mkdtempSync(path.join(os.tmpdir(),"nr-tarball-"));
+    const removeExtractedTar = function(done) {
+        fs.remove(tarballDir, err => {
+            done();
+        })
+    }
+    return new Promise((resolve,reject) => {
+        var writeStream = tar.x({
+            cwd: tarballDir
+        }).on('error', err => {
+            reject(err);
+        }).on('finish', () => {
+            try {
+                let moduleInfo = checkModulePath(path.join(tarballDir,"package"));
+                removeExtractedTar(err => {
+                    resolve(moduleInfo);
+                })
+            } catch(err) {
+                removeExtractedTar(() => {
+                    reject(err);
+                });
+            }
+        });
+        writeStream.end(tarball);
+    });
+}
+
 function uninstallModule(module) {
     activePromise = activePromise.then(() => {
         return new Promise((resolve,reject) => {
@@ -271,6 +370,7 @@ function checkPrereq() {
         })
     }
 }
+
 module.exports = {
     init: init,
     checkPrereq: checkPrereq,

packages/node_modules/@node-red/registry/package.json

@@ -18,6 +18,7 @@
     "dependencies": {
         "@node-red/util": "1.2.0-alpha.1",
         "semver": "6.3.0",
+        "tar": "6.0.2",
         "uglify-js": "3.10.0",
         "when": "3.7.8"
     }

packages/node_modules/@node-red/runtime/lib/api/nodes.js

@@ -159,6 +159,7 @@ var api = module.exports = {
     * @param {User} opts.user - the user calling the api
     * @param {String} opts.module - the id of the module to install
     * @param {String} opts.version - (optional) the version of the module to install
+    * @param {Object} opts.tarball - (optional) a tarball file to install. Object has properties `name`, `size` and `buffer`.
     * @param {String} opts.url - (optional) url to install
     * @param {Object} opts.req - the request to log (optional)
     * @return {Promise<ModuleInfo>} - the node module info
@@ -173,6 +174,37 @@ var api = module.exports = {
                 err.status = 400;
                 return reject(err);
             }
+            if (opts.tarball) {
+                if (runtime.settings.editorTheme && runtime.settings.editorTheme.palette && runtime.settings.editorTheme.palette.upload === false) {
+                    runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,error:"invalid_request"}, opts.req);
+                    var err = new Error("Invalid request");
+                    err.code = "invalid_request";
+                    err.status = 400;
+                    return reject(err);
+                }
+                if (opts.module || opts.version || opts.url) {
+                    runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:opts.module,error:"invalid_request"}, opts.req);
+                    var err = new Error("Invalid request");
+                    err.code = "invalid_request";
+                    err.status = 400;
+                    return reject(err);
+                }
+                runtime.nodes.installModule(opts.tarball.buffer).then(function(info) {
+                    runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:info.id}, opts.req);
+                    return resolve(info);
+                }).catch(function(err) {
+
+                    if (err.code) {
+                        err.status = 400;
+                        runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code}, opts.req);
+                    } else {
+                        err.status = 400;
+                        runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code||"unexpected_error",message:err.toString()}, opts.req);
+                    }
+                    return reject(err);
+                })
+                return;
+            }
             if (opts.module) {
                 var existingModule = runtime.nodes.getModuleInfo(opts.module);
                 if (existingModule) {

packages/node_modules/@node-red/runtime/lib/nodes/index.js

@@ -151,11 +151,9 @@ function reportNodeStateChange(info,enabled) {
 }
 
 function installModule(module,version,url) {
-    var existingModule = registry.getModuleInfo(module);
-    var isUpgrade = !!existingModule;
     return registry.installModule(module,version,url).then(function(info) {
-        if (isUpgrade) {
+        if (info.pending_version) {
-            events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:module,version:version}});
+            events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:info.name,version:info.pending_version}});
         } else {
             events.emit("runtime-event",{id:"node/added",retain:false,payload:info.nodes});
         }

test/unit/@node-red/editor-api/lib/admin/nodes_spec.js

@@ -53,6 +53,7 @@ describe("api/admin/nodes", function() {
     describe('get nodes', function() {
         it('returns node list', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeList: function() {
                         return Promise.resolve([1,2,3]);
@@ -75,6 +76,7 @@ describe("api/admin/nodes", function() {
 
         it('returns node configs', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeConfigs: function() {
                         return Promise.resolve("<script></script>");
@@ -99,6 +101,7 @@ describe("api/admin/nodes", function() {
 
         it('returns node module info', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleInfo: function(opts) {
                         return Promise.resolve({"node-red":{name:"node-red"}}[opts.module]);
@@ -119,6 +122,7 @@ describe("api/admin/nodes", function() {
 
         it('returns 404 for unknown module', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleInfo: function(opts) {
                         var errInstance = new Error("Not Found");
@@ -143,6 +147,7 @@ describe("api/admin/nodes", function() {
 
         it('returns individual node info', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeInfo: function(opts) {
                         return Promise.resolve({"node-red/123":{id:"node-red/123"}}[opts.id]);
@@ -164,6 +169,7 @@ describe("api/admin/nodes", function() {
 
         it('returns individual node configs', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeConfig: function(opts) {
                         return Promise.resolve({"node-red/123":"<script></script>"}[opts.id]);
@@ -187,6 +193,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns 404 for unknown node', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeInfo: function(opts) {
                         var errInstance = new Error("Not Found");
@@ -215,6 +222,7 @@ describe("api/admin/nodes", function() {
         it('installs the module and returns module info', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     addModule: function(_opts) {
                         opts = _opts;
@@ -244,6 +252,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns error', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     addModule: function(opts) {
                         var errInstance = new Error("Message");
@@ -272,6 +281,7 @@ describe("api/admin/nodes", function() {
         it('uninstalls the module', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     removeModule: function(_opts) {
                         opts = _opts;
@@ -292,6 +302,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns error', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     removeModule: function(opts) {
                         var errInstance = new Error("Message");
@@ -319,6 +330,7 @@ describe("api/admin/nodes", function() {
     describe('enable/disable node set', function() {
         it('returns 400 for invalid request payload', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     setNodeSetState: function(opts) {return Promise.resolve()}
                 }
@@ -340,6 +352,7 @@ describe("api/admin/nodes", function() {
         it('sets node state and returns node info', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     setNodeSetState: function(_opts) {
                         opts = _opts;
@@ -368,6 +381,7 @@ describe("api/admin/nodes", function() {
     describe('enable/disable module' ,function() {
         it('returns 400 for invalid request payload', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     setModuleState: function(opts) {return Promise.resolve()}
                 }
@@ -388,6 +402,7 @@ describe("api/admin/nodes", function() {
         it('sets module state and returns module info', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     setModuleState: function(_opts) {
                         opts = _opts;
@@ -416,6 +431,7 @@ describe("api/admin/nodes", function() {
     describe('get icons', function() {
         it('returns icon list', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getIconList: function() {
                         return Promise.resolve({module:[1,2,3]});
@@ -440,6 +456,7 @@ describe("api/admin/nodes", function() {
     describe('get module messages', function() {
         it('returns message catalog', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleCatalog: function(opts) {
                         return Promise.resolve({a:123});
@@ -459,6 +476,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns all node catalogs', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleCatalogs: function(opts) {
                         return Promise.resolve({a:1});

test/unit/@node-red/registry/lib/installer_spec.js

@@ -18,7 +18,7 @@ var should = require("should");
 var sinon = require("sinon");
 var when = require("when");
 var path = require("path");
-var fs = require('fs');
+var fs = require('fs-extra');
 var EventEmitter = require('events');
 
 var NR_TEST_UTILS = require("nr-test-utils");
@@ -36,7 +36,7 @@ describe('nodes/registry/installer', function() {
         warn: sinon.stub(),
         info: sinon.stub(),
         metric: sinon.stub(),
-        _: function() { return "abc"}
+        _: function(msg) { return msg }
     }
 
     beforeEach(function() {
@@ -70,8 +70,8 @@ describe('nodes/registry/installer', function() {
             typeRegistry.getModuleInfo.restore();
         }
 
-        if (require('fs').statSync.restore) {
+        if (fs.statSync.restore) {
-            require('fs').statSync.restore();
+            fs.statSync.restore();
         }
 
     });