Merge branch 'dev' of github.com:node-red/node-red into dev

package.json

@@ -71,6 +71,7 @@
         "raw-body": "2.4.1",
         "request": "2.88.0",
         "semver": "6.3.0",
+        "tar": "6.0.2",
         "uglify-js": "3.10.0",
         "when": "3.7.8",
         "ws": "6.2.1",

packages/node_modules/@node-red/editor-api/lib/admin/index.js

@@ -49,7 +49,14 @@ module.exports = {
 
         // Nodes
         adminApp.get("/nodes",needsPermission("nodes.read"),nodes.getAll,apiUtil.errorHandler);
+
+        if (!settings.editorTheme || !settings.editorTheme.palette || settings.editorTheme.palette.upload !== false) {
+            const multer  = require('multer');
+            const upload = multer({ storage: multer.memoryStorage() });
+            adminApp.post("/nodes",needsPermission("nodes.write"),upload.single("tarball"),nodes.post,apiUtil.errorHandler);
+        } else {
             adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);
+        }
         adminApp.get(/^\/nodes\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalogs,apiUtil.errorHandler);
         adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+\/[^\/]+)\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalog,apiUtil.errorHandler);
         adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)$/,needsPermission("nodes.read"),nodes.getModule,apiUtil.errorHandler);

packages/node_modules/@node-red/editor-api/lib/admin/nodes.js

@@ -45,8 +45,18 @@ module.exports = {
             module: req.body.module,
             version: req.body.version,
             url: req.body.url,
+            tarball: undefined,
             req: apiUtils.getRequestLogObject(req)
         }
+        if (!runtimeAPI.settings.editorTheme || !runtimeAPI.settings.editorTheme.palette || runtimeAPI.settings.editorTheme.palette.upload !== false) {
+            if (req.file) {
+                opts.tarball = {
+                    name: req.file.originalname,
+                    size: req.file.size,
+                    buffer: req.file.buffer
+                }
+            }
+        }
         runtimeAPI.nodes.addModule(opts).then(function(info) {
             res.json(info);
         }).catch(function(err) {

packages/node_modules/@node-red/editor-api/package.json

@@ -26,6 +26,7 @@
         "express": "4.17.1",
         "memorystore": "1.6.2",
         "mime": "2.4.6",
+        "multer": "1.4.2",
         "mustache": "4.0.1",
         "oauth2orize": "1.11.0",
         "passport-http-bearer": "1.0.1",

packages/node_modules/@node-red/editor-client/locales/en-US/editor.json

@@ -22,7 +22,8 @@
             "color": "Color",
             "position": "Position",
             "enable": "Enable",
-            "disable": "Disable"
+            "disable": "Disable",
+            "upload": "Upload"
         },
         "type": {
             "string": "string",
@@ -532,6 +533,7 @@
             "sortAZ": "a-z",
             "sortRecent": "recent",
             "more": "+ __count__ more",
+            "upload": "Upload module tgz file",
             "errors": {
                 "catalogLoadFailed": "<p>Failed to load node catalogue.</p><p>Check the browser console for more information</p>",
                 "installFailed": "<p>Failed to install: __module__</p><p>__message__</p><p>Check the log for more information</p>",

packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js

@@ -29,7 +29,7 @@ RED.tabs = (function() {
         var currentTabWidth;
         var currentActiveTabWidth = 0;
         var collapsibleMenu;
-
+        var preferredOrder = options.order;
         var ul = options.element || $("#"+options.id);
         var wrapper = ul.wrap( "<div>" ).parent();
         var scrollContainer = ul.wrap( "<div>" ).parent();
@@ -132,11 +132,11 @@ RED.tabs = (function() {
                                     activateTab(id);
                                 }
                             };
-                            if (tabs[id].pinned) {
+                            // if (tabs[id].pinned) {
-                                pinnedOptions.push(opt);
+                            //     pinnedOptions.push(opt);
-                            } else {
+                            // } else {
                                 options.push(opt);
-                            }
+                            // }
                         });
                         options = pinnedOptions.concat(options);
                         collapsibleMenu = RED.menu.init({options: options});
@@ -363,23 +363,39 @@ RED.tabs = (function() {
             var tabWidth;
 
             if (options.collapsible) {
+                var availableCount = collapsedButtonsRow.children().length;
+                var visibleCount = collapsedButtonsRow.children(":visible").length;
                 tabWidth = width - collapsedButtonsRow.width()-10;
-                if (tabWidth < 198) {
+                var maxTabWidth = 198;
-                    var delta = 198 - tabWidth;
+                var minTabWidth = 80;
+                if (tabWidth <= minTabWidth || (tabWidth < maxTabWidth && visibleCount > 5)) {
+                    // The tab is too small. Hide the next button to make room
+                    // Start at the end of the button row, -1 for the menu button
                     var b = collapsedButtonsRow.find("a:last").prev();
+                    var index = collapsedButtonsRow.children().length - 2;
+                    // Work backwards to find the first visible button
                     while (b.is(":not(:visible)")) {
                         b = b.prev();
+                        index--;
                     }
-                    if (!b.hasClass("red-ui-tab-link-button-pinned")) {
+                    // If it isn't a pinned button, hide it to get the room
+                    if (tabWidth <= minTabWidth || visibleCount>6) {//}!b.hasClass("red-ui-tab-link-button-pinned")) {
                         b.hide();
                     }
-                    tabWidth = width - collapsedButtonsRow.width()-10;
+                    tabWidth = Math.max(minTabWidth,width - collapsedButtonsRow.width()-10);
                 } else {
-                    var space = width - 198 - collapsedButtonsRow.width();
+                    if (visibleCount !== availableCount) {
+                        if (visibleCount < 6) {
+                            tabWidth = minTabWidth;
+                        } else {
+                            tabWidth = maxTabWidth;
+                        }
+                    }
+                    var space = width - tabWidth - collapsedButtonsRow.width();
                     if (space > 40) {
                         collapsedButtonsRow.find("a:not(:visible):first").show();
-                        tabWidth = width - collapsedButtonsRow.width()-10;
                     }
+                    tabWidth = width - collapsedButtonsRow.width()-10;
                 }
                 tabs.css({width:tabWidth});
 
@@ -469,7 +485,7 @@ RED.tabs = (function() {
             }
         }
 
-        return {
+        var tabAPI =  {
             addTab: function(tab,targetIndex) {
                 if (options.onselect) {
                     var selection = ul.find("li.red-ui-tab.selected");
@@ -531,11 +547,92 @@ RED.tabs = (function() {
                         evt.preventDefault();
                         activateTab(tab.id);
                     });
+                    pinnedLink.data("tabId",tab.id)
                     if (tab.pinned) {
                         pinnedLink.addClass("red-ui-tab-link-button-pinned");
                         pinnedTabsCount++;
                     }
                     RED.popover.tooltip($(pinnedLink), tab.name, tab.action);
+                    if (options.onreorder) {
+                        var pinnedLinkIndex;
+                        var pinnedLinks = [];
+                        var startPinnedIndex;
+                        pinnedLink.draggable({
+                            distance: 10,
+                            axis:"x",
+                            containment: ".red-ui-tab-link-buttons",
+                            start: function(event,ui) {
+                                dragActive = true;
+                                $(".red-ui-tab-link-buttons").width($(".red-ui-tab-link-buttons").width());
+                                if (dblClickArmed) { dblClickArmed = false; return false }
+                                collapsedButtonsRow.children().each(function(i) {
+                                    pinnedLinks[i] = {
+                                        el:$(this),
+                                        text: $(this).text(),
+                                        left: $(this).position().left,
+                                        width: $(this).width(),
+                                        menu: $(this).hasClass("red-ui-tab-link-button-menu")
+                                    };
+                                    if ($(this).is(pinnedLink)) {
+                                        pinnedLinkIndex = i;
+                                        startPinnedIndex = i;
+                                    }
+                                });
+                                collapsedButtonsRow.children().each(function(i) {
+                                    if (i!==pinnedLinkIndex) {
+                                        $(this).css({
+                                            position: 'absolute',
+                                            left: pinnedLinks[i].left+"px",
+                                            width: pinnedLinks[i].width+2,
+                                            transition: "left 0.3s"
+                                        });
+                                    }
+                                })
+                                if (!pinnedLink.hasClass('active')) {
+                                    pinnedLink.css({'zIndex':1});
+                                }
+                            },
+                            drag: function(event,ui) {
+                                ui.position.left += pinnedLinks[pinnedLinkIndex].left;
+                                var tabCenter = ui.position.left + pinnedLinks[pinnedLinkIndex].width/2;
+                                for (var i=0;i<pinnedLinks.length;i++) {
+                                    if (i === pinnedLinkIndex || pinnedLinks[i].menu || pinnedLinks[i].el.is(":not(:visible)")) {
+                                        continue;
+                                    }
+                                    if (tabCenter > pinnedLinks[i].left && tabCenter < pinnedLinks[i].left+pinnedLinks[i].width) {
+                                        if (i < pinnedLinkIndex) {
+                                            pinnedLinks[i].left += pinnedLinks[pinnedLinkIndex].width+8;
+                                            pinnedLinks[pinnedLinkIndex].el.detach().insertBefore(pinnedLinks[i].el);
+                                        } else {
+                                            pinnedLinks[i].left -= pinnedLinks[pinnedLinkIndex].width+8;
+                                            pinnedLinks[pinnedLinkIndex].el.detach().insertAfter(pinnedLinks[i].el);
+                                        }
+                                        pinnedLinks[i].el.css({left:pinnedLinks[i].left+"px"});
+
+                                        pinnedLinks.splice(i, 0, pinnedLinks.splice(pinnedLinkIndex, 1)[0]);
+
+                                        pinnedLinkIndex = i;
+                                        break;
+                                    }
+                                }
+                            },
+                            stop: function(event,ui) {
+                                collapsedButtonsRow.children().css({position:"relative",left:"",transition:""});
+                                $(".red-ui-tab-link-buttons").width('auto');
+                                pinnedLink.css({zIndex:""});
+                                updateTabWidths();
+                                if (startPinnedIndex !== pinnedLinkIndex) {
+                                    if (collapsibleMenu) {
+                                        collapsibleMenu.remove();
+                                        collapsibleMenu = null;
+                                    }
+                                    var newOrder = $.makeArray(collapsedButtonsRow.children().map(function() { return $(this).data('tabId');}));
+                                    tabAPI.order(newOrder);
+                                    options.onreorder(newOrder);
+                                }
+                            }
+                        });
+                    }
 
                 }
                 link.on("mouseup",onTabClick);
@@ -565,7 +662,7 @@ RED.tabs = (function() {
                 if (ul.find("li.red-ui-tab").length == 1) {
                     activateTab(link);
                 }
-                if (options.onreorder) {
+                if (options.onreorder && !options.collapsible) {
                     var originalTabOrder;
                     var tabDragIndex;
                     var tabElements = [];
@@ -652,6 +749,9 @@ RED.tabs = (function() {
                     collapsibleMenu.remove();
                     collapsibleMenu = null;
                 }
+                if (preferredOrder) {
+                    tabAPI.order(preferredOrder);
+                }
             },
             removeTab: removeTab,
             activateTab: activateTab,
@@ -673,10 +773,8 @@ RED.tabs = (function() {
             },
             selection: getSelection,
             order: function(order) {
+                preferredOrder = order;
                 var existingTabOrder = $.makeArray(ul.children().map(function() { return $(this).data('tabId');}));
-                if (existingTabOrder.length !== order.length) {
-                    return
-                }
                 var i;
                 var match = true;
                 for (i=0;i<order.length;i++) {
@@ -692,13 +790,42 @@ RED.tabs = (function() {
                 var existingTabs = ul.children().detach().each(function() {
                     existingTabMap[$(this).data("tabId")] = $(this);
                 });
+                var pinnedButtons = {};
+                if (options.collapsible) {
+                    collapsedButtonsRow.children().detach().each(function() {
+                        var id = $(this).data("tabId");
+                        if (!id) {
+                            id = "__menu__"
+                        }
+                        pinnedButtons[id] = $(this);
+                    });
+                }
                 for (i=0;i<order.length;i++) {
+                    if (existingTabMap[order[i]]) {
                         existingTabMap[order[i]].appendTo(ul);
+                        if (options.collapsible) {
+                            pinnedButtons[order[i]].appendTo(collapsedButtonsRow);
+                        }
+                        delete existingTabMap[order[i]];
                     }
                 }
-
+                // Add any tabs that aren't known in the order
+                for (i in existingTabMap) {
+                    if (existingTabMap.hasOwnProperty(i)) {
+                        existingTabMap[i].appendTo(ul);
+                        if (options.collapsible) {
+                            pinnedButtons[i].appendTo(collapsedButtonsRow);
                         }
                     }
+                }
+                if (options.collapsible) {
+                    pinnedButtons["__menu__"].appendTo(collapsedButtonsRow);
+                    updateTabWidths();
+                }
+            }
+        }
+        return tabAPI;
+    }
 
     return {
         create: createTabs

packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js

@@ -542,8 +542,6 @@ RED.palette.editor = (function() {
         return settingsPane;
     }
 
-
-
     function createSettingsPane() {
         settingsPane = $('<div id="red-ui-settings-tab-palette"></div>');
         var content = $('<div id="red-ui-palette-editor">'+
@@ -574,7 +572,11 @@ RED.palette.editor = (function() {
             minimumActiveTabWidth: 110
         });
 
+        createNodeTab(content);
+        createInstallTab(content);
+    }
 
+    function createNodeTab(content) {
         var modulesTab = $('<div>',{class:"red-ui-palette-editor-tab"}).appendTo(content);
 
         editorTabs.addTab({
@@ -726,9 +728,9 @@ RED.palette.editor = (function() {
                 }
             }
         });
+    }
 
-
+    function createInstallTab(content) {
-
         var installTab = $('<div>',{class:"red-ui-palette-editor-tab hide"}).appendTo(content);
 
         editorTabs.addTab({
@@ -761,7 +763,6 @@ RED.palette.editor = (function() {
                 }
             });
 
-
         $('<span>').text(RED._("palette.editor.sort")+' ').appendTo(toolBar);
         var sortGroup = $('<span class="button-group"></span>').appendTo(toolBar);
         var sortRelevance = $('<a href="#" class="red-ui-palette-editor-install-sort-option red-ui-sidebar-header-button-toggle selected"><i class="fa fa-sort-amount-desc"></i></a>').appendTo(sortGroup);
@@ -795,6 +796,7 @@ RED.palette.editor = (function() {
             loadedIndex = {};
             initInstallTab();
         })
+        RED.popover.tooltip(refreshButton,"NLS-TODO: Refresh module list");
 
         packageList = $('<ol>',{style:"position: absolute;top: 79px;bottom: 0;left: 0;right: 0px;"}).appendTo(installTab).editableList({
             addButton: false,
@@ -878,8 +880,87 @@ RED.palette.editor = (function() {
             }
         });
 
+        if (RED.settings.theme('palette.upload') !== false) {
+            var uploadSpan = $('<span class="button-group">').prependTo(toolBar);
+            var uploadButton = $('<button type="button" class="red-ui-sidebar-header-button red-ui-palette-editor-upload-button"><label><i class="fa fa-upload"></i><form id="red-ui-palette-editor-upload-form" enctype="multipart/form-data"><input name="tarball" type="file" accept=".tgz"></label></button>').appendTo(uploadSpan);
+
+            var uploadInput = uploadButton.find('input[type="file"]');
+            uploadInput.on("change", function(evt) {
+                if (this.files.length > 0) {
+                    uploadFilenameLabel.text(this.files[0].name)
+                    uploadToolbar.slideDown(200);
+                }
+            })
+
+            var uploadToolbar = $('<div class="red-ui-palette-editor-upload"></div>').appendTo(installTab);
+            var uploadForm = $('<div>').appendTo(uploadToolbar);
+            var uploadFilename = $('<div class="placeholder-input"><i class="fa fa-upload"></i> </div>').appendTo(uploadForm);
+            var uploadFilenameLabel = $('<span></span>').appendTo(uploadFilename);
+            var uploadButtons = $('<div class="red-ui-palette-editor-upload-buttons"></div>').appendTo(uploadForm);
+            $('<button class="editor-button"></button>').text(RED._("common.label.cancel")).appendTo(uploadButtons).on("click", function(evt) {
+                evt.preventDefault();
+                uploadToolbar.slideUp(200);
+                uploadInput.val("");
+            });
+            $('<button class="editor-button primary"></button>').text(RED._("common.label.upload")).appendTo(uploadButtons).on("click", function(evt) {
+                evt.preventDefault();
+
+                var spinner = RED.utils.addSpinnerOverlay(uploadToolbar, true);
+                var buttonRow = $('<div style="position: relative;bottom: calc(50% + 17px); padding-right: 10px;text-align: right;"></div>').appendTo(spinner);
+                $('<button class="red-ui-button"></button>').text(RED._("eventLog.view")).appendTo(buttonRow).on("click", function(evt) {
+                    evt.preventDefault();
+                    RED.actions.invoke("core:show-event-log");
+                });
+                RED.eventLog.startEvent(RED._("palette.editor.confirm.button.install")+" : "+uploadInput[0].files[0].name);
+
+                var data = new FormData();
+                data.append("tarball",uploadInput[0].files[0]);
+                var filename = uploadInput[0].files[0].name;
+                $.ajax({
+                    url: 'nodes',
+                    data: data,
+                    cache: false,
+                    contentType: false,
+                    processData: false,
+                    method: 'POST',
+                }).always(function(data,textStatus,xhr) {
+                    spinner.remove();
+                    uploadInput.val("");
+                    uploadToolbar.slideUp(200);
+                }).fail(function(xhr,textStatus,err) {
+                    var message = textStatus;
+                    if (xhr.responseJSON) {
+                        message = xhr.responseJSON.message;
+                    }
+                    var notification = RED.notify(RED._('palette.editor.errors.installFailed',{module: filename,message:message}),{
+                        type: 'error',
+                        modal: true,
+                        fixed: true,
+                        buttons: [
+                            {
+                                text: RED._("common.label.close"),
+                                click: function() {
+                                    notification.close();
+                                }
+                            },{
+                                text: RED._("eventLog.view"),
+                                click: function() {
+                                    notification.close();
+                                    RED.actions.invoke("core:show-event-log");
+                                }
+                            }
+                        ]
+                    });
+                    uploadInput.val("");
+                    uploadToolbar.slideUp(200);
+                })
+            })
+            RED.popover.tooltip(uploadButton,RED._("palette.editor.upload"));
+        }
+
         $('<div id="red-ui-palette-module-install-shade" class="red-ui-palette-module-shade hide"><div class="red-ui-palette-module-shade-status"></div><img src="red/images/spin.svg" class="red-ui-palette-spinner"/></div>').appendTo(installTab);
     }
+
     function update(entry,version,url,container,done) {
         if (RED.settings.theme('palette.editable') === false) {
             done(new Error('Palette not editable'));

packages/node_modules/@node-red/editor-client/src/js/ui/sidebar.js

@@ -232,7 +232,11 @@ RED.sidebar = (function() {
                 }
             },
             // minimumActiveTabWidth: 70,
-            collapsible: true
+            collapsible: true,
+            onreorder: function(order) {
+                RED.settings.set("editor.sidebar.order",order);
+            },
+            order: RED.settings.get("editor.sidebar.order",["info", "help", "version-control", "debug"])
             // scrollable: true
         });
 

packages/node_modules/@node-red/editor-client/src/sass/palette-editor.scss

@@ -237,3 +237,42 @@ ul.red-ui-palette-module-error-list {
 #red-ui-palette-module-install-shade {
     padding-top: 80px;
 }
+button.red-ui-palette-editor-upload-button {
+    padding: 0;
+    height: 25px;
+    margin-top: -1px;
+
+    input[type="file"] {
+        opacity: 0;
+        margin: 0;
+        height: 0;
+        width: 0;
+    }
+    .red-ui-settings-tabs-content & label {
+        margin: 0;
+        min-width: 0;
+        padding: 2px 8px;
+    }
+}
+.red-ui-palette-editor-upload {
+    display: none;
+    position: absolute;
+    left: 0;
+    right: 0;
+    top: 44px;
+    padding: 20px;
+    background: $secondary-background;
+    border-bottom: 1px $secondary-border-color solid;
+    box-shadow: 1px 1px 4px $shadow;
+
+    .placeholder-input {
+        width: calc(100% - 180px);
+        margin: 0;
+    }
+}
+.red-ui-palette-editor-upload-buttons {
+    float: right;
+    button {
+        margin-left: 10px;
+    }
+}

packages/node_modules/@node-red/registry/lib/installer.js

@@ -16,7 +16,9 @@
 
 
 var path = require("path");
-var fs = require("fs");
+var os = require("os");
+var fs = require("fs-extra");
+var tar = require("tar");
 
 var registry = require("./registry");
 var library = require("./library");
@@ -30,9 +32,10 @@ var npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm';
 var paletteEditorEnabled = false;
 
 var settings;
-var moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
+const moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
-var slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
+const slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
-var pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
+const pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
+const localtgzRe = /^\/.+tgz$/;
 
 function init(runtime) {
     events = runtime.events;
@@ -45,12 +48,14 @@ var activePromise = Promise.resolve();
 
 function checkModulePath(folder) {
     var moduleName;
+    var moduleVersion;
     var err;
     var fullPath = path.resolve(folder);
     var packageFile = path.join(fullPath,'package.json');
     try {
         var pkg = require(packageFile);
         moduleName = pkg.name;
+        moduleVersion = pkg.version;
         if (!pkg['node-red']) {
             // TODO: nls
             err = new Error("Invalid Node-RED module");
@@ -62,7 +67,10 @@ function checkModulePath(folder) {
         err.code = 404;
         throw err;
     }
-    return moduleName;
+    return {
+        name: moduleName,
+        version: moduleVersion
+    };
 }
 
 function checkExistingModule(module,version) {
@@ -77,7 +85,11 @@ function checkExistingModule(module,version) {
     }
     return false;
 }
+
 function installModule(module,version,url) {
+    if (Buffer.isBuffer(module)) {
+        return installTarball(module)
+    }
     module = module || "";
     activePromise = activePromise.then(() => {
         //TODO: ensure module is 'safe'
@@ -86,7 +98,7 @@ function installModule(module,version,url) {
             var isUpgrade = false;
             try {
                 if (url) {
-                    if (pkgurlRe.test(url)) {
+                    if (pkgurlRe.test(url) || localtgzRe.test(url)) {
                         // Git remote url or Tarball url - check the valid package url
                         installName = url;
                     } else {
@@ -104,7 +116,8 @@ function installModule(module,version,url) {
                 } else if (slashRe.test(module)) {
                     // A path - check if there's a valid package.json
                     installName = module;
-                    module = checkModulePath(module);
+                    let info = checkModulePath(module);
+                    module = info.name;
                 } else {
                     log.warn(log._("server.install.install-failed-name",{name:module}));
                     e = new Error("Invalid module name");
@@ -168,7 +181,6 @@ function installModule(module,version,url) {
     return activePromise;
 }
 
-
 function reportAddedModules(info) {
     //comms.publish("node/added",info.nodes,false);
     if (info.nodes.length > 0) {
@@ -197,6 +209,93 @@ function reportRemovedModules(removedNodes) {
     return removedNodes;
 }
 
+async function getExistingPackageVersion(moduleName) {
+    try {
+        const packageFilename = path.join(settings.userDir || process.env.NODE_RED_HOME || "." , "package.json");
+        const pkg = await fs.readJson(packageFilename);
+        if (pkg.dependencies) {
+            return pkg.dependencies[moduleName];
+        }
+    } catch(err) {
+    }
+    return null;
+}
+
+async function installTarball(tarball) {
+    // Check this tarball contains a valid node-red module.
+    // Get its module name/version
+    const moduleInfo = await getTarballModuleInfo(tarball);
+
+    // Write the tarball to <userDir>/nodes/<filename.tgz>
+    // where the filename is the normalised form based on module name/version
+    let normalisedModuleName = moduleInfo.name[0] === '@'
+          ? moduleInfo.name.substr(1).replace(/\//g, '-')
+          : moduleInfo.name
+    const tarballFile = `${normalisedModuleName}-${moduleInfo.version}.tgz`;
+    let tarballPath = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes", tarballFile));
+
+    // (from fs-extra - move to writeFile with promise once Node 8 dropped)
+    await fs.outputFile(tarballPath, tarball);
+
+    // Next, need to check to see if this module is listed in `<userDir>/package.json`
+    let existingVersion = await getExistingPackageVersion(moduleInfo.name);
+    let existingFile = null;
+    let isUpdate = false;
+
+    // If this is a known module, need to check if there will be an old tarball
+    // to remove after the install of this one
+    if (existingVersion) {
+        // - Known module
+        if (/^file:nodes\//.test(existingVersion)) {
+            existingFile = existingVersion.substring(11);
+            isUpdate = true;
+            if (tarballFile === existingFile) {
+                // Edge case: a tar with the same name has bee uploaded.
+                // Carry on with the install, but don't remove the 'old' file
+                // as it will have been overwritten by the new one
+                existingFile = null;
+            }
+        }
+    }
+
+    // Install the tgz
+    return installModule(moduleInfo.name, moduleInfo.version, tarballPath).then(function(info) {
+        if (existingFile) {
+            // Remove the old file
+            return fs.remove(path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes",existingFile))).then(() => info).catch(() => info)
+        }
+        return info;
+    })
+}
+
+async function getTarballModuleInfo(tarball) {
+    const tarballDir = fs.mkdtempSync(path.join(os.tmpdir(),"nr-tarball-"));
+    const removeExtractedTar = function(done) {
+        fs.remove(tarballDir, err => {
+            done();
+        })
+    }
+    return new Promise((resolve,reject) => {
+        var writeStream = tar.x({
+            cwd: tarballDir
+        }).on('error', err => {
+            reject(err);
+        }).on('finish', () => {
+            try {
+                let moduleInfo = checkModulePath(path.join(tarballDir,"package"));
+                removeExtractedTar(err => {
+                    resolve(moduleInfo);
+                })
+            } catch(err) {
+                removeExtractedTar(() => {
+                    reject(err);
+                });
+            }
+        });
+        writeStream.end(tarball);
+    });
+}
+
 function uninstallModule(module) {
     activePromise = activePromise.then(() => {
         return new Promise((resolve,reject) => {
@@ -271,6 +370,7 @@ function checkPrereq() {
         })
     }
 }
+
 module.exports = {
     init: init,
     checkPrereq: checkPrereq,

packages/node_modules/@node-red/registry/lib/loader.js

@@ -15,7 +15,7 @@
  **/
 
 var when = require("when");
-var fs = require("fs");
+var fs = require("fs-extra");
 var path = require("path");
 var semver = require("semver");
 
@@ -113,52 +113,8 @@ function loadNodeFiles(nodeFiles) {
     });
 }
 
-function loadNodeConfig(fileInfo) {
+async function loadNodeTemplate(node) {
-    return new Promise(function(resolve) {
+    return fs.readFile(node.template,'utf8').then(content => {
-        var file = fileInfo.file;
-        var module = fileInfo.module;
-        var name = fileInfo.name;
-        var version = fileInfo.version;
-
-        var id = module + "/" + name;
-        var info = registry.getNodeInfo(id);
-        var isEnabled = true;
-        if (info) {
-            if (info.hasOwnProperty("loaded")) {
-                throw new Error(file+" already loaded");
-            }
-            isEnabled = info.enabled;
-        }
-
-        var node = {
-            id: id,
-            module: module,
-            name: name,
-            file: file,
-            template: file.replace(/\.js$/,".html"),
-            enabled: isEnabled,
-            loaded:false,
-            version: version,
-            local: fileInfo.local
-        };
-        if (fileInfo.hasOwnProperty("types")) {
-            node.types = fileInfo.types;
-        }
-
-        fs.readFile(node.template,'utf8', function(err,content) {
-            if (err) {
-                node.types = [];
-                if (err.code === 'ENOENT') {
-                    if (!node.types) {
-                        node.types = [];
-                    }
-                    node.err = "Error: "+node.template+" does not exist";
-                } else {
-                    node.types = [];
-                    node.err = err.toString();
-                }
-                resolve(node);
-            } else {
         var types = [];
 
         var regExp = /<script (?:[^>]*)data-template-name\s*=\s*['"]([^'"]*)['"]/gi;
@@ -202,29 +158,78 @@ function loadNodeConfig(fileInfo) {
                 break;
             }
         }
+        return node
+    }).catch(err => {
+        node.types = [];
+        if (err.code === 'ENOENT') {
+            if (!node.types) {
+                node.types = [];
+            }
+            node.err = "Error: "+node.template+" does not exist";
+        } else {
+            node.types = [];
+            node.err = err.toString();
+        }
+        return node;
+    });
+}
+
+async function loadNodeLocales(node) {
     if (node.module === 'node-red') {
         // do not look up locales directory for core nodes
         node.namespace = node.module;
-                    resolve(node);
+        return node
-                    return;
     }
-                fs.stat(path.join(path.dirname(file),"locales"),function(err,stat) {
+    return fs.stat(path.join(path.dirname(node.file),"locales")).then(stat => {
-                    if (!err) {
         node.namespace = node.id;
-                        i18n.registerMessageCatalog(node.id,
+        return i18n.registerMessageCatalog(node.id,
-                                path.join(path.dirname(file),"locales"),
+                path.join(path.dirname(node.file),"locales"),
-                                path.basename(file,".js")+".json")
+                path.basename(node.file,".js")+".json")
-                            .then(function() {
+            .then(() => node);
-                                resolve(node);
+    }).catch(err => {
-                            });
-                    } else {
         node.namespace = node.module;
-                        resolve(node);
+        return node;
-                    }
     });
 }
-        });
+
-    });
+async function loadNodeConfig(fileInfo) {
+    var file = fileInfo.file;
+    var module = fileInfo.module;
+    var name = fileInfo.name;
+    var version = fileInfo.version;
+
+    var id = module + "/" + name;
+    var info = registry.getNodeInfo(id);
+    var isEnabled = true;
+    if (info) {
+        if (info.hasOwnProperty("loaded")) {
+            throw new Error(file+" already loaded");
+        }
+        isEnabled = info.enabled;
+    }
+
+    var node = {
+        id: id,
+        module: module,
+        name: name,
+        file: file,
+        template: file.replace(/\.js$/,".html"),
+        enabled: isEnabled,
+        loaded:false,
+        version: version,
+        local: fileInfo.local,
+        types: [],
+        config: "",
+        help: {}
+    };
+    if (fileInfo.hasOwnProperty("types")) {
+        node.types = fileInfo.types;
+    }
+    await loadNodeLocales(node)
+    if (!settings.disableEditor) {
+        return loadNodeTemplate(node);
+    }
+    return node;
 }
 
 /**

packages/node_modules/@node-red/registry/package.json

@@ -18,6 +18,7 @@
     "dependencies": {
         "@node-red/util": "1.2.0-alpha.1",
         "semver": "6.3.0",
+        "tar": "6.0.2",
         "uglify-js": "3.10.0",
         "when": "3.7.8"
     }

packages/node_modules/@node-red/runtime/lib/api/nodes.js

@@ -159,6 +159,7 @@ var api = module.exports = {
     * @param {User} opts.user - the user calling the api
     * @param {String} opts.module - the id of the module to install
     * @param {String} opts.version - (optional) the version of the module to install
+    * @param {Object} opts.tarball - (optional) a tarball file to install. Object has properties `name`, `size` and `buffer`.
     * @param {String} opts.url - (optional) url to install
     * @param {Object} opts.req - the request to log (optional)
     * @return {Promise<ModuleInfo>} - the node module info
@@ -173,6 +174,37 @@ var api = module.exports = {
                 err.status = 400;
                 return reject(err);
             }
+            if (opts.tarball) {
+                if (runtime.settings.editorTheme && runtime.settings.editorTheme.palette && runtime.settings.editorTheme.palette.upload === false) {
+                    runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,error:"invalid_request"}, opts.req);
+                    var err = new Error("Invalid request");
+                    err.code = "invalid_request";
+                    err.status = 400;
+                    return reject(err);
+                }
+                if (opts.module || opts.version || opts.url) {
+                    runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:opts.module,error:"invalid_request"}, opts.req);
+                    var err = new Error("Invalid request");
+                    err.code = "invalid_request";
+                    err.status = 400;
+                    return reject(err);
+                }
+                runtime.nodes.installModule(opts.tarball.buffer).then(function(info) {
+                    runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:info.id}, opts.req);
+                    return resolve(info);
+                }).catch(function(err) {
+
+                    if (err.code) {
+                        err.status = 400;
+                        runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code}, opts.req);
+                    } else {
+                        err.status = 400;
+                        runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code||"unexpected_error",message:err.toString()}, opts.req);
+                    }
+                    return reject(err);
+                })
+                return;
+            }
             if (opts.module) {
                 var existingModule = runtime.nodes.getModuleInfo(opts.module);
                 if (existingModule) {

packages/node_modules/@node-red/runtime/lib/nodes/index.js

@@ -151,11 +151,9 @@ function reportNodeStateChange(info,enabled) {
 }
 
 function installModule(module,version,url) {
-    var existingModule = registry.getModuleInfo(module);
-    var isUpgrade = !!existingModule;
     return registry.installModule(module,version,url).then(function(info) {
-        if (isUpgrade) {
+        if (info.pending_version) {
-            events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:module,version:version}});
+            events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:info.name,version:info.pending_version}});
         } else {
             events.emit("runtime-event",{id:"node/added",retain:false,payload:info.nodes});
         }

packages/node_modules/@node-red/runtime/lib/storage/localfilesystem/util.js

@@ -17,7 +17,6 @@
 var fs = require('fs-extra');
 var fspath = require('path');
 var when = require('when');
-var nodeFn = require('when/node/function');
 
 var log = require("@node-red/util").log; // TODO: separate module
 
@@ -80,32 +79,51 @@ module.exports = {
     * the write hits disk.
     */
     writeFile: function(path,content,backupPath) {
+        var backupPromise;
         if (backupPath) {
-            if (fs.existsSync(path)) {
+            backupPromise = fs.copy(path,backupPath);
-                fs.renameSync(path,backupPath);
+        } else {
+            backupPromise = Promise.resolve();
         }
+
+        const dirname = fspath.dirname(path);
+        const tempFile = `${path}.$$$`;
+
+        return backupPromise.then(() => {
+            if (backupPath) {
+                log.trace(`utils.writeFile - copied ${path} TO ${backupPath}`)
             }
-        return when.promise(function(resolve,reject) {
+            return fs.ensureDir(dirname)
-            fs.ensureDir(fspath.dirname(path), (err)=>{
+        }).then(() => {
-                if (err) {
+            return new Promise(function(resolve,reject) {
-                    reject(err);
+                var stream = fs.createWriteStream(tempFile);
-                    return;
-                }
-                var stream = fs.createWriteStream(path);
                 stream.on('open',function(fd) {
                     stream.write(content,'utf8',function() {
                         fs.fsync(fd,function(err) {
                             if (err) {
-                                log.warn(log._("storage.localfilesystem.fsync-fail",{path: path, message: err.toString()}));
+                                log.warn(log._("storage.localfilesystem.fsync-fail",{path: tempFile, message: err.toString()}));
                             }
                             stream.end(resolve);
                         });
                     });
                 });
                 stream.on('error',function(err) {
+                    log.warn(log._("storage.localfilesystem.fsync-fail",{path: tempFile, message: err.toString()}));
                     reject(err);
                 });
             });
+        }).then(() => {
+            log.trace(`utils.writeFile - written content to ${tempFile}`)
+            return new Promise(function(resolve,reject) {
+                fs.rename(tempFile,path,err => {
+                    if (err) {
+                        log.warn(log._("storage.localfilesystem.fsync-fail",{path: path, message: err.toString()}));
+                        return reject(err);
+                    }
+                    log.trace(`utils.writeFile - renamed ${tempFile} to ${path}`)
+                    resolve();
+                })
+            });
         });
     },
     readFile: readFile,

test/unit/@node-red/editor-api/lib/admin/nodes_spec.js

@@ -53,6 +53,7 @@ describe("api/admin/nodes", function() {
     describe('get nodes', function() {
         it('returns node list', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeList: function() {
                         return Promise.resolve([1,2,3]);
@@ -75,6 +76,7 @@ describe("api/admin/nodes", function() {
 
         it('returns node configs', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeConfigs: function() {
                         return Promise.resolve("<script></script>");
@@ -99,6 +101,7 @@ describe("api/admin/nodes", function() {
 
         it('returns node module info', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleInfo: function(opts) {
                         return Promise.resolve({"node-red":{name:"node-red"}}[opts.module]);
@@ -119,6 +122,7 @@ describe("api/admin/nodes", function() {
 
         it('returns 404 for unknown module', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleInfo: function(opts) {
                         var errInstance = new Error("Not Found");
@@ -143,6 +147,7 @@ describe("api/admin/nodes", function() {
 
         it('returns individual node info', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeInfo: function(opts) {
                         return Promise.resolve({"node-red/123":{id:"node-red/123"}}[opts.id]);
@@ -164,6 +169,7 @@ describe("api/admin/nodes", function() {
 
         it('returns individual node configs', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeConfig: function(opts) {
                         return Promise.resolve({"node-red/123":"<script></script>"}[opts.id]);
@@ -187,6 +193,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns 404 for unknown node', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getNodeInfo: function(opts) {
                         var errInstance = new Error("Not Found");
@@ -215,6 +222,7 @@ describe("api/admin/nodes", function() {
         it('installs the module and returns module info', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     addModule: function(_opts) {
                         opts = _opts;
@@ -244,6 +252,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns error', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     addModule: function(opts) {
                         var errInstance = new Error("Message");
@@ -272,6 +281,7 @@ describe("api/admin/nodes", function() {
         it('uninstalls the module', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     removeModule: function(_opts) {
                         opts = _opts;
@@ -292,6 +302,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns error', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     removeModule: function(opts) {
                         var errInstance = new Error("Message");
@@ -319,6 +330,7 @@ describe("api/admin/nodes", function() {
     describe('enable/disable node set', function() {
         it('returns 400 for invalid request payload', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     setNodeSetState: function(opts) {return Promise.resolve()}
                 }
@@ -340,6 +352,7 @@ describe("api/admin/nodes", function() {
         it('sets node state and returns node info', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     setNodeSetState: function(_opts) {
                         opts = _opts;
@@ -368,6 +381,7 @@ describe("api/admin/nodes", function() {
     describe('enable/disable module' ,function() {
         it('returns 400 for invalid request payload', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     setModuleState: function(opts) {return Promise.resolve()}
                 }
@@ -388,6 +402,7 @@ describe("api/admin/nodes", function() {
         it('sets module state and returns module info', function(done) {
             var opts;
             nodes.init({
+                settings: {},
                 nodes:{
                     setModuleState: function(_opts) {
                         opts = _opts;
@@ -416,6 +431,7 @@ describe("api/admin/nodes", function() {
     describe('get icons', function() {
         it('returns icon list', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getIconList: function() {
                         return Promise.resolve({module:[1,2,3]});
@@ -440,6 +456,7 @@ describe("api/admin/nodes", function() {
     describe('get module messages', function() {
         it('returns message catalog', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleCatalog: function(opts) {
                         return Promise.resolve({a:123});
@@ -459,6 +476,7 @@ describe("api/admin/nodes", function() {
         });
         it('returns all node catalogs', function(done) {
             nodes.init({
+                settings: {},
                 nodes:{
                     getModuleCatalogs: function(opts) {
                         return Promise.resolve({a:1});

test/unit/@node-red/registry/lib/installer_spec.js

@@ -18,7 +18,7 @@ var should = require("should");
 var sinon = require("sinon");
 var when = require("when");
 var path = require("path");
-var fs = require('fs');
+var fs = require('fs-extra');
 var EventEmitter = require('events');
 
 var NR_TEST_UTILS = require("nr-test-utils");
@@ -36,7 +36,7 @@ describe('nodes/registry/installer', function() {
         warn: sinon.stub(),
         info: sinon.stub(),
         metric: sinon.stub(),
-        _: function() { return "abc"}
+        _: function(msg) { return msg }
     }
 
     beforeEach(function() {
@@ -70,8 +70,8 @@ describe('nodes/registry/installer', function() {
             typeRegistry.getModuleInfo.restore();
         }
 
-        if (require('fs').statSync.restore) {
+        if (fs.statSync.restore) {
-            require('fs').statSync.restore();
+            fs.statSync.restore();
         }
 
     });

test/unit/@node-red/registry/lib/loader_spec.js

@@ -18,7 +18,7 @@ var should = require("should");
 var when = require("when");
 var sinon = require("sinon");
 var path = require("path");
-var fs = require("fs");
+var fs = require("fs-extra");
 
 var NR_TEST_UTILS = require("nr-test-utils");
 
@@ -111,7 +111,73 @@ describe("red/nodes/registry/loader",function() {
                 module.nodes.TestNode1.types.should.have.a.length(1);
                 module.nodes.TestNode1.types[0].should.eql('test-node-1');
                 module.nodes.TestNode1.should.have.property("config");
+                module.nodes.TestNode1.config.should.not.eql("");
                 module.nodes.TestNode1.should.have.property("help");
+                module.nodes.TestNode1.help.should.have.property("en-US");
+                module.nodes.TestNode1.should.have.property("namespace","node-red");
+
+                nodes.registerType.calledOnce.should.be.true();
+                nodes.registerType.lastCall.args[0].should.eql('node-red/TestNode1');
+                nodes.registerType.lastCall.args[1].should.eql('test-node-1');
+
+                done();
+            }).catch(function(err) {
+                done(err);
+            });
+        });
+
+        it("load core node files scanned by lfs - ignore html if disableEditor true", function(done) {
+            stubs.push(sinon.stub(localfilesystem,"getNodeFiles", function(){
+                var result = {};
+                result["node-red"] = {
+                    "name": "node-red",
+                    "version": "1.2.3",
+                    "nodes": {
+                        "TestNode1": {
+                            "file": path.join(resourcesDir,"TestNode1","TestNode1.js"),
+                            "module": "node-red",
+                            "name": "TestNode1"
+                        }
+                    }
+                };
+                return result;
+            }));
+
+            stubs.push(sinon.stub(registry,"saveNodeList", function(){ return }));
+            stubs.push(sinon.stub(registry,"addModule", function(){ return }));
+            // This module isn't already loaded
+            stubs.push(sinon.stub(registry,"getNodeInfo", function(){ return null; }));
+
+            stubs.push(sinon.stub(nodes,"registerType"));
+            loader.init({nodes:nodes,log:{info:function(){},_:function(){}},settings:{disableEditor: true, available:function(){return true;}}});
+            loader.load().then(function(result) {
+                registry.addModule.called.should.be.true();
+                var module = registry.addModule.lastCall.args[0];
+                module.should.have.property("name","node-red");
+                module.should.have.property("version","1.2.3");
+                module.should.have.property("nodes");
+                module.nodes.should.have.property("TestNode1");
+                module.nodes.TestNode1.should.have.property("id","node-red/TestNode1");
+                module.nodes.TestNode1.should.have.property("module","node-red");
+                module.nodes.TestNode1.should.have.property("name","TestNode1");
+                module.nodes.TestNode1.should.have.property("file");
+                module.nodes.TestNode1.should.have.property("template");
+                module.nodes.TestNode1.should.have.property("enabled",true);
+                module.nodes.TestNode1.should.have.property("loaded",true);
+                // With disableEditor true, the types property is not populated by the
+                // html file - but instead is populated as nodes register themselves.
+                // But for this test, we have stubbed out registerType, so we won't get any types
+                // module.nodes.TestNode1.should.have.property("types");
+                // module.nodes.TestNode1.types.should.have.a.length(1);
+                // module.nodes.TestNode1.types[0].should.eql('test-node-1');
+
+                // With disableEditor set, config should be blank
+                module.nodes.TestNode1.should.have.property("config");
+                module.nodes.TestNode1.config.should.eql("");
+
+                // help should be an empty object
+                module.nodes.TestNode1.should.have.property("help");
+                module.nodes.TestNode1.help.should.eql({})
                 module.nodes.TestNode1.should.have.property("namespace","node-red");
 
                 nodes.registerType.calledOnce.should.be.true();
@@ -336,9 +402,9 @@ describe("red/nodes/registry/loader",function() {
                 module.nodes.DoesNotExist.should.have.property("loaded",false);
                 module.nodes.DoesNotExist.should.have.property("types");
                 module.nodes.DoesNotExist.types.should.have.a.length(0);
-                module.nodes.DoesNotExist.should.not.have.property("config");
+                module.nodes.DoesNotExist.should.have.property("config","");
-                module.nodes.DoesNotExist.should.not.have.property("help");
+                module.nodes.DoesNotExist.should.have.property("help",{});
-                module.nodes.DoesNotExist.should.not.have.property("namespace","node-red");
+                module.nodes.DoesNotExist.should.have.property("namespace","node-red");
                 module.nodes.DoesNotExist.should.have.property('err');
 
                 nodes.registerType.called.should.be.false();
@@ -390,9 +456,9 @@ describe("red/nodes/registry/loader",function() {
                 module.nodes.DuffNode.should.have.property("loaded",false);
                 module.nodes.DuffNode.should.have.property("types");
                 module.nodes.DuffNode.types.should.have.a.length(0);
-                module.nodes.DuffNode.should.not.have.property("config");
+                module.nodes.DuffNode.should.have.property("config","");
-                module.nodes.DuffNode.should.not.have.property("help");
+                module.nodes.DuffNode.should.have.property("help",{});
-                module.nodes.DuffNode.should.not.have.property("namespace","node-red");
+                module.nodes.DuffNode.should.have.property("namespace","node-red");
                 module.nodes.DuffNode.should.have.property('err');
                 module.nodes.DuffNode.err.should.endWith("DuffNode.html does not exist");