frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2023-10-10 13:36:53 +02:00
Code Issues Releases Wiki Activity

Merge pull request #2682 from node-red/upload-npm

Browse Source 
Add support for file upload on /nodes api
This commit is contained in:
Nick O'Leary 2020-09-03 15:01:31 +01:00 committed by GitHub
commit be880c25f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 313 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -71,6 +71,7 @@
"raw-body": "2.4.1",
"request": "2.88.0",
"semver": "6.3.0",
"tar": "6.0.2",
"uglify-js": "3.10.0",
"when": "3.7.8",
"ws": "6.2.1",

9
packages/node_modules/@node-red/editor-api/lib/admin/index.js vendored
View File

@ -49,7 +49,14 @@ module.exports = {
// Nodes
adminApp.get("/nodes",needsPermission("nodes.read"),nodes.getAll,apiUtil.errorHandler);
adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);
if (!settings.editorTheme || !settings.editorTheme.palette || settings.editorTheme.palette.upload !== false) {
const multer = require('multer');
const upload = multer({ storage: multer.memoryStorage() });
adminApp.post("/nodes",needsPermission("nodes.write"),upload.single("tarball"),nodes.post,apiUtil.errorHandler);
} else {
adminApp.post("/nodes",needsPermission("nodes.write"),nodes.post,apiUtil.errorHandler);
}
adminApp.get(/^\/nodes\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalogs,apiUtil.errorHandler);
adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+\/[^\/]+)\/messages/,needsPermission("nodes.read"),nodes.getModuleCatalog,apiUtil.errorHandler);
adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)$/,needsPermission("nodes.read"),nodes.getModule,apiUtil.errorHandler);

10
packages/node_modules/@node-red/editor-api/lib/admin/nodes.js vendored
View File

@ -45,8 +45,18 @@ module.exports = {
module: req.body.module,
version: req.body.version,
url: req.body.url,
tarball: undefined,
req: apiUtils.getRequestLogObject(req)
}
if (!runtimeAPI.settings.editorTheme || !runtimeAPI.settings.editorTheme.palette || runtimeAPI.settings.editorTheme.palette.upload !== false) {
if (req.file) {
opts.tarball = {
name: req.file.originalname,
size: req.file.size,
buffer: req.file.buffer
}
}
}
runtimeAPI.nodes.addModule(opts).then(function(info) {
res.json(info);
}).catch(function(err) {

1
packages/node_modules/@node-red/editor-api/package.json vendored
View File

@ -26,6 +26,7 @@
"express": "4.17.1",
"memorystore": "1.6.2",
"mime": "2.4.6",
"multer": "1.4.2",
"mustache": "4.0.1",
"oauth2orize": "1.11.0",
"passport-http-bearer": "1.0.1",

4
packages/node_modules/@node-red/editor-client/locales/en-US/editor.json vendored
View File

@ -22,7 +22,8 @@
"color": "Color",
"position": "Position",
"enable": "Enable",
"disable": "Disable"
"disable": "Disable",
"upload": "Upload"
},
"type": {
"string": "string",
@ -532,6 +533,7 @@
"sortAZ": "a-z",
"sortRecent": "recent",
"more": "+ __count__ more",
"upload": "Upload module tgz file",
"errors": {
"catalogLoadFailed": "<p>Failed to load node catalogue.</p><p>Check the browser console for more information</p>",
"installFailed": "<p>Failed to install: __module__</p><p>__message__</p><p>Check the log for more information</p>",

91
packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js vendored
View File

@ -542,8 +542,6 @@ RED.palette.editor = (function() {
return settingsPane;
}
function createSettingsPane() {
settingsPane = $('<div id="red-ui-settings-tab-palette"></div>');
var content = $('<div id="red-ui-palette-editor">'+
@ -574,7 +572,11 @@ RED.palette.editor = (function() {
minimumActiveTabWidth: 110
});
createNodeTab(content);
createInstallTab(content);
}
function createNodeTab(content) {
var modulesTab = $('<div>',{class:"red-ui-palette-editor-tab"}).appendTo(content);
editorTabs.addTab({
@ -726,9 +728,9 @@ RED.palette.editor = (function() {
}
}
});
}
function createInstallTab(content) {
var installTab = $('<div>',{class:"red-ui-palette-editor-tab hide"}).appendTo(content);
editorTabs.addTab({
@ -761,7 +763,6 @@ RED.palette.editor = (function() {
}
});
$('<span>').text(RED._("palette.editor.sort")+' ').appendTo(toolBar);
var sortGroup = $('<span class="button-group"></span>').appendTo(toolBar);
var sortRelevance = $('<a href="#" class="red-ui-palette-editor-install-sort-option red-ui-sidebar-header-button-toggle selected"><i class="fa fa-sort-amount-desc"></i></a>').appendTo(sortGroup);
@ -795,6 +796,7 @@ RED.palette.editor = (function() {
loadedIndex = {};
initInstallTab();
})
RED.popover.tooltip(refreshButton,"NLS-TODO: Refresh module list");
packageList = $('<ol>',{style:"position: absolute;top: 79px;bottom: 0;left: 0;right: 0px;"}).appendTo(installTab).editableList({
addButton: false,
@ -878,8 +880,87 @@ RED.palette.editor = (function() {
}
});
if (RED.settings.theme('palette.upload') !== false) {
var uploadSpan = $('<span class="button-group">').prependTo(toolBar);
var uploadButton = $('<button type="button" class="red-ui-sidebar-header-button red-ui-palette-editor-upload-button"><label><i class="fa fa-upload"></i><form id="red-ui-palette-editor-upload-form" enctype="multipart/form-data"><input name="tarball" type="file" accept=".tgz"></label></button>').appendTo(uploadSpan);
var uploadInput = uploadButton.find('input[type="file"]');
uploadInput.on("change", function(evt) {
if (this.files.length > 0) {
uploadFilenameLabel.text(this.files[0].name)
uploadToolbar.slideDown(200);
}
})
var uploadToolbar = $('<div class="red-ui-palette-editor-upload"></div>').appendTo(installTab);
var uploadForm = $('<div>').appendTo(uploadToolbar);
var uploadFilename = $('<div class="placeholder-input"><i class="fa fa-upload"></i> </div>').appendTo(uploadForm);
var uploadFilenameLabel = $('<span></span>').appendTo(uploadFilename);
var uploadButtons = $('<div class="red-ui-palette-editor-upload-buttons"></div>').appendTo(uploadForm);
$('<button class="editor-button"></button>').text(RED._("common.label.cancel")).appendTo(uploadButtons).on("click", function(evt) {
evt.preventDefault();
uploadToolbar.slideUp(200);
uploadInput.val("");
});
$('<button class="editor-button primary"></button>').text(RED._("common.label.upload")).appendTo(uploadButtons).on("click", function(evt) {
evt.preventDefault();
var spinner = RED.utils.addSpinnerOverlay(uploadToolbar, true);
var buttonRow = $('<div style="position: relative;bottom: calc(50% + 17px); padding-right: 10px;text-align: right;"></div>').appendTo(spinner);
$('<button class="red-ui-button"></button>').text(RED._("eventLog.view")).appendTo(buttonRow).on("click", function(evt) {
evt.preventDefault();
RED.actions.invoke("core:show-event-log");
});
RED.eventLog.startEvent(RED._("palette.editor.confirm.button.install")+" : "+uploadInput[0].files[0].name);
var data = new FormData();
data.append("tarball",uploadInput[0].files[0]);
var filename = uploadInput[0].files[0].name;
$.ajax({
url: 'nodes',
data: data,
cache: false,
contentType: false,
processData: false,
method: 'POST',
}).always(function(data,textStatus,xhr) {
spinner.remove();
uploadInput.val("");
uploadToolbar.slideUp(200);
}).fail(function(xhr,textStatus,err) {
var message = textStatus;
if (xhr.responseJSON) {
message = xhr.responseJSON.message;
}
var notification = RED.notify(RED._('palette.editor.errors.installFailed',{module: filename,message:message}),{
type: 'error',
modal: true,
fixed: true,
buttons: [
{
text: RED._("common.label.close"),
click: function() {
notification.close();
}
},{
text: RED._("eventLog.view"),
click: function() {
notification.close();
RED.actions.invoke("core:show-event-log");
}
}
]
});
uploadInput.val("");
uploadToolbar.slideUp(200);
})
})
RED.popover.tooltip(uploadButton,RED._("palette.editor.upload"));
}
$('<div id="red-ui-palette-module-install-shade" class="red-ui-palette-module-shade hide"><div class="red-ui-palette-module-shade-status"></div><img src="red/images/spin.svg" class="red-ui-palette-spinner"/></div>').appendTo(installTab);
}
function update(entry,version,url,container,done) {
if (RED.settings.theme('palette.editable') === false) {
done(new Error('Palette not editable'));

39
packages/node_modules/@node-red/editor-client/src/sass/palette-editor.scss vendored
View File

@ -237,3 +237,42 @@ ul.red-ui-palette-module-error-list {
#red-ui-palette-module-install-shade {
padding-top: 80px;
}
button.red-ui-palette-editor-upload-button {
padding: 0;
height: 25px;
margin-top: -1px;
input[type="file"] {
opacity: 0;
margin: 0;
height: 0;
width: 0;
}
.red-ui-settings-tabs-content & label {
margin: 0;
min-width: 0;
padding: 2px 8px;
}
}
.red-ui-palette-editor-upload {
display: none;
position: absolute;
left: 0;
right: 0;
top: 44px;
padding: 20px;
background: $secondary-background;
border-bottom: 1px $secondary-border-color solid;
box-shadow: 1px 1px 4px $shadow;
.placeholder-input {
width: calc(100% - 180px);
margin: 0;
}
}
.red-ui-palette-editor-upload-buttons {
float: right;
button {
margin-left: 10px;
}
}

116
packages/node_modules/@node-red/registry/lib/installer.js vendored
View File

@ -16,7 +16,9 @@
var path = require("path");
var fs = require("fs");
var os = require("os");
var fs = require("fs-extra");
var tar = require("tar");
var registry = require("./registry");
var library = require("./library");
@ -30,9 +32,10 @@ var npmCommand = process.platform === 'win32' ? 'npm.cmd' : 'npm';
var paletteEditorEnabled = false;
var settings;
var moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
var slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
var pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
const moduleRe = /^(@[^/@]+?[/])?[^/@]+?$/;
const slashRe = process.platform === "win32" ? /\\|[/]/ : /[/]/;
const pkgurlRe = /^(https?|git(|\+https?|\+ssh|\+file)):\/\//;
const localtgzRe = /^\/.+tgz$/;
function init(runtime) {
events = runtime.events;
@ -45,12 +48,14 @@ var activePromise = Promise.resolve();
function checkModulePath(folder) {
var moduleName;
var moduleVersion;
var err;
var fullPath = path.resolve(folder);
var packageFile = path.join(fullPath,'package.json');
try {
var pkg = require(packageFile);
moduleName = pkg.name;
moduleVersion = pkg.version;
if (!pkg['node-red']) {
// TODO: nls
err = new Error("Invalid Node-RED module");
@ -62,7 +67,10 @@ function checkModulePath(folder) {
err.code = 404;
throw err;
}
return moduleName;
return {
name: moduleName,
version: moduleVersion
};
}
function checkExistingModule(module,version) {
@ -77,7 +85,11 @@ function checkExistingModule(module,version) {
}
return false;
}
function installModule(module,version,url) {
if (Buffer.isBuffer(module)) {
return installTarball(module)
}
module = module || "";
activePromise = activePromise.then(() => {
//TODO: ensure module is 'safe'
@ -86,7 +98,7 @@ function installModule(module,version,url) {
var isUpgrade = false;
try {
if (url) {
if (pkgurlRe.test(url)) {
if (pkgurlRe.test(url) || localtgzRe.test(url)) {
// Git remote url or Tarball url - check the valid package url
installName = url;
} else {
@ -104,7 +116,8 @@ function installModule(module,version,url) {
} else if (slashRe.test(module)) {
// A path - check if there's a valid package.json
installName = module;
module = checkModulePath(module);
let info = checkModulePath(module);
module = info.name;
} else {
log.warn(log._("server.install.install-failed-name",{name:module}));
e = new Error("Invalid module name");
@ -168,7 +181,6 @@ function installModule(module,version,url) {
return activePromise;
}
function reportAddedModules(info) {
//comms.publish("node/added",info.nodes,false);
if (info.nodes.length > 0) {
@ -197,6 +209,93 @@ function reportRemovedModules(removedNodes) {
return removedNodes;
}
async function getExistingPackageVersion(moduleName) {
try {
const packageFilename = path.join(settings.userDir || process.env.NODE_RED_HOME || "." , "package.json");
const pkg = await fs.readJson(packageFilename);
if (pkg.dependencies) {
return pkg.dependencies[moduleName];
}
} catch(err) {
}
return null;
}
async function installTarball(tarball) {
// Check this tarball contains a valid node-red module.
// Get its module name/version
const moduleInfo = await getTarballModuleInfo(tarball);
// Write the tarball to <userDir>/nodes/<filename.tgz>
// where the filename is the normalised form based on module name/version
let normalisedModuleName = moduleInfo.name[0] === '@'
? moduleInfo.name.substr(1).replace(/\//g, '-')
: moduleInfo.name
const tarballFile = `${normalisedModuleName}-${moduleInfo.version}.tgz`;
let tarballPath = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes", tarballFile));
// (from fs-extra - move to writeFile with promise once Node 8 dropped)
await fs.outputFile(tarballPath, tarball);
// Next, need to check to see if this module is listed in `<userDir>/package.json`
let existingVersion = await getExistingPackageVersion(moduleInfo.name);
let existingFile = null;
let isUpdate = false;
// If this is a known module, need to check if there will be an old tarball
// to remove after the install of this one
if (existingVersion) {
// - Known module
if (/^file:nodes\//.test(existingVersion)) {
existingFile = existingVersion.substring(11);
isUpdate = true;
if (tarballFile === existingFile) {
// Edge case: a tar with the same name has bee uploaded.
// Carry on with the install, but don't remove the 'old' file
// as it will have been overwritten by the new one
existingFile = null;
}
}
}
// Install the tgz
return installModule(moduleInfo.name, moduleInfo.version, tarballPath).then(function(info) {
if (existingFile) {
// Remove the old file
return fs.remove(path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "nodes",existingFile))).then(() => info).catch(() => info)
}
return info;
})
}
async function getTarballModuleInfo(tarball) {
const tarballDir = fs.mkdtempSync(path.join(os.tmpdir(),"nr-tarball-"));
const removeExtractedTar = function(done) {
fs.remove(tarballDir, err => {
done();
})
}
return new Promise((resolve,reject) => {
var writeStream = tar.x({
cwd: tarballDir
}).on('error', err => {
reject(err);
}).on('finish', () => {
try {
let moduleInfo = checkModulePath(path.join(tarballDir,"package"));
removeExtractedTar(err => {
resolve(moduleInfo);
})
} catch(err) {
removeExtractedTar(() => {
reject(err);
});
}
});
writeStream.end(tarball);
});
}
function uninstallModule(module) {
activePromise = activePromise.then(() => {
return new Promise((resolve,reject) => {
@ -271,6 +370,7 @@ function checkPrereq() {
})
}
}
module.exports = {
init: init,
checkPrereq: checkPrereq,

1
packages/node_modules/@node-red/registry/package.json vendored
View File

@ -18,6 +18,7 @@
"dependencies": {
"@node-red/util": "1.2.0-alpha.1",
"semver": "6.3.0",
"tar": "6.0.2",
"uglify-js": "3.10.0",
"when": "3.7.8"
}

32
packages/node_modules/@node-red/runtime/lib/api/nodes.js vendored
View File

@ -159,6 +159,7 @@ var api = module.exports = {
* @param {User} opts.user - the user calling the api
* @param {String} opts.module - the id of the module to install
* @param {String} opts.version - (optional) the version of the module to install
* @param {Object} opts.tarball - (optional) a tarball file to install. Object has properties `name`, `size` and `buffer`.
* @param {String} opts.url - (optional) url to install
* @param {Object} opts.req - the request to log (optional)
* @return {Promise<ModuleInfo>} - the node module info
@ -173,6 +174,37 @@ var api = module.exports = {
err.status = 400;
return reject(err);
}
if (opts.tarball) {
if (runtime.settings.editorTheme && runtime.settings.editorTheme.palette && runtime.settings.editorTheme.palette.upload === false) {
runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,error:"invalid_request"}, opts.req);
var err = new Error("Invalid request");
err.code = "invalid_request";
err.status = 400;
return reject(err);
}
if (opts.module || opts.version || opts.url) {
runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:opts.module,error:"invalid_request"}, opts.req);
var err = new Error("Invalid request");
err.code = "invalid_request";
err.status = 400;
return reject(err);
}
runtime.nodes.installModule(opts.tarball.buffer).then(function(info) {
runtime.log.audit({event: "nodes.install",tarball:opts.tarball.file,module:info.id}, opts.req);
return resolve(info);
}).catch(function(err) {
if (err.code) {
err.status = 400;
runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code}, opts.req);
} else {
err.status = 400;
runtime.log.audit({event: "nodes.install",module:opts.module,version:opts.version,url:opts.url,error:err.code||"unexpected_error",message:err.toString()}, opts.req);
}
return reject(err);
})
return;
}
if (opts.module) {
var existingModule = runtime.nodes.getModuleInfo(opts.module);
if (existingModule) {

6
packages/node_modules/@node-red/runtime/lib/nodes/index.js vendored
View File

@ -151,11 +151,9 @@ function reportNodeStateChange(info,enabled) {
}
function installModule(module,version,url) {
var existingModule = registry.getModuleInfo(module);
var isUpgrade = !!existingModule;
return registry.installModule(module,version,url).then(function(info) {
if (isUpgrade) {
events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:module,version:version}});
if (info.pending_version) {
events.emit("runtime-event",{id:"node/upgraded",retain:false,payload:{module:info.name,version:info.pending_version}});
} else {
events.emit("runtime-event",{id:"node/added",retain:false,payload:info.nodes});
}

18
test/unit/@node-red/editor-api/lib/admin/nodes_spec.js
View File

@ -53,6 +53,7 @@ describe("api/admin/nodes", function() {
describe('get nodes', function() {
it('returns node list', function(done) {
nodes.init({
settings: {},
nodes:{
getNodeList: function() {
return Promise.resolve([1,2,3]);
@ -75,6 +76,7 @@ describe("api/admin/nodes", function() {
it('returns node configs', function(done) {
nodes.init({
settings: {},
nodes:{
getNodeConfigs: function() {
return Promise.resolve("<script></script>");
@ -99,6 +101,7 @@ describe("api/admin/nodes", function() {
it('returns node module info', function(done) {
nodes.init({
settings: {},
nodes:{
getModuleInfo: function(opts) {
return Promise.resolve({"node-red":{name:"node-red"}}[opts.module]);
@ -119,6 +122,7 @@ describe("api/admin/nodes", function() {
it('returns 404 for unknown module', function(done) {
nodes.init({
settings: {},
nodes:{
getModuleInfo: function(opts) {
var errInstance = new Error("Not Found");
@ -143,6 +147,7 @@ describe("api/admin/nodes", function() {
it('returns individual node info', function(done) {
nodes.init({
settings: {},
nodes:{
getNodeInfo: function(opts) {
return Promise.resolve({"node-red/123":{id:"node-red/123"}}[opts.id]);
@ -164,6 +169,7 @@ describe("api/admin/nodes", function() {
it('returns individual node configs', function(done) {
nodes.init({
settings: {},
nodes:{
getNodeConfig: function(opts) {
return Promise.resolve({"node-red/123":"<script></script>"}[opts.id]);
@ -187,6 +193,7 @@ describe("api/admin/nodes", function() {
});
it('returns 404 for unknown node', function(done) {
nodes.init({
settings: {},
nodes:{
getNodeInfo: function(opts) {
var errInstance = new Error("Not Found");
@ -215,6 +222,7 @@ describe("api/admin/nodes", function() {
it('installs the module and returns module info', function(done) {
var opts;
nodes.init({
settings: {},
nodes:{
addModule: function(_opts) {
opts = _opts;
@ -244,6 +252,7 @@ describe("api/admin/nodes", function() {
});
it('returns error', function(done) {
nodes.init({
settings: {},
nodes:{
addModule: function(opts) {
var errInstance = new Error("Message");
@ -272,6 +281,7 @@ describe("api/admin/nodes", function() {
it('uninstalls the module', function(done) {
var opts;
nodes.init({
settings: {},
nodes:{
removeModule: function(_opts) {
opts = _opts;
@ -292,6 +302,7 @@ describe("api/admin/nodes", function() {
});
it('returns error', function(done) {
nodes.init({
settings: {},
nodes:{
removeModule: function(opts) {
var errInstance = new Error("Message");
@ -319,6 +330,7 @@ describe("api/admin/nodes", function() {
describe('enable/disable node set', function() {
it('returns 400 for invalid request payload', function(done) {
nodes.init({
settings: {},
nodes:{
setNodeSetState: function(opts) {return Promise.resolve()}
}
@ -340,6 +352,7 @@ describe("api/admin/nodes", function() {
it('sets node state and returns node info', function(done) {
var opts;
nodes.init({
settings: {},
nodes:{
setNodeSetState: function(_opts) {
opts = _opts;
@ -368,6 +381,7 @@ describe("api/admin/nodes", function() {
describe('enable/disable module' ,function() {
it('returns 400 for invalid request payload', function(done) {
nodes.init({
settings: {},
nodes:{
setModuleState: function(opts) {return Promise.resolve()}
}
@ -388,6 +402,7 @@ describe("api/admin/nodes", function() {
it('sets module state and returns module info', function(done) {
var opts;
nodes.init({
settings: {},
nodes:{
setModuleState: function(_opts) {
opts = _opts;
@ -416,6 +431,7 @@ describe("api/admin/nodes", function() {
describe('get icons', function() {
it('returns icon list', function(done) {
nodes.init({
settings: {},
nodes:{
getIconList: function() {
return Promise.resolve({module:[1,2,3]});
@ -440,6 +456,7 @@ describe("api/admin/nodes", function() {
describe('get module messages', function() {
it('returns message catalog', function(done) {
nodes.init({
settings: {},
nodes:{
getModuleCatalog: function(opts) {
return Promise.resolve({a:123});
@ -459,6 +476,7 @@ describe("api/admin/nodes", function() {
});
it('returns all node catalogs', function(done) {
nodes.init({
settings: {},
nodes:{
getModuleCatalogs: function(opts) {
return Promise.resolve({a:1});

8
test/unit/@node-red/registry/lib/installer_spec.js
View File

@ -18,7 +18,7 @@ var should = require("should");
var sinon = require("sinon");
var when = require("when");
var path = require("path");
var fs = require('fs');
var fs = require('fs-extra');
var EventEmitter = require('events');
var NR_TEST_UTILS = require("nr-test-utils");
@ -36,7 +36,7 @@ describe('nodes/registry/installer', function() {
warn: sinon.stub(),
info: sinon.stub(),
metric: sinon.stub(),
_: function() { return "abc"}
_: function(msg) { return msg }
}
beforeEach(function() {
@ -70,8 +70,8 @@ describe('nodes/registry/installer', function() {
typeRegistry.getModuleInfo.restore();
}
if (require('fs').statSync.restore) {
require('fs').statSync.restore();
if (fs.statSync.restore) {
fs.statSync.restore();
}
});