frodovdr/node-red
1
0
Fork 0
mirror of https://github.com/node-red/node-red.git synced 2025-03-01 10:36:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

add "split/stream" ability to file in node

Browse Source 
and add teste
This commit is contained in:
Dave Conway-Jones
2017-06-22 18:41:49 +01:00
parent 4f34980c9f
commit b91c178200
3 changed files with 216 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
nodes/core/storage/50-file.html
View File

@@ -55,7 +55,9 @@
<label for="node-input-format"><i class="fa fa-sign-out"></i> <span data-i18n="file.label.outputas"></span></label>
<select id="node-input-format">
<option value="utf8" data-i18n="file.output.utf8"></option>
<option value="lines" data-i18n="file.output.lines"></option>
<option value="" data-i18n="file.output.buffer"></option>
<option value="stream" data-i18n="file.output.stream"></option>
</select>
</div>
<div class="form-row">
@@ -83,6 +85,8 @@
<p>The filename should be an absolute path, otherwise it will be relative to
the working directory of the Node-RED process.</p>
<p>On Windows, path separators may need to be escaped, for example: <code>\\Users\\myUser</code>.</p>
<p>Optionally, a text file can be split into lines, outputting one message per line, or a binary file
file into smaller buffer chunks, the chunk size is operating system dependant, but typically 64k (linux/mac) or 41k (Windows).</p>
</script>
<script type="text/javascript">
@@ -124,6 +128,7 @@
name: {value:""},
filename: {value:""},
format: {value:"utf8"},
chunk: {value:false}
},
color:"BurlyWood",
inputs:1,
@@ -137,7 +142,18 @@
},
labelStyle: function() {
return this.name?"node_label_italic":"";
},
oneditprepare: function() {
$("#node-input-format").on("change",function() {
if ($("#node-input-format").val() === "utf8") {
$("#buffer-input-type").hide();
$("#line-input-type").show();
}
else {
$("#buffer-input-type").show();
$("#line-input-type").hide();
}
});
}
});
</script>

181
nodes/core/storage/50-file.js
View File

@@ -18,6 +18,7 @@ module.exports = function(RED) {
"use strict";
var fs = require("fs-extra");
var os = require("os");
var path = require("path");
function FileNode(n) {
RED.nodes.createNode(this,n);
@@ -26,15 +27,27 @@ module.exports = function(RED) {
this.overwriteFile = n.overwriteFile.toString();
this.createDir = n.createDir || false;
var node = this;
node.wstream = null;
node.data = [];
this.on("input",function(msg) {
var filename = node.filename || msg.filename || "";
if (!node.filename) {
node.status({fill:"grey",shape:"dot",text:filename});
if (!node.filename) { node.status({fill:"grey",shape:"dot",text:filename}); }
if (filename === "") { node.warn(RED._("file.errors.nofilename")); }
else if (node.overwriteFile === "delete") {
fs.unlink(filename, function (err) {
if (err) { node.error(RED._("file.errors.deletefail",{error:err.toString()}),msg); }
else if (RED.settings.verbose) { node.log(RED._("file.status.deletedfile",{file:filename})); }
});
}
if (filename === "") {
node.warn(RED._("file.errors.nofilename"));
} else if (msg.hasOwnProperty("payload") && (typeof msg.payload !== "undefined")) {
else if (msg.hasOwnProperty("payload") && (typeof msg.payload !== "undefined")) {
var dir = path.dirname(filename);
if (node.createDir) {
fs.ensureDir(dir, function(err) {
if (err) { node.error(RED._("file.errors.createfail",{error:err.toString()}),msg); }
});
}
var data = msg.payload;
if ((typeof data === "object") && (!Buffer.isBuffer(data))) {
data = JSON.stringify(data);
@@ -42,52 +55,26 @@ module.exports = function(RED) {
if (typeof data === "boolean") { data = data.toString(); }
if (typeof data === "number") { data = data.toString(); }
if ((this.appendNewline) && (!Buffer.isBuffer(data))) { data += os.EOL; }
data = new Buffer(data);
if (this.overwriteFile === "true") {
// using "binary" not {encoding:"binary"} to be 0.8 compatible for a while
//fs.writeFile(filename, data, "binary", function (err) {
fs.writeFile(filename, data, {encoding:"binary"}, function (err) {
if (err) {
if ((err.code === "ENOENT") && node.createDir) {
fs.ensureFile(filename, function (err) {
if (err) { node.error(RED._("file.errors.createfail",{error:err.toString()}),msg); }
else {
fs.writeFile(filename, data, "binary", function (err) {
if (err) { node.error(RED._("file.errors.writefail",{error:err.toString()}),msg); }
});
}
});
}
else { node.error(RED._("file.errors.writefail",{error:err.toString()}),msg); }
node.data.push(new Buffer(data));
while (node.data.length > 0) {
if (this.overwriteFile === "true") {
if (!node.wstream) {
node.wstream = fs.createWriteStream(filename, { encoding:'binary', flags:'w' });
node.wstream.on("error", function(err) {
node.error(RED._("file.errors.writefail",{error:err.toString()}),msg);
});
}
else if (RED.settings.verbose) { node.log(RED._("file.status.wrotefile",{file:filename})); }
});
}
else if (this.overwriteFile === "delete") {
fs.unlink(filename, function (err) {
if (err) { node.error(RED._("file.errors.deletefail",{error:err.toString()}),msg); }
else if (RED.settings.verbose) { node.log(RED._("file.status.deletedfile",{file:filename})); }
});
}
else {
// using "binary" not {encoding:"binary"} to be 0.8 compatible for a while longer
//fs.appendFile(filename, data, "binary", function (err) {
fs.appendFile(filename, data, {encoding:"binary"}, function (err) {
if (err) {
if ((err.code === "ENOENT") && node.createDir) {
fs.ensureFile(filename, function (err) {
if (err) { node.error(RED._("file.errors.createfail",{error:err.toString()}),msg); }
else {
fs.appendFile(filename, data, "binary", function (err) {
if (err) { node.error(RED._("file.errors.appendfail",{error:err.toString()}),msg); }
});
}
});
}
else { node.error(RED._("file.errors.appendfail",{error:err.toString()}),msg); }
}
else {
if (!node.wstream) {
node.wstream = fs.createWriteStream(filename, { encoding:'binary', flags:'a' });
node.wstream.on("error", function(err) {
node.error(RED._("file.errors.appendfail",{error:err.toString()}),msg);
});
}
else if (RED.settings.verbose) { node.log(RED._("file.status.appendedfile",{file:filename})); }
});
}
node.wstream.write(node.data.shift());
}
}
});
@@ -100,14 +87,13 @@ module.exports = function(RED) {
function FileInNode(n) {
RED.nodes.createNode(this,n);
this.filename = n.filename;
this.format = n.format;
this.chunk = false;
if (this.format === "lines") { this.chunk = true; }
if (this.format === "stream") { this.chunk = true; }
var node = this;
var options = {};
if (this.format) {
options['encoding'] = this.format;
}
this.on("input",function(msg) {
var filename = node.filename || msg.filename || "";
if (!node.filename) {
@@ -115,19 +101,82 @@ module.exports = function(RED) {
}
if (filename === "") {
node.warn(RED._("file.errors.nofilename"));
} else {
}
else {
msg.filename = filename;
fs.readFile(filename,options,function(err,data) {
if (err) {
node.error(err,msg);
msg.error = err;
delete msg.payload;
} else {
msg.payload = data;
delete msg.error;
}
node.send(msg);
});
var lines = new Buffer.from([]);
var spare = "";
var count = 0;
var type = "buffer";
var ch = "";
if (node.format === "lines") {
ch = "\n";
type = "string";
}
var hwm;
var getout = false;
var rs = fs.createReadStream(filename)
.on('readable', function () {
var chunk;
var hwm = rs._readableState.highWaterMark;
while (null !== (chunk = rs.read())) {
if (node.chunk === true) {
getout = true;
if (node.format === "lines") {
spare += chunk.toString();
var bits = spare.split("\n");
for (var i=0; i < bits.length - 1; i++) {
var m = {
payload:bits[i],
topic:msg.topic,
filename:msg.filename,
parts:{index:count, ch:ch, type:type, id:msg._msgid}
}
count += 1;
if ((chunk.length < hwm) && (bits[i+1].length === 0)) {
m.parts.count = count;
}
node.send(m);
}
spare = bits[i];
if (chunk.length !== hwm) { getout = false; }
//console.log("LEFT",bits[i].length,bits[i]);
}
if (node.format === "stream") {
var m = {
payload:chunk,
topic:msg.topic,
filename:msg.filename,
parts:{index:count, ch:ch, type:type, id:msg._msgid}
}
count += 1;
if (chunk.length < hwm) { // last chunk is smaller that high water mark = eof
getout = false;
m.parts.count = count;
}
node.send(m);
}
}
else {
lines = Buffer.concat([lines,chunk]);
}
}
})
.on('error', function(err) {
node.error('Error while reading file.', msg);
})
.on('end', function() {
if (node.chunk === false) {
if (node.format === "utf8") { msg.payload = lines.toString(); }
else { msg.payload = lines; }
node.send(msg);
}
else if (getout) { // last chunk same size as high water mark - have to send empty extra packet.
var m = { parts:{index:count, count:count, ch:ch, type:type, id:msg._msgid} };
node.send(m);
}
});
}
});
this.on('close', function() {