Update logging/metric system

This commit is contained in:
Nick O'Leary
2015-02-03 22:02:26 +00:00
parent 7d6ce1ec12
commit 0aaea1ec40
24 changed files with 245 additions and 209 deletions

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,8 +17,8 @@ var express = require('express');
var fs = require("fs");
var events = require("../events");
var path = require("path");
var util = require("util");
var log = require("../log");
var redNodes = require("../nodes");
var settings = require("../settings");
@@ -32,9 +32,9 @@ module.exports = {
redNodes.setFlows(flows,deploymentType).then(function() {
res.send(204);
}).otherwise(function(err) {
util.log("[red] Error saving flows : "+err);
log.warn("Error saving flows : "+err.message);
log.warn(err.stack);
res.send(500,err.message);
console.log(err.stack);
});
}
}

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2013, 2014 IBM Corp.
* Copyright 2013, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,10 +14,9 @@
* limitations under the License.
**/
var util = require("util");
var redApp = null;
var storage = require("../storage");
var log = require("../log");
function createLibrary(type) {
if (redApp) {
@@ -33,7 +32,7 @@ function createLibrary(type) {
}
}).otherwise(function(err) {
if (err) {
util.log("[red] Error loading library entry '"+path+"' : "+err);
log.warn("Error loading library entry '"+path+"' : "+err);
if (err.message.indexOf('forbidden') === 0) {
res.send(403);
return;
@@ -53,7 +52,7 @@ function createLibrary(type) {
storage.saveLibraryEntry(type,path,req.query,fullBody).then(function() {
res.send(204);
}).otherwise(function(err) {
util.log("[red] Error saving library entry '"+path+"' : "+err);
log.warn("Error saving library entry '"+path+"' : "+err);
if (err.message.indexOf('forbidden') === 0) {
res.send(403);
return;
@@ -81,7 +80,7 @@ module.exports = {
res.send(data);
}).otherwise(function(err) {
if (err) {
util.log("[red] Error loading flow '"+req.params[0]+"' : "+err);
log.warn("Error loading flow '"+req.params[0]+"' : "+err);
if (err.message.indexOf('forbidden') === 0) {
res.send(403);
return;
@@ -95,7 +94,7 @@ module.exports = {
storage.saveFlow(req.params[0],flow).then(function() {
res.send(204);
}).otherwise(function(err) {
util.log("[red] Error loading flow '"+req.params[0]+"' : "+err);
log.warn("Error loading flow '"+req.params[0]+"' : "+err);
if (err.message.indexOf('forbidden') === 0) {
res.send(403);
return;

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,12 +17,12 @@ var express = require('express');
var fs = require("fs");
var path = require("path");
var when = require('when');
var util = require('util');
var events = require("../events");
var redNodes = require("../nodes");
var comms = require("../comms");
var server = require("../server");
var log = require("../log");
var settings = require("../settings");
@@ -196,13 +196,13 @@ module.exports = {
}
if (info.enabled === body.enabled && !info.err) {
comms.publish("node/"+(body.enabled?"enabled":"disabled"),info,false);
util.log("[red] "+(body.enabled?"Enabled":"Disabled")+" node types:");
log.info(" "+(body.enabled?"Enabled":"Disabled")+" node types:");
for (var j = 0; j < info.types.length; j++) {
util.log("[red] - " + info.types[j]);
log.info(" - " + info.types[j]);
}
} else if (body.enabled && info.err) {
util.log("[red] Failed to enable node:");
util.log("[red] - "+info.name+" : "+info.err);
log.warn("Failed to enable node:");
log.warn(" - "+info.name+" : "+info.err);
}
}
}
@@ -241,13 +241,13 @@ function putNode(node, enabled) {
if (info.enabled === enabled && !info.err) {
comms.publish("node/"+(enabled?"enabled":"disabled"),info,false);
util.log("[red] "+(enabled?"Enabled":"Disabled")+" node types:");
log.info(" "+(enabled?"Enabled":"Disabled")+" node types:");
for (var i=0;i<info.types.length;i++) {
util.log("[red] - "+info.types[i]);
log.info(" - "+info.types[i]);
}
} else if (enabled && info.err) {
util.log("[red] Failed to enable node:");
util.log("[red] - "+info.name+" : "+info.err);
log.warn("Failed to enable node:");
log.warn(" - "+info.name+" : "+info.err);
}
}

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
**/
var ws = require("ws");
var util = require("util");
var log = require("./log");
var server;
var settings;
@@ -57,7 +57,7 @@ function start() {
try {
msg = JSON.parse(data);
} catch(err) {
util.log("[red:comms] received malformed message : "+err.toString());
log.warn("comms received malformed message : "+err.toString());
return;
}
if (msg.subscribe) {
@@ -65,12 +65,12 @@ function start() {
}
});
ws.on('error', function(err) {
util.log("[red:comms] error : "+err.toString());
log.warn("comms error : "+err.toString());
});
});
wsServer.on('error', function(err) {
util.log("[red:comms] server error : "+err.toString());
log.warn("comms server error : "+err.toString());
});
lastSentTime = Date.now();
@@ -110,7 +110,7 @@ function publishTo(ws,topic,data) {
try {
ws.send(msg);
} catch(err) {
util.log("[red:comms] send error : "+err.toString());
log.warn("comms send error : "+err.toString());
}
}

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,64 +17,63 @@
var util = require("util");
var EventEmitter = require("events").EventEmitter;
var logLevel;
var metricsOn;
var levels = {
"fatal" : 10,
"error" : 20,
"warn" : 30,
"info" : 40,
"debug" : 50,
"trace" : 60
fatal: 10,
error: 20,
warn: 30,
info: 40,
debug: 50,
trace: 60,
metric: 99
}
var levelNames = {
10: "fatal",
20: "error",
30: "warn",
40: "info",
50: "debug",
60: "trace",
99: "metric",
}
var logHandlers = [];
var ConsoleLogHandler = new EventEmitter();
ConsoleLogHandler.on("log",function(msg) {
if ((msg.level === 'metric')) {
if (metricsOn) {
util.log("["+msg.level+"] ["+msg.event+":"+ msg.nodeid+":"+ msg.msguuid+"]"+metrics);
var ConsoleLogHandler = function(settings) {
this.logLevel = levels[settings.level]||levels.info;
this.metricsOn = settings.metrics||false;
this.on("log",function(msg) {
if (this.shouldReportMessage(msg.level)) {
if (msg.level == log.METRIC) {
util.log("[metric] "+JSON.stringify(msg));
} else {
util.log("["+levelNames[msg.level]+"] "+(msg.type?"["+msg.type+":"+(msg.name||msg.id)+"] ":"")+msg.msg);
}
}
} else if (shouldReportMessage(msg.level)) {
util.log("["+msg.level+"] ["+msg.type+":"+(msg.name||msg.id)+"] "+msg.msg);
}
});
});
}
util.inherits(ConsoleLogHandler, EventEmitter);
function shouldReportMessage(msglevel) {
if (((msglevel === 'fatal') && (logLevel === 10)) ||
((msglevel === 'error') && (logLevel >= 20)) ||
((msglevel === 'warn') && (logLevel >= 30)) ||
((msglevel === 'info') && (logLevel >= 40)) ||
((msglevel === 'debug') && (logLevel >= 50)) ||
((msglevel === 'trace') && (logLevel === 60))) {
return true;
}
return false;
ConsoleLogHandler.prototype.shouldReportMessage = function(msglevel) {
return msglevel <= this.logLevel || (msglevel == log.METRIC && this.metricsOn);
}
var log = module.exports = {
FATAL: 10,
ERROR: 20,
WARN: 30,
INFO: 40,
DEBUG: 50,
TRACE: 60,
METRIC: 99,
init: function(settings) {
if (settings.logLevel) {
var levelNames = Object.keys(levels);
levelNames.forEach(function(levelName) {
if (levelName === settings.logLevel) {
logLevel =levels[levelName];
}
});
}
if (!logLevel) {
// handles case if someone has put in garbage for the log level
logLevel = 40;
}
if (settings.metricsOn) {
metricsOn = settings.metricsOn
} else {
metricsOn = false;
var consoleSettings = {};
if (settings.logging) {
consoleSettings = settings.logging.console || {};
}
log.addHandler(new ConsoleLogHandler(consoleSettings));
},
addHandler: function(func) {
@@ -86,7 +85,12 @@ var log = module.exports = {
logHandlers.forEach(function(handler) {
handler.emit("log",msg);
});
},
info: function(msg) {
log.log({level:log.INFO,msg:msg});
},
warn: function(msg) {
log.log({level:log.WARN,msg:msg});
}
}
log.addHandler(ConsoleLogHandler);

View File

@@ -14,7 +14,6 @@
* limitations under the License.
**/
var util = require("util");
var when = require("when");
var clone = require("clone");
@@ -22,6 +21,7 @@ var typeRegistry = require("./registry");
var credentials = require("./credentials");
var redUtil = require("../util");
var events = require("../events");
var log = require("../log");
function getID() {
return (1+Math.random()*4294967295).toString(16);
@@ -35,10 +35,10 @@ function createNode(type,config) {
nn = new nt(clone(config));
}
catch (err) {
util.log("[red] "+type+" : "+err);
log.warn(type+" : "+err);
}
} else {
util.log("[red] unknown type: "+type);
log.warn("unknown type: "+type);
}
return nn;
}

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -96,10 +96,10 @@ Node.prototype.send = function(msg) {
// A single message and a single wire on output 0
// TODO: pre-load flows.get calls - cannot do in constructor
// as not all nodes are defined at that point
if (!msg._messageUuid) {
msg._messageUuid = constructUniqueIdentifier();
if (!msg._id) {
msg._id = constructUniqueIdentifier();
}
this.metric("Node.prototype.send", msg);
this.metric("send",msg);
node = flows.get(this._wire);
if (node) {
node.receive(msg);
@@ -116,6 +116,8 @@ Node.prototype.send = function(msg) {
// any calls to node.receive
var sendEvents = [];
var sentMessageId = null;
// for each output of node eg. [msgs to output 0, msgs to output 1, ...]
for (var i = 0; i < numOutputs; i++) {
var wires = this.wires[i]; // wires leaving output i
@@ -132,20 +134,15 @@ Node.prototype.send = function(msg) {
if (node) {
// for each msg to send eg. [[m1, m2, ...], ...]
for (k = 0; k < msgs.length; k++) {
var m = msgs[k];
if (!sentMessageId) {
sentMessageId = m._id;
}
if (msgSent) {
var clonedmsg = redUtil.cloneMessage(msgs[k]);
// overwriting any previously written uuid because a cloned
// message is a different one
clonedmsg._messageUuid = constructUniqueIdentifier();
this.metric("Node.prototype.send",clonedmsg,msgs[k]._messageUuid);
var clonedmsg = redUtil.cloneMessage(m);
sendEvents.push({n:node,m:clonedmsg});
} else {
// first msg sent so don't clone
if (!msgs[k]._messageUuid) {
msgs[k]._messageUuid = constructUniqueIdentifier();
}
this.metric("Node.prototype.send", msgs[k]);
sendEvents.push({n:node,m:msgs[k]});
sendEvents.push({n:node,m:m});
msgSent = true;
}
}
@@ -154,9 +151,16 @@ Node.prototype.send = function(msg) {
}
}
}
if (!sentMessageId) {
sentMessageId = constructUniqueIdentifier();
}
this.metric("send",{_id:sentMessageId});
for (i=0;i<sendEvents.length;i++) {
var ev = sendEvents[i];
if (!ev.m._id) {
ev.m._id = sentMessageId;
}
ev.n.receive(ev.m);
}
};
@@ -165,10 +169,10 @@ Node.prototype.receive = function(msg) {
if (!msg) {
msg = {};
}
if (!msg._messageUuid) {
msg._messageUuid = constructUniqueIdentifier();
if (!msg._id) {
msg._id = constructUniqueIdentifier();
}
this.metric("Node.prototype.receive",msg);
this.metric("receive",msg);
this.emit("input", msg);
};
@@ -186,25 +190,24 @@ function log_helper(self, level, msg) {
}
Node.prototype.log = function(msg) {
log_helper(this, 'info', msg);
log_helper(this, Log.INFO, msg);
};
Node.prototype.warn = function(msg) {
log_helper(this, 'warn', msg);
log_helper(this, Log.WARN, msg);
};
Node.prototype.error = function(msg) {
log_helper(this, 'error', msg);
log_helper(this, Log.ERROR, msg);
};
Node.prototype.metric = function(eventname, msg, metricValue) {
var metrics = {};
metrics.level = "metric";
metrics.level = Log.METRIC;
metrics.nodeid = this.id;
metrics.event = eventname;
metrics.msguuid = msg._messageUuid;
metrics.event = "node."+this.type+"."+eventname;
metrics.msgid = msg._id;
metrics.metric = metricValue;
Log.log(metrics);
}

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,9 +14,10 @@
* limitations under the License.
**/
var util = require("util");
var when = require("when");
var log = require("../log");
var credentialCache = {};
var storage = null;
var credentialsDef = {};
@@ -75,7 +76,7 @@ module.exports = {
return storage.getCredentials().then(function (creds) {
credentialCache = creds;
}).otherwise(function (err) {
util.log("[red] Error loading credentials : " + err);
log.warn("Error loading credentials : " + err);
});
},
@@ -168,7 +169,7 @@ module.exports = {
var dashedType = nodeType.replace(/\s+/g, '-');
var definition = credentialsDef[dashedType];
if (!definition) {
util.log('Credential Type ' + nodeType + ' is not registered.');
log.warn('Credential Type ' + nodeType + ' is not registered.');
return;
}

View File

@@ -14,7 +14,6 @@
* limitations under the License.
**/
var util = require("util");
var clone = require("clone");
var when = require("when");
@@ -37,7 +36,7 @@ var activeConfigNodes = {};
events.on('type-registered',function(type) {
if (activeFlow) {
if (activeFlow.typeRegistered(type)) {
util.log("[red] Missing type registered: "+type);
log.info("Missing type registered: "+type);
}
}
});
@@ -58,7 +57,7 @@ var flowNodes = module.exports = {
flowNodes.startFlows();
});
}).otherwise(function(err) {
util.log("[red] Error loading flows : "+err);
log.warn("Error loading flows : "+err);
console.log(err.stack);
});
},
@@ -162,22 +161,22 @@ var flowNodes = module.exports = {
}
},
startFlows: function() {
util.log("[red] Starting flows");
log.info("Starting flows");
try {
activeFlow.start();
} catch(err) {
var missingTypes = activeFlow.getMissingTypes();
if (missingTypes.length > 0) {
util.log("[red] Waiting for missing types to be registered:");
log.info("Waiting for missing types to be registered:");
for (var i=0;i<missingTypes.length;i++) {
util.log("[red] - "+missingTypes[i]);
log.info(" - "+missingTypes[i]);
}
}
}
},
stopFlows: function() {
util.log("[red] Stopping flows");
log.info("Stopping flows");
return activeFlow.stop();
}
};

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2013, 2014 IBM Corp.
* Copyright 2013, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -109,6 +109,7 @@ module.exports = {
// Node registry
createNode: createNode,
getNode: flows.get,
eachNode: flows.eachNode,
addNode: registry.addNode,
removeNode: removeNode,

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2014 IBM Corp.
* Copyright 2014, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2013 IBM Corp.
* Copyright 2013, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

View File

@@ -1,5 +1,5 @@
/**
* Copyright 2013, 2014 IBM Corp.
* Copyright 2013, 2015 IBM Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@
**/
var express = require('express');
var util = require('util');
var when = require('when');
var child_process = require('child_process');
var redNodes = require("./nodes");
var comms = require("./comms");
var storage = require("./storage");
var log = require("./log");
var app = null;
var nodeApp = null;
@@ -49,9 +49,9 @@ function start() {
settings.load(storage).then(function() {
console.log("\nWelcome to Node-RED\n===================\n");
if (settings.version) {
util.log("[red] Version: "+settings.version);
log.info("Version: "+settings.version);
}
util.log("[red] Loading palette nodes");
log.info("Loading palette nodes");
redNodes.init(settings,storage);
redNodes.load().then(function() {
var i;
@@ -59,19 +59,19 @@ function start() {
var nodeErrors = nodes.filter(function(n) { return n.err!=null;});
var nodeMissing = nodes.filter(function(n) { return n.module && n.enabled && !n.loaded && !n.err;});
if (nodeErrors.length > 0) {
util.log("------------------------------------------");
log.warn("------------------------------------------");
if (settings.verbose) {
for (i=0;i<nodeErrors.length;i+=1) {
util.log("["+nodeErrors[i].name+"] "+nodeErrors[i].err);
log.warn("["+nodeErrors[i].name+"] "+nodeErrors[i].err);
}
} else {
util.log("[red] Failed to register "+nodeErrors.length+" node type"+(nodeErrors.length==1?"":"s"));
util.log("[red] Run with -v for details");
log.warn("Failed to register "+nodeErrors.length+" node type"+(nodeErrors.length==1?"":"s"));
log.warn("Run with -v for details");
}
util.log("------------------------------------------");
log.warn("------------------------------------------");
}
if (nodeMissing.length > 0) {
util.log("[red] Missing node modules:");
log.warn("Missing node modules:");
var missingModules = {};
for (i=0;i<nodeMissing.length;i++) {
var missing = nodeMissing[i];
@@ -80,7 +80,7 @@ function start() {
var promises = [];
for (i in missingModules) {
if (missingModules.hasOwnProperty(i)) {
util.log("[red] - "+i+": "+missingModules[i].join(", "));
log.warn(" - "+i+": "+missingModules[i].join(", "));
if (settings.autoInstallModules) {
installModule(i).otherwise(function(err) {
// Error already reported. Need the otherwise handler
@@ -90,7 +90,7 @@ function start() {
}
}
if (!settings.autoInstallModules) {
util.log("[red] Removing modules from config");
log.info("Removing modules from config");
redNodes.cleanModuleList();
}
}
@@ -113,10 +113,10 @@ function start() {
function reportAddedModules(info) {
comms.publish("node/added",info,false);
if (info.length > 0) {
util.log("[red] Added node types:");
log.info("Added node types:");
for (var i=0;i<info.length;i++) {
for (var j=0;j<info[i].types.length;j++) {
util.log("[red] - "+
log.info(" - "+
(info[i].module?info[i].module+":":"")+
info[i].types[j]+
(info[i].err?" : "+info[i].err:"")
@@ -129,10 +129,10 @@ function reportAddedModules(info) {
function reportRemovedModules(removedNodes) {
comms.publish("node/removed",removedNodes,false);
util.log("[red] Removed node types:");
log.info("Removed node types:");
for (var j=0;j<removedNodes.length;j++) {
for (var i=0;i<removedNodes[j].types.length;i++) {
util.log("[red] - "+(removedNodes[i].module?removedNodes[i].module+":":"")+removedNodes[j].types[i]);
log.info(" - "+(removedNodes[i].module?removedNodes[i].module+":":"")+removedNodes[j].types[i]);
}
}
return removedNodes;
@@ -145,24 +145,24 @@ function installModule(module) {
reject(new Error("Invalid module name"));
return;
}
util.log("[red] Installing module: "+module);
log.info("Installing module: "+module);
var child = child_process.exec('npm install --production '+module, function(err, stdin, stdout) {
if (err) {
var lookFor404 = new RegExp(" 404 .*"+module+"$","m");
if (lookFor404.test(stdout)) {
util.log("[red] Installation of module "+module+" failed: module not found");
log.warn("Installation of module "+module+" failed: module not found");
var e = new Error();
e.code = 404;
reject(e);
} else {
util.log("[red] Installation of module "+module+" failed:");
util.log("------------------------------------------");
console.log(err.toString());
util.log("------------------------------------------");
log.warn("Installation of module "+module+" failed:");
log.warn("------------------------------------------");
log.warn(err.toString());
log.warn("------------------------------------------");
reject(new Error("Install failed"));
}
} else {
util.log("[red] Installed module: "+module);
log.info("Installed module: "+module);
resolve(redNodes.addModule(module).then(reportAddedModules));
}
});
@@ -176,16 +176,16 @@ function uninstallModule(module) {
return;
}
var list = redNodes.removeModule(module);
util.log("[red] Removing module: "+module);
log.info("Removing module: "+module);
var child = child_process.exec('npm remove '+module, function(err, stdin, stdout) {
if (err) {
util.log("[red] Removal of module "+module+" failed:");
util.log("------------------------------------------");
console.log(err.toString());
util.log("------------------------------------------");
log.warn("Removal of module "+module+" failed:");
log.warn("------------------------------------------");
log.warn(err.toString());
log.warn("------------------------------------------");
reject(new Error("Removal failed"));
} else {
util.log("[red] Removed module: "+module);
log.info("Removed module: "+module);
reportRemovedModules(list);
resolve(list);
}

View File

@@ -18,10 +18,11 @@ var fs = require('fs');
var when = require('when');
var nodeFn = require('when/node/function');
var keys = require('when/keys');
var util = require('util');
var fspath = require("path");
var mkdirp = require("mkdirp");
var log = require("../log");
var promiseDir = nodeFn.lift(mkdirp);
var settings;
@@ -170,12 +171,12 @@ var localfilesystem = {
var defer = when.defer();
fs.exists(flowsFullPath, function(exists) {
if (exists) {
util.log("[red] Loading flows : "+flowsFile);
log.info("Loading flows : "+flowsFile);
defer.resolve(nodeFn.call(fs.readFile,flowsFullPath,'utf8').then(function(data) {
return JSON.parse(data);
}));
} else {
util.log("[red] Flows file not found : "+flowsFile );
log.info("Flows file not found : "+flowsFile );
defer.resolve([]);
}
});
@@ -236,7 +237,7 @@ var localfilesystem = {
try {
return JSON.parse(data);
} catch(err) {
util.log("[red] Corrupted config detected - resetting");
log.info("Corrupted config detected - resetting");
return {};
}
} else {