Merge branch 'dev' into 3934-csv-rfc4180

packages/node_modules/@node-red/editor-api/lib/admin/context.js

@@ -33,6 +33,9 @@ module.exports = {
             store: req.query['store'],
             req: apiUtils.getRequestLogObject(req)
         }
+        if (req.query['keysOnly'] !== undefined) {
+            opts.keysOnly = true
+        }
         runtimeAPI.context.getValue(opts).then(function(result) {
             res.json(result);
         }).catch(function(err) {

packages/node_modules/@node-red/editor-client/src/js/ui/common/typedInput.js

@@ -54,25 +54,26 @@
         return icon;
     }
 
-    var autoComplete = function(options) {
+    function getMatch(value, searchValue) {
-        function getMatch(value, searchValue) {
+        const idx = value.toLowerCase().indexOf(searchValue.toLowerCase());
-            const idx = value.toLowerCase().indexOf(searchValue.toLowerCase());
+        const len = idx > -1 ? searchValue.length : 0;
-            const len = idx > -1 ? searchValue.length : 0;
+        return {
-            return {
+            index: idx,
-                index: idx,
+            found: idx > -1,
-                found: idx > -1,
+            pre: value.substring(0,idx),
-                pre: value.substring(0,idx),
+            match: value.substring(idx,idx+len),
-                match: value.substring(idx,idx+len),
+            post: value.substring(idx+len),
-                post: value.substring(idx+len),
-            }
-        }
-        function generateSpans(match) {
-            const els = [];
-            if(match.pre) { els.push($('<span/>').text(match.pre)); }
-            if(match.match) { els.push($('<span/>',{style:"font-weight: bold; color: var(--red-ui-text-color-link);"}).text(match.match)); }
-            if(match.post) { els.push($('<span/>').text(match.post)); }
-            return els;
         }
+    }
+    function generateSpans(match) {
+        const els = [];
+        if(match.pre) { els.push($('<span/>').text(match.pre)); }
+        if(match.match) { els.push($('<span/>',{style:"font-weight: bold; color: var(--red-ui-text-color-link);"}).text(match.match)); }
+        if(match.post) { els.push($('<span/>').text(match.post)); }
+        return els;
+    }
+    
+    const msgAutoComplete = function(options) {
         return function(val) {
             var matches = [];
             options.forEach(opt => {
@@ -102,6 +103,197 @@
         }
     }
 
+    function getEnvVars (obj, envVars = {}) {
+        contextKnownKeys.env = contextKnownKeys.env || {}
+        if (contextKnownKeys.env[obj.id]) {
+            return contextKnownKeys.env[obj.id]
+        }
+        let parent
+        if (obj.type === 'tab' || obj.type === 'subflow') {
+            RED.nodes.eachConfig(function (conf) {
+                if (conf.type === "global-config") {
+                    parent = conf;
+                }
+            })
+        } else if (obj.g) {
+            parent = RED.nodes.group(obj.g)
+        } else if (obj.z) {
+            parent = RED.nodes.workspace(obj.z) || RED.nodes.subflow(obj.z)
+        }
+        if (parent) {
+            getEnvVars(parent, envVars)
+        }
+        if (obj.env) {
+            obj.env.forEach(env => {
+                envVars[env.name] = obj
+            })
+        }
+        contextKnownKeys.env[obj.id] = envVars
+        return envVars
+    }
+
+    const envAutoComplete = function (val) {
+        const editStack = RED.editor.getEditStack()
+        if (editStack.length === 0) {
+            done([])
+            return
+        }
+        const editingNode = editStack.pop()
+        if (!editingNode) {
+            return []
+        }
+        const envVarsMap = getEnvVars(editingNode)
+        const envVars = Object.keys(envVarsMap)
+        const matches = []
+        const i = val.lastIndexOf('${')
+        let searchKey = val
+        let isSubkey = false
+        if (i > -1) {
+            if (val.lastIndexOf('}') < i) {
+                searchKey = val.substring(i+2)
+                isSubkey = true
+            }
+        }
+        envVars.forEach(v => {
+            let valMatch = getMatch(v, searchKey);
+            if (valMatch.found) {
+                const optSrc = envVarsMap[v]
+                const element = $('<div>',{style: "display: flex"});
+                const valEl = $('<div/>',{style:"font-family: var(--red-ui-monospace-font); white-space:nowrap; overflow: hidden; flex-grow:1"});
+                valEl.append(generateSpans(valMatch))
+                valEl.appendTo(element)
+
+                if (optSrc) {
+                    const optEl = $('<div>').css({ "font-size": "0.8em" });
+                    let label
+                    if (optSrc.type === 'global-config') {
+                        label = RED._('sidebar.context.global')
+                    } else if (optSrc.type === 'group') {
+                        label = RED.utils.getNodeLabel(optSrc) || (RED._('sidebar.info.group') + ': '+optSrc.id)
+                    } else {
+                        label = RED.utils.getNodeLabel(optSrc) || optSrc.id
+                    }
+
+                    optEl.append(generateSpans({ match: label }));
+                    optEl.appendTo(element);
+                }
+                matches.push({
+                    value: isSubkey ? val + v + '}' : v,
+                    label: element,
+                    i: valMatch.index
+                });
+            }
+        })
+        matches.sort(function(A,B){return A.i-B.i})
+        return matches
+    }
+
+    let contextKnownKeys = {}
+    let contextCache = {}
+    if (RED.events) {
+        RED.events.on("editor:close", function () {
+            contextCache = {}
+            contextKnownKeys = {}
+        });
+    }
+
+    const contextAutoComplete = function() {
+        const that = this
+        const getContextKeysFromRuntime = function(scope, store, searchKey, done) {
+            contextKnownKeys[scope] = contextKnownKeys[scope] || {}
+            contextKnownKeys[scope][store] = contextKnownKeys[scope][store] || new Set()
+            if (searchKey.length > 0) {
+                try {
+                    RED.utils.normalisePropertyExpression(searchKey)
+                } catch (err) {
+                    // Not a valid context key, so don't try looking up
+                    done()
+                    return
+                }
+            }
+            const url = `context/${scope}/${encodeURIComponent(searchKey)}?store=${store}&keysOnly`
+            if (contextCache[url]) {
+                // console.log('CACHED', url)
+                done()
+            } else {
+                // console.log('GET', url)
+                $.getJSON(url, function(data) {
+                    // console.log(data)
+                    contextCache[url] = true
+                    const result = data[store] || {}
+                    const keys = result.keys || []
+                    const keyPrefix = searchKey + (searchKey.length > 0 ? '.' : '')
+                    keys.forEach(key => {
+                        if (/^[a-zA-Z_$][0-9a-zA-Z_$]*$/.test(key)) {
+                            contextKnownKeys[scope][store].add(keyPrefix + key)
+                        } else {
+                            contextKnownKeys[scope][store].add(searchKey + "[\""+key.replace(/"/,"\\\"")+"\"]")
+                        }                        
+                    })
+                    done()
+                })
+            }
+        }
+        const getContextKeys = function(key, done) {
+            const keyParts = key.split('.')
+            const partialKey = keyParts.pop()
+            let scope = that.propertyType
+            if (scope === 'flow') {
+                // Get the flow id of the node we're editing
+                const editStack = RED.editor.getEditStack()
+                if (editStack.length === 0) {
+                    done([])
+                    return
+                }
+                const editingNode = editStack.pop()
+                if (editingNode.z) {
+                    scope = `${scope}/${editingNode.z}`
+                } else {
+                    done([])
+                    return
+                }
+            }
+            const store = (contextStoreOptions.length === 1) ? contextStoreOptions[0].value : that.optionValue
+            const searchKey = keyParts.join('.')
+           
+            getContextKeysFromRuntime(scope, store, searchKey, function() {
+                if (contextKnownKeys[scope][store].has(key) || key.endsWith(']')) {
+                    getContextKeysFromRuntime(scope, store, key, function() {
+                        done(contextKnownKeys[scope][store])
+                    })
+                }
+                done(contextKnownKeys[scope][store])
+            })
+        }
+
+        return function(val, done) {
+            getContextKeys(val, function (keys) {
+                const matches = []
+                keys.forEach(v => {
+                    let optVal = v
+                    let valMatch = getMatch(optVal, val);
+                    if (!valMatch.found && val.length > 0 && val.endsWith('.')) {
+                        // Search key ends in '.' - but doesn't match. Check again
+                        // with [" at the end instead so we match bracket notation
+                        valMatch = getMatch(optVal, val.substring(0, val.length - 1) + '["')
+                    }
+                    if (valMatch.found) {
+                        const element = $('<div>',{style: "display: flex"});
+                        const valEl = $('<div/>',{style:"font-family: var(--red-ui-monospace-font); white-space:nowrap; overflow: hidden; flex-grow:1"});
+                        valEl.append(generateSpans(valMatch))
+                        valEl.appendTo(element)
+                        matches.push({
+                            value: optVal,
+                            label: element,
+                        });
+                    }
+                })
+                matches.sort(function(a, b) { return a.value.localeCompare(b.value) });
+                done(matches);
+            })
+        }
+    }
+
     // This is a hand-generated list of completions for the core nodes (based on the node help html).
     var msgCompletions = [
         { value: "payload" },
@@ -166,20 +358,22 @@
         { value: "_session", source: ["websocket out","tcp out"] },
     ]
     var allOptions = {
-        msg: {value:"msg",label:"msg.",validate:RED.utils.validatePropertyExpression, autoComplete: autoComplete(msgCompletions)},
+        msg: {value:"msg",label:"msg.",validate:RED.utils.validatePropertyExpression, autoComplete: msgAutoComplete(msgCompletions)},
         flow: {value:"flow",label:"flow.",hasValue:true,
             options:[],
             validate:RED.utils.validatePropertyExpression,
             parse: contextParse,
             export: contextExport,
-            valueLabel: contextLabel
+            valueLabel: contextLabel,
+            autoComplete: contextAutoComplete
         },
         global: {value:"global",label:"global.",hasValue:true,
             options:[],
             validate:RED.utils.validatePropertyExpression,
             parse: contextParse,
             export: contextExport,
-            valueLabel: contextLabel
+            valueLabel: contextLabel,
+            autoComplete: contextAutoComplete
         },
         str: {value:"str",label:"string",icon:"red/images/typedInput/az.svg"},
         num: {value:"num",label:"number",icon:"red/images/typedInput/09.svg",validate: function(v) {
@@ -251,7 +445,8 @@
         env: {
             value: "env",
             label: "env variable",
-            icon: "red/images/typedInput/env.svg"
+            icon: "red/images/typedInput/env.svg",
+            autoComplete: envAutoComplete
         },
         node: {
             value: "node",
@@ -427,6 +622,7 @@
     }
 
     var nlsd = false;
+    let contextStoreOptions;
 
     $.widget( "nodered.typedInput", {
         _create: function() {
@@ -438,7 +634,7 @@
                     }
                 }
                 var contextStores = RED.settings.context.stores;
-                var contextOptions = contextStores.map(function(store) {
+                contextStoreOptions = contextStores.map(function(store) {
                     return {value:store,label: store, icon:'<i class="red-ui-typedInput-icon fa fa-database"></i>'}
                 }).sort(function(A,B) {
                     if (A.value === RED.settings.context.default) {
@@ -449,12 +645,12 @@
                         return A.value.localeCompare(B.value);
                     }
                 })
-                if (contextOptions.length < 2) {
+                if (contextStoreOptions.length < 2) {
                     allOptions.flow.options = [];
                     allOptions.global.options = [];
                 } else {
-                    allOptions.flow.options = contextOptions;
+                    allOptions.flow.options = contextStoreOptions;
-                    allOptions.global.options = contextOptions;
+                    allOptions.global.options = contextStoreOptions;
                 }
             }
             nlsd = true;
@@ -544,7 +740,7 @@
                 that.element.trigger('paste',evt);
             });
             this.input.on('keydown', function(evt) {
-                if (that.typeMap[that.propertyType].autoComplete) {
+                if (that.typeMap[that.propertyType].autoComplete || that.input.hasClass('red-ui-autoComplete')) {
                     return
                 }
                 if (evt.keyCode >= 37 && evt.keyCode <= 40) {
@@ -967,6 +1163,9 @@
                     // If previousType is !null, then this is a change of the type, rather than the initialisation
                     var previousType = this.typeMap[this.propertyType];
                     previousValue = this.input.val();
+                    if (this.input.hasClass('red-ui-autoComplete')) {
+                        this.input.autoComplete("destroy");
+                    }
 
                     if (previousType && this.typeChanged) {
                         if (this.options.debug) { console.log(this.identifier,"typeChanged",{previousType,previousValue}) }
@@ -1013,7 +1212,9 @@
                             this.input.val(this.oldValues.hasOwnProperty("_")?this.oldValues["_"]:(opt.default||""))
                         }
                         if (previousType.autoComplete) {
-                            this.input.autoComplete("destroy");
+                            if (this.input.hasClass('red-ui-autoComplete')) {
+                                this.input.autoComplete("destroy");
+                            }
                         }
                     }
                     this.propertyType = type;
@@ -1141,6 +1342,16 @@
                                 } else {
                                     this.optionSelectTrigger.hide();
                                 }
+                                if (opt.autoComplete) {
+                                    let searchFunction = opt.autoComplete
+                                    if (searchFunction.length === 0) {
+                                        searchFunction = opt.autoComplete.call(this)
+                                    }
+                                    this.input.autoComplete({
+                                        search: searchFunction,
+                                        minLength: 0
+                                    })
+                                }
                             }
                             this.optionMenu = this._createMenu(opt.options,opt,function(v){
                                 if (!opt.multiple) {
@@ -1183,8 +1394,12 @@
                             this.valueLabelContainer.hide();
                             this.elementDiv.show();
                             if (opt.autoComplete) {
+                                let searchFunction = opt.autoComplete
+                                if (searchFunction.length === 0) {
+                                    searchFunction = opt.autoComplete.call(this)
+                                }
                                 this.input.autoComplete({
-                                    search: opt.autoComplete,
+                                    search: searchFunction,
                                     minLength: 0
                                 })
                             }

packages/node_modules/@node-red/editor-client/src/js/ui/editor.js

@@ -2082,6 +2082,7 @@ RED.editor = (function() {
             }
         },
         editBuffer: function(options) { showTypeEditor("_buffer", options) },
+        getEditStack: function () { return [...editStack] },
         buildEditForm: buildEditForm,
         validateNode: validateNode,
         updateNodeProperties: updateNodeProperties,

packages/node_modules/@node-red/nodes/core/function/15-change.js

@@ -233,7 +233,9 @@ module.exports = function(RED) {
                                                     // only replace if they match exactly
                                                     RED.util.setMessageProperty(msg,property,value);
                                                 } else {
-                                                    current = current.replace(fromRE,value);
+                                                    // if target is boolean then just replace it
+                                                    if (rule.tot === "bool") { current = value; }
+                                                    else { current = current.replace(fromRE,value); }
                                                     RED.util.setMessageProperty(msg,property,current);
                                                 }
                                             } else if ((typeof current === 'number' || current instanceof Number) && fromType === 'num') {

packages/node_modules/@node-red/runtime/lib/api/context.js

@@ -68,6 +68,7 @@ var api = module.exports = {
     * @param {String} opts.store - the context store
     * @param {String} opts.key - the context key
     * @param {Object} opts.req - the request to log (optional)
+    * @param {Boolean} opts.keysOnly - whether to return keys only
     * @return {Promise} - the node information
     * @memberof @node-red/runtime_context
     */
@@ -102,6 +103,15 @@ var api = module.exports = {
                 if (key) {
                     store = store || availableStores.default;
                     ctx.get(key,store,function(err, v) {
+                        if (opts.keysOnly) {
+                            if (Array.isArray(v)) {
+                                resolve({ [store]: { format: `array[${v.length}]`}})
+                            } else if (typeof v === 'object') {
+                                resolve({ [store]: { keys: Object.keys(v), format: 'Object' } })
+                            } else {
+                                resolve({ [store]: { keys: [] }})
+                            }
+                        }
                         var encoded = util.encodeObject({msg:v});
                         if (store !== availableStores.default) {
                             encoded.store = store;
@@ -118,32 +128,58 @@ var api = module.exports = {
                         stores = [store];
                     }
 
+
                     var result = {};
                     var c = stores.length;
                     var errorReported = false;
                     stores.forEach(function(store) {
-                        exportContextStore(scope,ctx,store,result,function(err) {
+                        if (opts.keysOnly) {
-                            if (err) {
+                            ctx.keys(store,function(err, keys) {
-                                // TODO: proper error reporting
+                                if (err) {
-                                if (!errorReported) {
+                                    // TODO: proper error reporting
-                                    errorReported = true;
+                                    if (!errorReported) {
-                                    runtime.log.audit({event: "context.get",scope:scope,id:id,store:store,key:key,error:"unexpected_error"}, opts.req);
+                                        errorReported = true;
-                                    var err = new Error();
+                                        runtime.log.audit({event: "context.get",scope:scope,id:id,store:store,key:key,error:"unexpected_error"}, opts.req);
-                                    err.code = "unexpected_error";
+                                        var err = new Error();
-                                    err.status = 400;
+                                        err.code = "unexpected_error";
-                                    return reject(err);
+                                        err.status = 400;
+                                        return reject(err);
+                                    }
+                                    return
                                 }
+                                result[store] = { keys }
+                                c--;
+                                if (c === 0) {
+                                    if (!errorReported) {
+                                        runtime.log.audit({event: "context.get",scope:scope,id:id,store:store,key:key},opts.req);
+                                        resolve(result);
+                                    }
+                                }
+                            })
+                        } else {
+                            exportContextStore(scope,ctx,store,result,function(err) {
+                                if (err) {
+                                    // TODO: proper error reporting
+                                    if (!errorReported) {
+                                        errorReported = true;
+                                        runtime.log.audit({event: "context.get",scope:scope,id:id,store:store,key:key,error:"unexpected_error"}, opts.req);
+                                        var err = new Error();
+                                        err.code = "unexpected_error";
+                                        err.status = 400;
+                                        return reject(err);
+                                    }
 
-                                return;
+                                    return;
-                            }
-                            c--;
-                            if (c === 0) {
-                                if (!errorReported) {
-                                    runtime.log.audit({event: "context.get",scope:scope,id:id,store:store,key:key},opts.req);
-                                    resolve(result);
                                 }
-                            }
+                                c--;
-                        });
+                                if (c === 0) {
+                                    if (!errorReported) {
+                                        runtime.log.audit({event: "context.get",scope:scope,id:id,store:store,key:key},opts.req);
+                                        resolve(result);
+                                    }
+                                }
+                            });
+                        }
                     })
                 }
             } else {