Allow regexes for filtered_attributes, filtered_cache and filtered_optimistic options (#13047)

* Ensure filtered_attributes and filtered_optimistic are also regex

Bring filtered_attributes and filtered_optimistic in line with the newer filtered_cache.
All three of them are now regex matches instead of absolute matches.

* update tests to use regex ^$ anchors

* utils: add filterProperties

lib/controller.ts

@@ -266,9 +266,7 @@ class Controller {
         }
 
         // Filter mqtt message attributes
-        if (entity.options.filtered_attributes) {
-            entity.options.filtered_attributes.forEach((a) => delete message[a]);
-        }
+        utils.filterProperties(entity.options.filtered_attributes, message);
 
         if (Object.entries(message).length) {
             const output = settings.get().advanced.output;

lib/extension/publish.ts

@@ -256,7 +256,8 @@ export default class Publish extends Extension {
                         }
 
                         // filter out attribute listed in filtered_optimistic
-                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        utils.filterProperties(entitySettings.filtered_optimistic, msg);
+
                         addToToPublish(re, msg);
                     }
 

lib/state.ts

@@ -1,6 +1,7 @@
 import logger from './util/logger';
 import data from './util/data';
 import * as settings from './util/settings';
+import utils from './util/utils';
 import fs from 'fs';
 import objectAssignDeep from 'object-assign-deep';
 
@@ -77,11 +78,7 @@ class State {
         const newCache = {...toState};
         const entityDontCacheProperties = entity.options.filtered_cache || [];
 
-        for (const property of Object.keys(newCache)) {
-            if (dontCacheProperties.concat(entityDontCacheProperties).find((p) => property.match(p))) {
-                delete newCache[property];
-            }
-        }
+        utils.filterProperties(dontCacheProperties.concat(entityDontCacheProperties), newCache);
 
         this.state[entity.ID] = newCache;
         this.eventBus.emitStateChange({entity, from: fromState, to: toState, reason, update});

lib/util/settings.schema.json

@@ -817,27 +817,27 @@
           "items": {
             "type": "string"
           },
-          "examples": ["temperature", "battery", "action"],
+          "examples": ["^temperature$", "^battery$", "^action$"],
           "title": "Filtered publish attributes",
-          "description": "Filter attributes from publish payload."
+          "description": "Filter attributes with regex from published payload."
         },
         "filtered_cache": {
           "type": "array",
           "items": {
             "type": "string"
           },
-          "examples": ["action", "input_actions"],
+          "examples": ["^input_actions$"],
           "title": "Filtered attributes from cache",
-          "description": "Filter attributes from being added to the cache, this prevents the attribute from being in the published payload when the value didn't change."
+          "description": "Filter attributes with regex from being added to the cache, this prevents the attribute from being in the published payload when the value didn't change."
         },
         "filtered_optimistic": {
           "type": "array",
           "items": {
             "type": "string"
           },
-          "examples": ["color_mode", "color_temp", "color"],
+          "examples": ["^color_(mode|temp)$", "color"],
           "title": "Filtered optimistic attributes",
-          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
+          "description": "Filter attributes with regex from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
         },
         "icon": {
           "type": "string",

lib/util/utils.ts

@@ -341,6 +341,15 @@ function publishLastSeen(data: eventdata.LastSeenChanged, settings: Settings, al
     }
 }
 
+function filterProperties(filterRegex: string[], data: KeyValue): void {
+    if (filterRegex) {
+        for (const property of Object.keys(data)) {
+            if (filterRegex.find((p) => property.match(p))) {
+                delete data[property];
+            }
+        }
+    }
+}
 
 export default {
     endpointNames, capitalize, getZigbee2MQTTVersion, getDependencyVersion, formatDate, objectHasProperties,
@@ -348,5 +357,5 @@ export default {
     getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
     parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
     sanitizeImageParameter, isAvailabilityEnabledForEntity, publishLastSeen, availabilityPayload,
-    getAllFiles,
+    getAllFiles, filterProperties,
 };

test/controller.test.js

@@ -465,7 +465,7 @@ describe('Controller', () => {
     it('Publish entity state attribute_json output filtered', async () => {
         await controller.start();
         settings.set(['experimental', 'output'], 'attribute_and_json');
-        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['color_temp', 'linkquality']);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['^color_temp$', '^linkquality$']);
         MQTT.publish.mockClear();
         const device = controller.zigbee.resolveEntity('bulb');
         await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
@@ -479,7 +479,7 @@ describe('Controller', () => {
     it('Publish entity state attribute_json output filtered (device_options)', async () => {
         await controller.start();
         settings.set(['experimental', 'output'], 'attribute_and_json');
-        settings.set(['device_options', 'filtered_attributes'], ['color_temp', 'linkquality']);
+        settings.set(['device_options', 'filtered_attributes'], ['^color_temp$', '^linkquality$']);
         MQTT.publish.mockClear();
         const device = controller.zigbee.resolveEntity('bulb');
         await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
@@ -493,7 +493,7 @@ describe('Controller', () => {
     it('Publish entity state attribute_json output filtered cache', async () => {
         await controller.start();
         settings.set(['advanced', 'output'], 'attribute_and_json');
-        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_cache'], ['linkquality']);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_cache'], ['^linkquality$']);
         MQTT.publish.mockClear();
 
         const device = controller.zigbee.resolveEntity('bulb');
@@ -513,7 +513,7 @@ describe('Controller', () => {
     it('Publish entity state attribute_json output filtered cache (device_options)', async () => {
         await controller.start();
         settings.set(['advanced', 'output'], 'attribute_and_json');
-        settings.set(['device_options', 'filtered_cache'], ['linkquality']);
+        settings.set(['device_options', 'filtered_cache'], ['^linkquality$']);
         MQTT.publish.mockClear();
 
         const device = controller.zigbee.resolveEntity('bulb');