1.14.0

Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -1,23 +1,38 @@
 <!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2mqtt version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
 Did you read the FAQ?
-https://www.zigbee2mqtt.io/information/FAQ.html
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2mqtt fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
 
 Unsupported device?
-https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
 
-A question?
-Read the documentation: https://www.zigbee2mqtt.io/
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
 
-Bug report:
-- First make sure you are on the latest version of zigbee2mqtt.
-- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
-- Provide a clear description of the problem.
+Bug report?
 - If applicable, provide steps how to reproduce the problem.
-- Provide a log with `log_level` set to `debug`, see https://www.zigbee2mqtt.io/configuration/configuration.html
-- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
 -->
 
 <!--Start your bug report below this line-->
+
 # Bug Report
 ## What happened
 
@@ -26,5 +41,6 @@ Bug report:
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-zigbee2mqtt version:
-CC253X firmware version:
+Zigbee2mqtt version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.gitignore

@@ -67,6 +67,7 @@ data-backup/
 data/coordinator_backup.json
 data/.storage
 data/database.db.backup
+data/extension
 
 # MacOS indexing file
 .DS_Store

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "*.yaml": "home-assistant"
+    }
+}

README.md

@@ -66,6 +66,9 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ## Architecture
 ![Architecture](images/architecture.png)
 
+### Internal Architecture
+Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

azure-pipelines.yaml

@@ -76,15 +76,13 @@ steps:
   displayName: 'Docker build release'
   condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
 
-# Trigger downstream
+# Trigger Home Assistant Zigbee2mqtt addon build
 - bash: |
-    BODY="{\"request\": {\"branch\": \"master\"}}"
-    curl -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "Accept: application/json" \
-        -H "Travis-API-Version: 3" \
-        -H "Authorization: token $(TRAVIS_TOKEN)" \
-        -d "$BODY" \
-        "https://api.travis-ci.org/repo/danielwelch%2Fhassio-zigbee2mqtt/requests"
+    curl -s \
+        --request POST \
+        --header "Content-Type: application/json" \
+        --data '{"definition": {"id": 1}}' \
+        -u "$(HASSIO_TRIGGER_SECRET)" \
+        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
   displayName: 'Trigger Hass.io build'
   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

lib/controller.js

@@ -6,21 +6,25 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const fs = require('fs');
+const data = require('./util/data');
+const path = require('path');
 
 // Extensions
-const ExtensionEntityPublish = require('./extension/entityPublish');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/softReset');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
-const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionBridgeLegacy = require('./extension/bridgeLegacy');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
-const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
-const ExtensionDeviceBind = require('./extension/deviceBind');
-const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionDeviceEvent = require('./extension/deviceEvent');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/report');
+const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
 
 class Controller {
@@ -36,24 +40,27 @@ class Controller {
         // Initialize extensions.
         const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionEntityPublish(...args),
-            new ExtensionDeviceReceive(...args),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
-            new ExtensionDeviceConfigure(...args),
+            new ExtensionConfigure(...args),
             new ExtensionNetworkMap(...args),
             new ExtensionGroups(...args),
-            new ExtensionDeviceBind(...args),
-            new ExtensionDeviceEvent(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
             new ExtensionOTAUpdate(...args),
         ];
 
-        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.extensions.push(new ExtensionBridge(...args));
+        }
+
         if (settings.get().advanced.legacy_api) {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
 
         if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(...args));
+            this.extensions.push(new ExtensionReport(...args));
         }
 
         if (settings.get().homeassistant) {
@@ -66,7 +73,16 @@ class Controller {
         }
 
         if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(...args));
+            this.extensions.push(new ExtensionAvailability(...args));
+        }
+
+        const extensionPath = data.joinPath('extension');
+        if (fs.existsSync(extensionPath)) {
+            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
+            for (const extension of extensions) {
+                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
+                this.extensions.push(new Extension(...args, settings, logger));
+            }
         }
     }
 
@@ -173,7 +189,7 @@ class Controller {
 
     async onZigbeeEvent(type, data) {
         const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !resolvedEntity.settings) {
+        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
             resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
         }
@@ -237,12 +253,6 @@ class Controller {
             return;
         }
 
-        const isDevice = resolvedEntity.type === 'device';
-
-        if (isDevice && settings.get().advanced.last_seen !== 'disable' && resolvedEntity.device.lastSeen) {
-            payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, settings.get().advanced.last_seen);
-        }
-
         let messagePayload = {...payload};
         const currentState = this.state.exists(resolvedEntity.settings.ID) ?
             this.state.get(resolvedEntity.settings.ID) : {};
@@ -267,6 +277,7 @@ class Controller {
             options.properties = {messageExpiryInterval: retention};
         }
 
+        const isDevice = resolvedEntity.type === 'device';
         if (isDevice && settings.get().mqtt.include_device_information) {
             const attributes = [
                 'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
@@ -281,6 +292,11 @@ class Controller {
             attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
         }
 
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+        }
+
         // filter mqtt message attributes
         if (deviceOptions.filtered_attributes) {
             deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
@@ -289,8 +305,6 @@ class Controller {
             resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity});
-
         if (Object.entries(messagePayload).length) {
             if (settings.get().experimental.output === 'attribute_and_json') {
                 await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
@@ -304,6 +318,8 @@ class Controller {
                 }
             }
         }
+
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {

lib/extension/availability.js

@@ -2,26 +2,21 @@ const logger = require('../util/logger');
 const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-// Some EndDevices should be pinged
+// Pingable end devices, some end devices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableModels = [
-    'E11-G13',
-    '53170161',
+const pingableEndDevices = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
 ];
 
-const forcedPingable =
-    zigbeeHerdsmanConverters.devices.filter((d) => pingableModels.includes(d.model));
-
-const toZigbeeCandidates = ['state', 'brightness', 'color', 'color_temp'];
-
 const Hours25 = 1000 * 60 * 60 * 25;
 
 /**
  * This extensions pings devices to check if they are online.
  */
-class DeviceAvailability extends BaseExtension {
+class Availability extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -29,22 +24,14 @@ class DeviceAvailability extends BaseExtension {
         this.timers = {};
         this.state = {};
 
-        // Initialize blacklist
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
-
-        // Initialize whitelist
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
+        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
+        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
     }
 
-    isAllowed(device) {
+    inWhitelistOrNotInBlacklist(device) {
         const ieeeAddr = device.ieeeAddr;
-
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings.friendly_name;
+        const name = deviceSettings.friendlyName;
 
         // Whitelist is not empty and device is in it, enable availability
         if (this.whitelist.length > 0) {
@@ -60,27 +47,25 @@ class DeviceAvailability extends BaseExtension {
     }
 
     isPingable(device) {
-        // Device is on forcedPingable-list, enable availability
-        if (forcedPingable.find((d) => d.zigbeeModel.includes(device.modelID))) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
             return true;
         }
 
         // Device is a mains powered router
-        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
-
-        return result;
+        return utils.isRouter(device) && !utils.isBatteryPowered(device);
     }
 
     onMQTTConnected() {
         for (const device of this.zigbee.getClients()) {
             // Mark all devices as online on start
-            this.publishAvailability(device, true);
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
 
-            if (this.isAllowed(device)) {
+            if (this.inWhitelistOrNotInBlacklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
-                    this.timers[device.ieeeAddr] = setInterval(() => {
+                    this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
                     }, utils.secondsToMilliseconds(300));
                 }
@@ -90,34 +75,33 @@ class DeviceAvailability extends BaseExtension {
 
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity) {
             logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
             return;
         }
 
-        const ieeeAddr = device.ieeeAddr;
-        const level = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
         try {
             await device.ping();
             this.publishAvailability(device, true);
-            logger.debug(`Successfully pinged '${entity.name}'`);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
         } catch (error) {
             this.publishAvailability(device, false);
-            logger[level](`Failed to ping '${entity.name}'`);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
         } finally {
             this.setTimerPingable(device);
         }
     }
 
     async handleIntervalNotPingable(device) {
-        const ago = Date.now() - device.lastSeen;
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity || !device.lastSeen) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
             return;
         }
 
-        logger.debug(`Non-pingable device '${entity.name}' was last seen '${ago / 1000}' seconds ago.`);
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
 
         if (ago > Hours25) {
             this.publishAvailability(device, false);
@@ -143,38 +127,34 @@ class DeviceAvailability extends BaseExtension {
     }
 
     async onReconnect(device) {
-        if (device && device.modelID) {
-            const definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-
-            if (definition) {
-                const used = [];
-                try {
-                    for (const key of toZigbeeCandidates) {
-                        const converter = definition.toZigbee.find((tz) => tz.key.includes(key));
-                        if (converter && !used.includes(converter)) {
-                            await converter.convertGet(device.endpoints[0], key, {});
-                            used.push(converter);
-                        }
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            const used = [];
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter && !used.includes(converter)) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                        used.push(converter);
                     }
-                } catch (error) {
-                    const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-                    logger.error(`Failed to read state of '${entity.name}' after reconnect`);
                 }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
             }
         }
     }
 
-    publishAvailability(device, available) {
+    publishAvailability(device, available, force=false) {
         const ieeeAddr = device.ieeeAddr;
         if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
             this.onReconnect(device);
         }
 
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
+        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
         const topic = `${name}/availability`;
         const payload = available ? 'online' : 'offline';
-        if (this.state[ieeeAddr] !== available) {
+        if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;
             this.mqtt.publish(topic, payload, {retain: true, qos: 0});
         }
@@ -186,7 +166,7 @@ class DeviceAvailability extends BaseExtension {
             return;
         }
 
-        if (this.isAllowed(device)) {
+        if (this.inWhitelistOrNotInBlacklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {
@@ -212,4 +192,4 @@ class DeviceAvailability extends BaseExtension {
     }
 }
 
-module.exports = DeviceAvailability;
+module.exports = Availability;

lib/extension/bind.js

@@ -1,30 +1,46 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
 const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-const BaseExtension = require('./baseExtension');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const Extension = require('./extension');
 
 const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
 
 // See zigbee-herdsman-converters devices.js
 const defaultBindGroup = {type: 'group_number', ID: 901};
 
-class DeviceBind extends BaseExtension {
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        }
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        }
+
+        return {type, sourceKey, targetKey};
     }
 
     async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Parse topic, retrieve type (bind or unbind) and source
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-        const type = topic.split('/')[0];
-        const sourceKey = topic.replace(`${type}/`, '');
-        const targetKey = message;
+        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
 
         // Find source; can only be a device and target
         const source = this.zigbee.resolveEntity(sourceKey);
@@ -64,10 +80,10 @@ class DeviceBind extends BaseExtension {
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const message = {from: sourceName, to: targetName, cluster};
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_${type}`, message}),
+                            JSON.stringify({type: `device_${type}`,
+                                message: {from: sourceName, to: targetName, cluster}}),
                         );
                     }
                 } catch (error) {
@@ -78,10 +94,10 @@ class DeviceBind extends BaseExtension {
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const message = {from: sourceName, to: targetName, cluster};
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_${type}_failed`, message}),
+                            JSON.stringify({type: `device_${type}_failed`,
+                                message: {from: sourceName, to: targetName, cluster}}),
                         );
                     }
                 }
@@ -93,14 +109,13 @@ class DeviceBind extends BaseExtension {
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const message = {from: sourceName, to: targetName};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `device_${type}_failed`, message}),
+                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
                 );
             }
         }
     }
 }
 
-module.exports = DeviceBind;
+module.exports = Bind;

lib/extension/bridge.js

@@ -0,0 +1,211 @@
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+
+        this.requestLookup = {
+            'permitjoin': this.permitJoin.bind(this),
+            // 'device/remove': this.deviceRemove.bind(this),
+            // 'device/forceremove': this.deviceForceRemove.bind(this),
+            // 'device/ban': this.deviceBan.bind(this),
+            // 'group/remove': this.groupRemove.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = JSON.parse(message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') payload = {friendlyName: resolvedEntity.settings.friendlyName, ieeeAddress};
+            else if (type === 'deviceInterview') {
+                payload = {friendlyName: resolvedEntity.settings.friendlyName, status: data.status, ieeeAddress};
+                if (data.status === 'successful') {
+                    payload.supported = !!resolvedEntity.definition;
+                    payload.definition = resolvedEntity.definition ? {
+                        model: resolvedEntity.definition.model,
+                        vendor: resolvedEntity.definition.vendor,
+                        description: resolvedEntity.definition.description,
+                        supports: resolvedEntity.definition.supports,
+                    } : null;
+                }
+            } else payload = {ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish('bridge/event', JSON.stringify({type, data: payload}), {retain: false, qos: 0});
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    // async deviceRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('remove', 'device', message);
+    // }
+
+    // async deviceForceRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('force_remove', 'device', message);
+    // }
+
+    // async deviceBan(message) {
+    //     return this.removeForceRemoveOrBanEntity('ban', 'device', message);
+    // }
+
+    // async groupRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('remove', 'group', message);
+    // }
+
+    async permitJoin(message) {
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.permitJoin(value);
+        await this.publishInfo();
+        return utils.getResponse(message, {value: value}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    // async removeForceRemoveOrBanEntity(action, entityType, message) {
+    //     const ID = typeof message === 'object' ? message.ID : message.trim();
+    //     const entity = this.zigbee.resolveEntity(ID);
+    //     if (!entity || entity.type !== entityType) {
+    //         throw new Error(`${ID} is not a ${entityType}`);
+    //     }
+
+    //     const lookup = {
+    //         ban: ['banned', 'Banning', 'ban'],
+    //         force_remove: ['force_removed', 'Force removing', 'force remove'],
+    //         remove: ['removed', 'Removing', 'remove'],
+    //     };
+
+    //     try {
+    //         logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+    //         if (entity.type === 'device') {
+    //             if (action === 'ban') {
+    //                 settings.banDevice(entity.settings.ID);
+    //             }
+
+    //             action === 'force_remove' ?
+    //                 await entity.device.removeFromDatabase() : await entity.device.removeFromNetwork();
+    //         } else {
+    //             await entity.group.removeFromDatabase();
+    //         }
+
+    //         // Fire event
+    //         if (entity.type === 'device') {
+    //             this.eventBus.emit('deviceRemoved', {device: entity.device});
+    //         }
+
+    //         // Remove from configuration.yaml
+    //         entity.type === 'device' ?
+    //             settings.removeDevice(entity.settings.ID) : settings.removeGroup(entity.settings.ID);
+
+    //         // Remove from state
+    //         this.state.remove(entity.settings.ID);
+
+    //         logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+    //         entity.type === 'device' ? this.publishDevices() : this.publishGroups();
+    //         return utils.getResponse(message, {ID}, null);
+    //     } catch (error) {
+    //         throw new Error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+    //     }
+    // }
+
+    async publishInfo() {
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            logLevel: logger.getLevel(),
+            permitJoin: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    async publishDevices(topic, message) {
+        const devices = this.zigbee.getClients().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const definitionPayload = definition ? {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                supports: definition.supports,
+            } : null;
+
+            return {
+                ieeeAddress: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+                supported: !!definition,
+                friendlyName: resolved.name,
+                definition: definitionPayload,
+                powerSource: device.powerSource,
+                softwareBuildID: device.softwareBuildID,
+                dateCode: device.dateCode,
+                interviewing: device.interviewing,
+                interviewCompleted: device.interviewCompleted,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+    }
+
+    async publishGroups(topic, message) {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                ID: group.groupID,
+                friendlyName: resolved.name,
+                members: group.members.map((m) => {
+                    return {
+                        ieeeAddress: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+    }
+}
+
+module.exports = Bridge;

lib/extension/configure.js

@@ -1,14 +1,19 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class DeviceConfigure extends BaseExtension {
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
         this.configuring = new Set();
         this.attempts = {};
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
     }
 
     shouldConfigure(resolvedEntity) {
@@ -30,26 +35,32 @@ class DeviceConfigure extends BaseExtension {
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        if (topic !== this.topic) {
-            return;
-        }
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (topic !== this.legacyTopic) {
+                return;
+            }
 
-        const resolvedEntity = this.zigbee.resolveEntity(message);
-        if (!resolvedEntity || resolvedEntity.type !== 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            return;
-        }
+            const resolvedEntity = this.zigbee.resolveEntity(message);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                logger.error(`Device '${message}' does not exist`);
+                return;
+            }
 
-        if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
-            logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
-            return;
-        }
+            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+                return;
+            }
 
-        this.configure(resolvedEntity, true);
+            this.configure(resolvedEntity, true);
+        }
     }
 
     async onZigbeeStarted() {
@@ -106,4 +117,4 @@ class DeviceConfigure extends BaseExtension {
     }
 }
 
-module.exports = DeviceConfigure;
+module.exports = Configure;

lib/extension/deviceEvent.js

@@ -1,36 +0,0 @@
-const BaseExtension = require('./baseExtension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-
-class DeviceEvent extends BaseExtension {
-    async onZigbeeStarted() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'start', {});
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        if (data.device && resolvedEntity.definition) {
-            this.callOnEvent(data.device, type, data, resolvedEntity.definition);
-        }
-    }
-
-    async stop() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'stop', {});
-        }
-    }
-
-    callOnEvent(device, type, data, definition) {
-        zigbeeHerdsmanConverters.onEvent(type, data, device);
-
-        if (!definition) {
-            definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-        }
-
-        if (definition && definition.onEvent) {
-            definition.onEvent(type, data, device);
-        }
-    }
-}
-
-module.exports = DeviceEvent;

lib/extension/extension.js

@@ -1,4 +1,4 @@
-class BaseExtension {
+class Extension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -49,4 +49,4 @@ class BaseExtension {
     // stop() {}
 }
 
-module.exports = BaseExtension;
+module.exports = Extension;

lib/extension/groups.js

@@ -1,26 +1,30 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 const utils = require('../util/utils');
 const postfixes = utils.getEndpointNames();
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
-const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups extends BaseExtension {
+class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
 
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}remove`);
-            this.mqtt.subscribe(`${topic}add`);
-            this.mqtt.subscribe(`${topic}remove_all`); // DEPRECATED
+            for (let step = 1; step < 20; step++) {
+                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
+                this.mqtt.subscribe(`${topic}remove`);
+                this.mqtt.subscribe(`${topic}add`);
+                this.mqtt.subscribe(`${topic}remove_all`);
+            }
         }
     }
 
@@ -87,33 +91,33 @@ class Groups extends BaseExtension {
         });
 
         if (Object.keys(payload).length) {
-            const entity = this.zigbee.resolveEntity(data.ID);
+            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
                 return settingsGroup && settingsGroup.optimistic;
             });
 
-            if (entity.type === 'device') {
+            if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(entity.endpoint)) {
-                        if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }
                 }
             } else {
                 const groupIDsToPublish = new Set();
-                for (const member of entity.group.members) {
+                for (const member of resolvedEntity.group.members) {
                     await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
                     for (const zigbeeGroup of zigbeeGroups) {
                         if (zigbeeGroup.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                                 groupIDsToPublish.add(zigbeeGroup.groupID);
                             }
                         }
                     }
                 }
-                groupIDsToPublish.delete(entity.group.groupID);
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
                 for (const groupID of groupIDsToPublish) {
                     await this.publishEntityState(groupID, payload, reason);
                 }
@@ -121,7 +125,7 @@ class Groups extends BaseExtension {
         }
     }
 
-    allMembersOff(zigbeeGroup) {
+    areAllMembersOff(zigbeeGroup) {
         for (const member of zigbeeGroup.members) {
             const device = member.getDevice();
             if (this.state.exists(device.ieeeAddr)) {
@@ -134,105 +138,121 @@ class Groups extends BaseExtension {
         return true;
     }
 
-    async onMQTTMessage(topic, message) {
-        let type;
-        let group;
-        const topicMatch = topic.match(topicRegex);
-        if (topicMatch) {
-            group = this.zigbee.resolveEntity(topicMatch[1]);
-            type = topicMatch[2];
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
 
-            if (!group || group.type !== 'group') {
-                logger.error(`Group '${topicMatch[1]}' does not exist`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            const topicMatch = topic.match(legacyTopicRegex);
+            if (topicMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
+                type = topicMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else if (topic.match(legacyTopicRegexRemoveAll)) {
+                type = 'remove_all';
+            } else {
+                return {};
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
                     this.mqtt.publish(
                         'bridge/log',
                         JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
                     );
                 }
 
-                return;
-            }
-        } else if (topic.match(topicRegexRemoveAll)) {
-            type = 'remove_all';
-        } else {
-            return;
-        }
-
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || !entity.type === 'device') {
-            logger.error(`Device '${message}' does not exist`);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
-                );
+                return {};
             }
 
-            return;
+            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
         }
 
+        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
+            this.parseMQTTMessage(topic, message);
+        if (!type) return;
+
         const keys = [
-            `${entity.device.ieeeAddr}/${entity.endpoint.ID}`,
-            `${entity.name}/${entity.endpoint.ID}`,
+            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
         ];
 
-        const definition = entity.definition;
-        const endpoints = definition && definition.endpoint ? definition.endpoint(entity.device) : null;
-        const endpointName = endpoints ? Object.entries(endpoints).find((e) => e[1] === entity.endpoint.ID)[0] : null;
+        const definition = resolvedEntityDevice.definition;
+        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
+        const endpointName = endpoints ?
+            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
 
         if (endpointName) {
-            keys.push(`${entity.device.ieeeAddr}/${endpointName}`);
-            keys.push(`${entity.name}/${endpointName}`);
+            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
         }
 
-        const hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
         if (!hasEndpointName) {
-            keys.push(entity.name);
-            keys.push(entity.device.ieeeAddr);
+            keys.push(resolvedEntityDevice.name);
+            keys.push(resolvedEntityDevice.device.ieeeAddr);
         }
 
         if (type === 'add') {
-            logger.info(`Adding '${entity.name}' to '${group.name}'`);
-            await entity.endpoint.addToGroup(group.group);
-            settings.addDeviceToGroup(group.settings.ID, keys);
+            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: entity.name, group: group.name};
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
                 this.mqtt.publish(
                     'bridge/log',
                     JSON.stringify({type: `device_group_add`, message: payload}),
                 );
             }
         } else if (type === 'remove') {
-            logger.info(`Removing '${entity.name}' from '${group.name}'`);
-            await entity.endpoint.removeFromGroup(group.group);
-            settings.removeDeviceFromGroup(group.settings.ID, keys);
+            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: entity.name, group: group.name};
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
                 this.mqtt.publish(
                     'bridge/log',
                     JSON.stringify({type: `device_group_remove`, message: payload}),
                 );
             }
         } else { // remove_all
-            logger.info(`Removing '${entity.name}' from all groups`);
-            await entity.endpoint.removeFromAllGroups();
+            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+            await resolvedEntityDevice.endpoint.removeFromAllGroups();
             for (const settingsGroup of settings.getGroups()) {
                 settings.removeDeviceFromGroup(settingsGroup.ID, keys);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: entity.name};
+                    const payload = {friendly_name: resolvedEntityDevice.name};
                     this.mqtt.publish(
                         'bridge/log',
                         JSON.stringify({type: `device_group_remove_all`, message: payload}),

lib/extension/legacy/bridgeLegacy.js

@@ -1,15 +1,15 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const utils = require('../util/utils');
+const utils = require('../../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('../extension');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
 const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
 
-class BridgeLegacy extends BaseExtension {
+class BridgeLegacy extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -150,7 +150,7 @@ class BridgeLegacy extends BaseExtension {
             };
 
             if (device.type !== 'Coordinator') {
-                const definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
                 const friendlyDevice = settings.getDevice(device.ieeeAddr);
                 payload.model = definition ? definition.model : device.modelID;
                 payload.vendor = definition ? definition.vendor : '-';
@@ -192,7 +192,8 @@ class BridgeLegacy extends BaseExtension {
     }
 
     rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} got ${message}`;
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
 
         let json = null;
         try {

lib/extension/legacy/deviceGroupMembership.js

@@ -1,11 +1,11 @@
 /* istanbul ignore file */
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
 
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
-class DeviceGroupMembership extends BaseExtension {
+class DeviceGroupMembership extends Extension {
     onMQTTConnected() {
         for (let step = 1; step < 20; step++) {
             const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;

lib/extension/legacy/softReset.js

@@ -1,14 +1,14 @@
 /* istanbul ignore file */
 // DEPRECATED
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
 
 /**
  * This extensions soft resets the ZNP after a certain timeout.
  */
-class SoftReset extends BaseExtension {
+class SoftReset extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;

lib/extension/networkMap.js

@@ -1,16 +1,18 @@
 const settings = require('../util/settings');
 const utils = require('../util/utils');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class NetworkMap extends BaseExtension {
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
-        // Subscribe to topic.
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.topicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
 
         // Bind
         this.raw = this.raw.bind(this);
@@ -24,16 +26,23 @@ class NetworkMap extends BaseExtension {
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-        this.mqtt.subscribe(this.topicRoutes);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+            this.mqtt.subscribe(this.legacyTopicRoutes);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        if ((topic === this.topic || topic === this.topicRoutes) && this.supportedFormats.hasOwnProperty(message)) {
-            const includeRoutes = topic === this.topicRoutes;
-            const topology = await this.networkScan(includeRoutes);
-            const converted = this.supportedFormats[message](topology);
-            this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(message)) {
+                const includeRoutes = topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                const converted = this.supportedFormats[message](topology);
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            }
         }
     }
 
@@ -47,32 +56,32 @@ class NetworkMap extends BaseExtension {
         let text = 'digraph G {\nnode[shape=record];\n';
         let style = '';
 
-        topology.nodes.forEach((device) => {
+        topology.nodes.forEach((node) => {
             const labels = [];
 
             // Add friendly name
-            labels.push(`${device.friendlyName}`);
+            labels.push(`${node.friendlyName}`);
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${device.ieeeAddr} (${device.networkAddress})` +
-                ((device.failed && device.failed.length) ? `failed: ${device.failed.join(',')}` : ''),
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
-            if (device.type !== 'Coordinator') {
-                const mappedModel = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                if (mappedModel) {
-                    labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
+            if (node.type !== 'Coordinator') {
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
-                    labels.push(`${device.manufacturerName} ${device.modelID}`);
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
                 }
             }
 
             // Add the device last_seen timestamp
             let lastSeen = 'unknown';
-            const date = device.type === 'Coordinator' ? Date.now() : device.lastSeen;
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
                 lastSeen = utils.formatDate(date, 'ISO_8601_local');
             }
@@ -80,10 +89,10 @@ class NetworkMap extends BaseExtension {
             labels.push(lastSeen);
 
             // Shape the record according to device type
-            if (device.type == 'Coordinator') {
+            if (node.type == 'Coordinator') {
                 style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
                     `fontcolor="${colors.font.coordinator}"`;
-            } else if (device.type == 'Router') {
+            } else if (node.type == 'Router') {
                 style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
                     `fontcolor="${colors.font.router}"`;
             } else {
@@ -92,22 +101,22 @@ class NetworkMap extends BaseExtension {
             }
 
             // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
 
             /**
              * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.links.filter((e) => (e.source.ieeeAddr === device.ieeeAddr)).forEach((e) => {
-                const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, ' :
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
                 const textRoutes = e.routes.map((r) => r.destinationAddress);
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
-                text += `  "${device.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
                 text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
             });
         });
@@ -119,31 +128,31 @@ class NetworkMap extends BaseExtension {
 
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
-        const devices = this.zigbee.getDevices();
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
         const lqis = new Map();
         const routingTables = new Map();
         const failed = new Map();
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             try {
                 const result = await device.lqi();
                 lqis.set(device, result);
-                logger.debug(`LQI succeeded for '${resolved.name}'`);
+                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
                 failed.get(device).push('lqi');
-                logger.error(`Failed to execute LQI for '${resolved.name}'`);
+                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
             }
 
             if (includeRoutes) {
                 try {
                     const result = await device.routingTable();
                     routingTables.set(device, result);
-                    logger.debug(`Routing table succeeded for '${resolved.name}'`);
+                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {
                     failed.get(device).push('routingTable');
-                    logger.error(`Failed to execute routing table for '${resolved.name}'`);
+                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
                 }
             }
         }
@@ -153,9 +162,9 @@ class NetworkMap extends BaseExtension {
         const networkMap = {nodes: [], links: []};
         // Add nodes
         for (const device of devices) {
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             networkMap.nodes.push({
-                ieeeAddr: device.ieeeAddr, friendlyName: resolved.name, type: device.type,
+                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
             });

lib/extension/onEvent.js

@@ -0,0 +1,37 @@
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends Extension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && resolvedEntity.type === 'device') {
+            this.callOnEvent(resolvedEntity, type, data);
+        }
+    }
+
+    async stop() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'stop', {});
+        }
+    }
+
+    callOnEvent(resolvedEntity, type, data) {
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+
+        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.js

@@ -1,20 +1,24 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
 const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
-const BaseExtension = require('./baseExtension');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const Extension = require('./extension');
 const MINUTES_10 = 1000 * 60 * 10;
 
-class OTAUpdate extends BaseExtension {
+class OTAUpdate extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.inProgress = new Set();
         this.lastChecked = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        }
     }
 
     async onZigbeeEvent(type, data, resolvedEntity) {
@@ -52,7 +56,10 @@ class OTAUpdate extends BaseExtension {
         // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
         // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
         const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
-        await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
     }
 
     async readSoftwareBuildIDAndDateCode(device, update) {
@@ -73,19 +80,26 @@ class OTAUpdate extends BaseExtension {
     }
 
     async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
+        let resolvedEntity = null;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (!topic.match(legacyTopicRegex)) {
+                return null;
+            }
+
+            resolvedEntity = this.zigbee.resolveEntity(message);
+        } else {
             return null;
         }
 
-        const device = this.zigbee.resolveEntity(message);
-        assert(device != null && device.type === 'device', 'Device not found or not a device');
-        if (!device.definition || !device.definition.ota) {
-            const message = `Device '${device.name}' does not support OTA updates`;
+        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
+        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
             logger.error(message);
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const meta = {status: `not_supported`, device: device.name};
+                const meta = {status: `not_supported`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
                     JSON.stringify({type: `ota_update`, message, meta}),
@@ -95,20 +109,20 @@ class OTAUpdate extends BaseExtension {
             return;
         }
 
-        if (this.inProgress.has(device.device.ieeeAddr)) {
-            logger.error(`Update or check already in progress for '${device.name}', skipping...`);
+        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
             return;
         }
-        this.inProgress.add(device.device.ieeeAddr);
+        this.inProgress.add(resolvedEntity.device.ieeeAddr);
 
         const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
         if (type === 'check') {
-            const message = `Checking if update available for '${device.name}'`;
+            const message = `Checking if update available for '${resolvedEntity.name}'`;
             logger.info(message);
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const meta = {status: `checking_if_available`, device: device.name};
+                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
                     JSON.stringify({type: `ota_update`, message, meta}),
@@ -116,29 +130,28 @@ class OTAUpdate extends BaseExtension {
             }
 
             try {
-                const available = await device.definition.ota.isUpdateAvailable(device.device, logger);
-                const message=(available ?
-                    `Update available for '${device.name}'` : `No update available for '${device.name}'`);
+                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
+                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
                 logger.info(message);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: available ? 'available' : 'not_available', device: device.name};
+                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
                         JSON.stringify({type: `ota_update`, message, meta}),
                     );
                 }
 
-                this.publishEntityState(device.device.ieeeAddr, {update_available: available});
-                this.lastChecked[device.device.ieeeAddr] = Date.now();
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
+                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
             } catch (error) {
-                const message = `Failed to check if update available for '${device.name}' (${error.message})`;
+                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
                 logger.error(message);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `check_failed`, device: device.name};
+                    const meta = {status: `check_failed`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
                         JSON.stringify({type: `ota_update`, message, meta}),
@@ -146,12 +159,12 @@ class OTAUpdate extends BaseExtension {
                 }
             }
         } else { // type === 'update'
-            const message = `Updating '${device.name}' to latest firmware`;
+            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
             logger.info(message);
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const meta = {status: `update_in_progress`, device: device.name};
+                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
                     JSON.stringify({type: `ota_update`, message, meta}),
@@ -160,7 +173,7 @@ class OTAUpdate extends BaseExtension {
 
             try {
                 const onProgress = (progress, remaining) => {
-                    let message = `Update of '${device.name}' at ${progress}%`;
+                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
                     if (remaining) {
                         message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
                     }
@@ -169,37 +182,38 @@ class OTAUpdate extends BaseExtension {
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_progress`, device: device.name, progress};
+                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
                         this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
                     }
                 };
 
-                const from_ = await this.readSoftwareBuildIDAndDateCode(device.device, false);
-                await device.definition.ota.updateToLatest(device.device, logger, onProgress);
-                const to = await this.readSoftwareBuildIDAndDateCode(device.device, true);
+                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
                 const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
-                const message = `Finished update of '${device.name}'` + (to ? `, from '${fromS}' to '${toS}'` : ``);
+                const message = `Finished update of '${resolvedEntity.name}'` +
+                    (to ? `, from '${fromS}' to '${toS}'` : ``);
                 logger.info(message);
-                this.publishEntityState(device.device.ieeeAddr, {update_available: false});
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
+                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
                     this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
                 }
             } catch (error) {
-                const message = `Update of '${device.name}' failed (${error.message})`;
+                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
                 logger.error(message);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_failed`, device: device.name};
+                    const meta = {status: `update_failed`, device: resolvedEntity.name};
                     this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
                 }
             }
         }
 
-        this.inProgress.delete(device.device.ieeeAddr);
+        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
     }
 }
 

lib/extension/publish.js

@@ -4,10 +4,9 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-const postfixes = utils.getEndpointNames();
-const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
 
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
@@ -19,9 +18,10 @@ const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
     zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
 ];
 
-class EntityPublish extends BaseExtension {
+class EntityPublish extends Extension {
     onMQTTConnected() {
         // Subscribe to topics.
         const baseTopic = settings.get().mqtt.base_topic;
@@ -46,7 +46,7 @@ class EntityPublish extends BaseExtension {
             return null;
         }
 
-        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
+        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
     }
 
     async onMQTTMessage(topic, message) {
@@ -55,20 +55,20 @@ class EntityPublish extends BaseExtension {
             return false;
         }
 
-        const entityName = `${topic.ID}` + (topic.postfix ? `/${topic.postfix}` : '');
-        const entity = this.zigbee.resolveEntity(entityName);
+        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
+        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
 
-        if (!entity) {
+        if (!resolvedEntity) {
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const message = {friendly_name: entityName};
+                const message = {friendly_name: entityKey};
                 this.mqtt.publish(
                     'bridge/log',
                     JSON.stringify({type: `entity_not_found`, message}),
                 );
             }
 
-            logger.error(`Entity '${entityName}' is unknown`);
+            logger.error(`Entity '${entityKey}' is unknown`);
             return;
         }
 
@@ -77,27 +77,26 @@ class EntityPublish extends BaseExtension {
         let target = null;
         let options = {};
         let device = null;
-        let mapped = null;
+        let definition = null;
 
-        assert(entity.type === 'device' || entity.type === 'group');
-        if (entity.type === 'device') {
-            // Map device to a model
-            if (!entity.definition) {
-                logger.warn(`Device with modelID '${entity.device.modelID}' is not supported.`);
+        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
+        if (resolvedEntity.type === 'device') {
+            if (!resolvedEntity.definition) {
+                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
                 return;
             }
 
-            device = entity.device;
-            mapped = entity.definition;
-            target = entity.endpoint;
-            converters = entity.definition.toZigbee;
-            options = entity.settings;
+            device = resolvedEntity.device;
+            definition = resolvedEntity.definition;
+            target = resolvedEntity.endpoint;
+            converters = resolvedEntity.definition.toZigbee;
+            options = resolvedEntity.settings;
         } else {
             converters = groupConverters;
-            target = entity.group;
-            options = entity.settings;
-            mapped = entity.group.members.map((e) => zigbeeHerdsmanConverters.findByZigbeeModel(e.getDevice().modelID));
+            target = resolvedEntity.group;
+            options = resolvedEntity.settings;
+            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
         }
 
         // Convert the MQTT message to a Zigbee message.
@@ -113,12 +112,13 @@ class EntityPublish extends BaseExtension {
             }
         }
 
+
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(entity.settings.ID) || {};
+        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
@@ -130,31 +130,47 @@ class EntityPublish extends BaseExtension {
             }
         }
 
-        // Ensure that state and brightness are executed before other commands.
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
         const entries = Object.entries(json);
-        entries.sort((a, b) => (['state', 'brightness'].includes(a[0]) ? -1 : 1));
+        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
 
         // For each attribute call the corresponding converter
-        const usedConverters = [];
+        const usedConverters = {};
         for (let [key, value] of entries) {
-            let postfix = topic.postfix;
+            let endpointName = topic.endpointName;
             let actualTarget = target;
 
-            // When the key has a postfix included (e.g. state_right), this will override the target.
-            if (entity.type === 'device' && key.includes('_')) {
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            if (resolvedEntity.type === 'device' && key.includes('_')) {
                 const underscoreIndex = key.lastIndexOf('_');
-                const possiblePostfix = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getEndpointNames().includes(possiblePostfix)) {
-                    postfix = possiblePostfix;
+                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
+                if (utils.getEndpointNames().includes(possibleEndpointName)) {
+                    endpointName = possibleEndpointName;
                     key = key.substring(0, underscoreIndex);
                     const device = target.getDevice();
-                    actualTarget = device.getEndpoint(mapped.endpoint(device)[postfix]);
+                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+
+                    if (!actualTarget) {
+                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
+                        continue;
+                    }
                 }
             }
 
+            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters.includes(converter)) {
+            if (usedConverters[endpointOrGroupID].includes(converter)) {
                 // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
@@ -166,32 +182,32 @@ class EntityPublish extends BaseExtension {
 
             // Converter didn't return a result, skip
             const meta = {
-                endpoint_name: postfix,
+                endpoint_name: endpointName,
                 options,
                 message: json,
                 logger,
                 device,
                 state: deviceState,
-                mapped,
+                mapped: definition,
             };
 
             try {
                 if (topic.type === 'set' && converter.convertSet) {
-                    logger.debug(`Publishing '${topic.type}' '${key}' to '${entity.name}'`);
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
                     if (result && result.state) {
                         const msg = result.state;
 
-                        if (postfix) {
-                            for (const key of ['state', 'brightness']) {
+                        if (endpointName) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                                 if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${postfix}`] = msg[key];
+                                    msg[`${key}_${endpointName}`] = msg[key];
                                     delete msg[key];
                                 }
                             }
                         }
 
-                        this.publishEntityState(entity.settings.ID, msg);
+                        this.publishEntityState(resolvedEntity.settings.ID, msg);
                     }
 
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
@@ -202,13 +218,13 @@ class EntityPublish extends BaseExtension {
                     // ever issue a read here, as we assume the device will properly report changes.
                     // Only do this when the retrieve_state option is enabled for this device.
                     if (
-                        entity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
-                        entity.settings.retrieve_state
+                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        resolvedEntity.settings.retrieve_state
                     ) {
                         setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
                     }
                 } else if (topic.type === 'get' && converter.convertGet) {
-                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${entity.name}'`);
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     await converter.convertGet(actualTarget, key, meta);
                 } else {
                     logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
@@ -216,13 +232,13 @@ class EntityPublish extends BaseExtension {
                 }
             } catch (error) {
                 const message =
-                    `Publish '${topic.type}' '${key}' to '${entity.name}' failed: '${error}'`;
+                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
                 logger.error(message);
                 logger.debug(error.stack);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {friendly_name: entity.name};
+                    const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
                         JSON.stringify({type: `zigbee_publish_error`, message, meta}),
@@ -230,7 +246,7 @@ class EntityPublish extends BaseExtension {
                 }
             }
 
-            usedConverters.push(converter);
+            usedConverters[endpointOrGroupID].push(converter);
         }
 
         return true;

lib/extension/receive.js

@@ -2,26 +2,40 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class DeviceReceive extends BaseExtension {
+class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.coordinator = null;
         this.elapsed = {};
         this.debouncers = {};
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
     }
 
     async onZigbeeStarted() {
         this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
     }
 
+    async onPublishEntityState(data) {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+            }
+        }
+    }
+
     publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
         if (!this.debouncers[ieeeAddr]) {
             this.debouncers[ieeeAddr] = {
                 payload: {},
                 publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload);
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
                     this.debouncers[ieeeAddr].payload = {};
                 }, time * 1000),
             };
@@ -54,7 +68,7 @@ class DeviceReceive extends BaseExtension {
         return result;
     }
 
-    canHandleEvent(type, data, resolvedEntity) {
+    shouldProcess(type, data, resolvedEntity) {
         if (type !== 'message' || !resolvedEntity) {
             return false;
         }
@@ -77,7 +91,7 @@ class DeviceReceive extends BaseExtension {
          * As the same message is also received directly from the end device, it makes no sense
          * to handle these messages.
          */
-        const hasGroupID = data.hasOwnProperty('groupID') && data.groupID != 0;
+        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
         if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
             logger.debug('Skipping re-transmitted Xiaomi message');
             return false;
@@ -98,7 +112,7 @@ class DeviceReceive extends BaseExtension {
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
-        if (!this.canHandleEvent(type, data, resolvedEntity)) {
+        if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
 
@@ -107,8 +121,8 @@ class DeviceReceive extends BaseExtension {
             return c.cluster === data.cluster && type;
         });
 
-        // Check if there is an available converter
-        if (!converters.length) {
+        // Check if there is an available converter, genOta messages are not interesting.
+        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
             logger.debug(
                 `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
                 `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
@@ -144,21 +158,6 @@ class DeviceReceive extends BaseExtension {
                 this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
-
-                if (settings.get().homeassistant && settings.get().advanced.homeassistant_legacy_triggers) {
-                    /**
-                     * Publish an empty value for click and action payload, in this way Home Assistant
-                     * can use Home Assistant entities in automations.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-                     */
-                    Object.keys(payload).forEach((key) => {
-                        if (['action', 'click'].includes(key)) {
-                            const counterPayload = {};
-                            counterPayload[key] = '';
-                            this.publishEntityState(data.device.ieeeAddr, counterPayload);
-                        }
-                    });
-                }
             }
         };
 
@@ -178,4 +177,4 @@ class DeviceReceive extends BaseExtension {
     }
 }
 
-module.exports = DeviceReceive;
+module.exports = Receive;

lib/extension/report.js

@@ -2,7 +2,7 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
 const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
 const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 const debounce = require('debounce');
 const ZigbeeHerdsman = require('zigbee-herdsman');
 
@@ -14,6 +14,7 @@ const devicesNotSupportingReporting = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
 ];
 
 const reportKey = 1;
@@ -99,10 +100,10 @@ const pollOnMessage = [
     },
 ];
 
-class DeviceReport extends BaseExtension {
+class Report extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.configuring = new Set();
+        this.queue = new Set();
         this.failed = new Set();
         this.pollDebouncers = {};
     }
@@ -120,8 +121,8 @@ class DeviceReport extends BaseExtension {
     async setupReporting(resolvedEntity) {
         const {device, definition} = resolvedEntity;
 
-        if (this.configuring.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
-        this.configuring.add(device.ieeeAddr);
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
 
         try {
             for (const ep of device.endpoints) {
@@ -157,7 +158,7 @@ class DeviceReport extends BaseExtension {
         }
 
         device.save();
-        this.configuring.delete(device.ieeeAddr);
+        this.queue.delete(device.ieeeAddr);
     }
 
     shouldSetupReporting(resolvedEntity, messageType) {
@@ -252,4 +253,4 @@ class DeviceReport extends BaseExtension {
     }
 }
 
-module.exports = DeviceReport;
+module.exports = Report;

lib/mqtt.js

@@ -95,11 +95,15 @@ class MQTT extends events.EventEmitter {
         this.emit('message', {topic, message: message.toString()});
     }
 
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
+
     async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
-        if (!this.client || this.client.reconnecting) {
+        if (!this.isConnected()) {
             logger.error(`Not connected to MQTT server!`);
             logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
             return;

lib/util/logger.js

@@ -44,7 +44,8 @@ const transportsToUse = [
         format: winston.format.combine(
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
+                let {timestamp, level, message} = info;
+                level = level === 'warning' ? 'warn' : level;
                 const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
@@ -61,7 +62,8 @@ const transportFileOptions = {
     format: winston.format.combine(
         winston.format.timestamp({format: timestampFormat}),
         winston.format.printf(/* istanbul ignore next */(info) => {
-            const {timestamp, level, message} = info;
+            let {timestamp, level, message} = info;
+            level = level === 'warning' ? 'warn' : level;
             return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
         }),
     ),
@@ -77,8 +79,15 @@ if (output.includes('file')) {
     transportsToUse.push(new winston.transports.File(transportFileOptions));
 }
 
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    transportsToUse.push(new winston.transports.Syslog(options));
+}
+
 // Create logger
-const logger = winston.createLogger({transports: transportsToUse});
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
 
 // Cleanup any old log directory.
 function cleanup() {
@@ -117,4 +126,8 @@ if (output.includes('file')) {
     logger.info(`Logging to console only'`);
 }
 
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
+
 module.exports = logger;

lib/util/settings.js

@@ -41,6 +41,7 @@ const defaults = {
     experimental: {
         // json or attribute or attribute_and_json
         output: 'json',
+        new_api: false,
     },
     advanced: {
         legacy_api: true,
@@ -49,6 +50,7 @@ const defaults = {
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
         log_file: 'log.txt',
         log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
         soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
@@ -178,7 +180,7 @@ const schema = {
                 baudrate: {type: 'number'},
                 rtscts: {type: 'boolean'},
                 soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: 'array', items: {type: 'number'}},
+                network_key: {type: ['array', 'string'], items: {type: 'number'}},
                 last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
                 elapsed: {type: 'boolean'},
                 availability_timeout: {type: 'number', minimum: 0},
@@ -282,13 +284,16 @@ function write() {
 
     // Read settings to check if we have to split devices/groups into separate file.
     const actual = yaml.read(file);
-    if (actual.mqtt && actual.mqtt.password && actual.mqtt.user) {
-        toWrite.mqtt.user = actual.mqtt.user;
-        toWrite.mqtt.password = actual.mqtt.password;
-    }
 
-    if (actual.advanced && actual.advanced.network_key) {
-        toWrite.advanced.network_key = actual.advanced.network_key;
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
     }
 
     if (typeof actual.devices === 'string') {
@@ -310,6 +315,11 @@ function write() {
 function validate() {
     const validate = ajv.compile(schema);
     const valid = validate(_settings);
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
     const postfixes = utils.getEndpointNames();
 
     // Verify that all friendly names are unique

lib/util/utils.js

@@ -29,9 +29,13 @@ function toLocalISOString(dDate) {
 
 const endpointNames = [
     'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
     'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
     'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
 function flatten(arr) {
@@ -43,7 +47,8 @@ function flatten(arr) {
 const forceEndDevice = flatten(
     ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
         .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
-        .map((mappedModel) => mappedModel.zigbeeModel));
+        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
+        .map((definition) => definition.zigbeeModel));
 
 async function getZigbee2mqttVersion() {
     return new Promise((resolve, reject) => {
@@ -119,6 +124,15 @@ function getObjectsProperty(objects, key, defaultValue) {
     return defaultValue;
 }
 
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
@@ -135,4 +149,5 @@ module.exports = {
     isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     formatDate: (date, type) => formatDate(date, type),
     equalsPartial,
+    getResponse,
 };

lib/util/yaml.js

@@ -33,4 +33,12 @@ function writeIfChanged(file, content) {
     }
 }
 
-module.exports = {read, readIfExists, writeIfChanged};
+function updateIfChanged(file, key, value) {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/zigbee.js

@@ -11,27 +11,6 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-const herdsmanSettings = {
-    network: {
-        panID: settings.get().advanced.pan_id,
-        extendedPanID: settings.get().advanced.ext_pan_id,
-        channelList: [settings.get().advanced.channel],
-        networkKey: settings.get().advanced.network_key,
-    },
-    databasePath: data.joinPath('database.db'),
-    databaseBackupPath: data.joinPath('database.db.backup'),
-    backupPath: data.joinPath('coordinator_backup.json'),
-    serialPort: {
-        baudRate: settings.get().advanced.baudrate,
-        rtscts: settings.get().advanced.rtscts,
-        path: settings.get().serial.port,
-        adapter: settings.get().serial.adapter,
-    },
-    adapter: {
-        concurrent: settings.get().advanced.adapter_concurrent,
-    },
-};
-
 class Zigbee extends events.EventEmitter {
     constructor() {
         super();
@@ -40,10 +19,37 @@ class Zigbee extends events.EventEmitter {
 
     async start() {
         logger.info(`Starting zigbee-herdsman...`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key,
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+            },
+        };
+
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
         logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
 
+        if (herdsmanSettings.network.networkKey === 'GENERATE') {
+            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+            settings.set(['advanced', 'network_key'], newKey);
+            herdsmanSettings.network.networkKey = newKey;
+        }
+
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
             this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
@@ -145,7 +151,8 @@ class Zigbee extends events.EventEmitter {
     resolveEntity(key) {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device', `Wrong type '${typeof key}'`,
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
         );
 
         if (typeof key === 'string' || typeof key === 'number') {
@@ -182,7 +189,7 @@ class Zigbee extends events.EventEmitter {
                     return null;
                 }
 
-                const definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
                 const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
                 let endpoint;
                 if (endpointKey) {
@@ -209,7 +216,7 @@ class Zigbee extends events.EventEmitter {
                 if (!group) group = this.createGroup(entity.ID);
                 return {type: 'group', group, settings: entity, name: entity.friendlyName};
             }
-        } else {
+        } else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
             return {
                 type: 'device',
@@ -217,7 +224,15 @@ class Zigbee extends events.EventEmitter {
                 endpoint: key.endpoints[0],
                 settings: setting,
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
-                definition: zigbeeHerdsmanConverters.findByZigbeeModel(key.modelID),
+                definition: zigbeeHerdsmanConverters.findByDevice(key),
+            };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: setting,
+                name: setting ? setting.friendlyName : key.groupID,
             };
         }
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.12.2",
+  "version": "1.14.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13"
+    "node": "^10 || ^12 || ^13 || ^14"
   },
   "keywords": [
     "xiaomi",
@@ -45,8 +45,9 @@
     "rimraf": "*",
     "semver": "*",
     "winston": "*",
-    "zigbee-herdsman": "0.12.78",
-    "zigbee-herdsman-converters": "12.0.69"
+    "winston-syslog": "*",
+    "zigbee-herdsman": "0.12.94",
+    "zigbee-herdsman-converters": "12.0.110"
   },
   "devDependencies": {
     "eslint": "*",

test/assets/exampleExtension.js

@@ -0,0 +1,11 @@
+class Example {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.mqtt = mqtt;
+    }
+
+    onMQTTConnected() {
+        this.mqtt.publish('example/extension', 'test')
+    }
+}
+
+module.exports = Example;

test/availability.test.js

@@ -15,11 +15,11 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device availability', () => {
+describe('Availability', () => {
     let controller;
 
     function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'DeviceAvailability');
+        return controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
     beforeEach(async () => {
@@ -352,4 +352,21 @@ describe('Device availability', () => {
 
         device.lastSeen = defaultLastSeen;
     });
+
+    it('Should republish existing state on MQTT connected', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        await controller.stop();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        getExtension().state[device.ieeeAddr] = false;
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
 });

test/bind.test.js

@@ -6,7 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
-describe('Device bind', () => {
+describe('Bind', () => {
     let controller;
 
     mockClear = (device) => {

test/bridge.test.js

@@ -0,0 +1,208 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const {coordinator, bulb, unsupported} = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported.ieeeAddr);
+
+describe('Bridge', () => {
+    let controller;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        settings.set(['advanced', 'legacy_api'], false);
+        settings.set(['experimental', 'new_api'], true);
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+    });
+
+    it('Should publish bridge info on startup', async () => {
+        const version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/info',
+          JSON.stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"logLevel":"info","permitJoin":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          JSON.stringify([{"ieeeAddress":"0x000b57fffec6a5b2","type":"Router","networkAddress":40369,"supported":true,"friendlyName":"bulb","definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"},"powerSource":"Mains (single phase)","dateCode":null,"interviewing":false,"interviewCompleted":true},{"ieeeAddress":"0x0017880104e45518","type":"EndDevice","networkAddress":6536,"supported":false,"friendlyName":"0x0017880104e45518","definition":null,"powerSource":"Battery","dateCode":null,"interviewing":false,"interviewCompleted":true}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/groups',
+          JSON.stringify([{"ID":1,"friendlyName":"group_1","members":[]},{"ID":15071,"friendlyName":"group_tradfri_remote","members":[]},{"ID":99,"friendlyName":99,"members":[]},{"ID":11,"friendlyName":"group_with_tradfri","members":[]},{"ID":2,"friendlyName":"group_2","members":[]}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device joined', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceJoined","data":{"friendlyName":"bulb","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device interview started', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'started'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"started","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview failed', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'failed'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"failed","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview successful', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'successful'});
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.unsupported, status: 'successful'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"successful","ieeeAddress":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+          'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"0x0017880104e45518","status":"successful","ieeeAddress":"0x0017880104e45518","supported":false,"definition":null}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device leaves', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: zigbeeHerdsman.devices.bulb.ieeeAddr});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceLeave","data":{"ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put transaction in response when request is done with transaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false, "transaction": 22}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when request fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error('Failed to connect to adapter')});
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Coverage satisfaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/random', JSON.stringify({"value": false}));
+        const device = zigbeeHerdsman.devices.bulb;
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+    });
+});

test/configure.test.js

@@ -9,7 +9,7 @@ const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device configure', () => {
+describe('Configure', () => {
     let controller;
 
     expectRemoteConfigured = () => {
@@ -153,7 +153,7 @@ describe('Device configure', () => {
     });
 
     it('Should configure max 3 times when fails', async () => {
-        controller.extensions.find((e) => e.constructor.name === 'DeviceConfigure').attempts = {};
+        controller.extensions.find((e) => e.constructor.name === 'Configure').attempts = {};
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
         const endpoint = device.getEndpoint(1);

test/controller.test.js

@@ -85,6 +85,14 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should generate network_key when set to GENERATE', async () => {
+        settings.set(['advanced', 'network_key'], 'GENERATE');
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(data.read().advanced.network_key.length).toStrictEqual(16);
+    });
+
     it('Start controller should publish cached states', async () => {
         data.writeDefaultState();
         await controller.start();
@@ -529,4 +537,15 @@ describe('Controller', () => {
         await flushPromises();
         expect(controller.state.state).toStrictEqual({});
     });
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
 });

test/group.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
@@ -20,6 +21,7 @@ describe('Groups', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         MQTT.publish.mockClear();
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     })
 
     it('Apply group updates add', async () => {
@@ -375,7 +377,7 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -400,8 +402,8 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -426,9 +428,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF"}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -454,7 +456,7 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 });

test/homeassistant.test.js

@@ -204,7 +204,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
-            'value_template': "{{ (value_json.temperature | float) | round(1) }}",
+            'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_temperature',
@@ -229,7 +229,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
-            'value_template': '{{ (value_json.humidity | float) | round(0) }}',
+            'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_humidity',
@@ -254,7 +254,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
-            'value_template': '{{ (value_json.pressure | float) | round(2) }}',
+            'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_pressure',
@@ -517,7 +517,7 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        // 1 publish is the publish from deviceReceive
+        // 1 publish is the publish from receive
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
@@ -558,7 +558,7 @@ describe('HomeAssistant extension', () => {
         );
     });
 
-    it('Shouldnt discover when message has no device yet', async () => {
+    it('Shouldnt discover when device leaves', async () => {
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -939,4 +939,25 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
+
+    it('Should counter an action/click payload with an empty payload', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/click_single/config');
+        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/click');
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -1,11 +1,11 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const MQTT = require('./stub/mqtt');
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+const MQTT = require('../stub/mqtt');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
 
@@ -13,7 +13,7 @@ describe('Bridge legacy', () => {
     let controller;
 
     beforeAll(async () => {
-        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
         controller = new Controller();
         await controller.start();
     })

test/onEvent.test.js

@@ -13,11 +13,11 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const mockOnEvent = jest.fn();
-const mappedLivolo = zigbeeHerdsmanConverters.findByZigbeeModel(zigbeeHerdsman.devices.LIVOLO.modelID);
+const mappedLivolo = zigbeeHerdsmanConverters.findByDevice(zigbeeHerdsman.devices.LIVOLO);
 mappedLivolo.onEvent = mockOnEvent;
 zigbeeHerdsmanConverters.onEvent = jest.fn();
 
-describe('Device event', () => {
+describe('On event', () => {
     let controller;
     const device = zigbeeHerdsman.devices.LIVOLO;
 

test/otaUpdate.test.js

@@ -38,7 +38,7 @@ describe('OTA update', () => {
             count++;
             return {swBuildId: count, dateCode: '2019010' + count}
         });
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
         logger.error.mockClear();
@@ -54,8 +54,8 @@ describe('OTA update', () => {
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0%`);
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
         expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
@@ -67,7 +67,7 @@ describe('OTA update', () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
         logger.error.mockClear();
@@ -84,7 +84,7 @@ describe('OTA update', () => {
 
     it('Should be able to check if OTA update is available', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
         logger.info.mockClear();
@@ -106,7 +106,7 @@ describe('OTA update', () => {
 
     it('Should handle if OTA update check fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.error.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
@@ -127,7 +127,7 @@ describe('OTA update', () => {
     it('Should refuse to check/update when already in progress', async () => {
         jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
         logger.info.mockClear();
@@ -154,7 +154,7 @@ describe('OTA update', () => {
             return {swBuildId: 1, dateCode: '2019010'}
         });
 
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
@@ -165,7 +165,7 @@ describe('OTA update', () => {
     it('Should check for update when device requests it', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const data = {imageType: 12382};
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
@@ -201,4 +201,15 @@ describe('OTA update', () => {
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
     });
+
+    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.error.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
 });

test/publish.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -21,7 +22,7 @@ const expectNothingPublished = () => {
     })
 }
 
-describe('Entity publish', () => {
+describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
@@ -48,6 +49,8 @@ describe('Entity publish', () => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => {
             g.command.mockClear();
         });
+
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
     it('Should publish messages to zigbee devices', async () => {
@@ -249,14 +252,14 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {x: 100, y: 50}, brightness: 20, state: 'OFF'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
-        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -266,10 +269,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -278,10 +281,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -337,7 +340,7 @@ describe('Entity publish', () => {
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('Should publish messages to groups color', async () => {
@@ -411,6 +414,31 @@ describe('Entity publish', () => {
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
+    it('Should handle get with multiple endpoints', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_left: '', state_right: ''}));
+        await flushPromises();
+        expect(endpoint2.read).toHaveBeenCalledTimes(1);
+        expect(endpoint2.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
+    it('Should log error when device has no such endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        logger.error.mockClear();
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_center: '', state_right: ''}));
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'wall_switch_double' has no endpoint 'center'`);
+        expect(endpoint2.read).toHaveBeenCalledTimes(0);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
     it('Should not respond to bridge/config/devices/get', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', JSON.stringify({state: 'ON'}));
         await flushPromises();
@@ -475,10 +503,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -496,6 +524,21 @@ describe('Entity publish', () => {
         expectNothingPublished();
     });
 
+    it('Should send state update on toggle specific endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(2);
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'ON');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'TOGGLE');
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "toggle", {}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'ON'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'OFF'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
     it('Should parse set with postfix topic and attribute', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(2);
@@ -586,6 +629,22 @@ describe('Entity publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
     });
 
+    it('Transition parameter should not influence brightness on state ON', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3563
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 50}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON"}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 10}, {}]);
+    });
+
     it('Should use transition when color temp', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
@@ -642,11 +701,20 @@ describe('Entity publish', () => {
     it('Should turn device off when brightness 0 is send', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        const payload = {'brightness': 0};
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 50, state: 'ON'}));
         await flushPromises();
-        expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "off", {}, {}]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 0}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should turn device off when brightness 0 is send with light_brightness converter', async () => {
@@ -666,7 +734,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 45510.416666666664, "saturation": 127, "transtime": 0,}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891.475, "saturation": 199.21474, "transtime": 0,}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}});
@@ -719,7 +787,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 30}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 30}, {}]);
     });
 
     it('Should turn device on with on/off with transition when transition 0 is provided', async () => {
@@ -729,7 +797,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 0}, {}]);
     });
 
     it('Should turn device off with onOff on off with transition', async () => {
@@ -742,7 +810,7 @@ describe('Entity publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 10}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('When device is turned off and on with transition with report enabled it should restore correct brightness', async () => {
@@ -760,7 +828,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Bulb reports brightness while decreasing brightness
         await zigbeeHerdsman.events.message({data: {currentLevel: 1}, cluster: 'genLevelCtrl', device, endpoint, type: 'attributeReport', linkquality: 10});
@@ -792,7 +860,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Turn on again
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
@@ -885,7 +953,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -898,7 +966,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/led_controller_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -1051,4 +1119,31 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(4);
         expect(endpoint.command.mock.calls[3]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 10}, {}]);
     });
+
+    it('Shouldnt use moveToLevelWithOnOff on turn on when no transition has been used as some devices do not turn on in that case', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3332
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'brightness': 150}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'OFF'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 150, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genOnOff", "on", {}, {}]);
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON"}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"OFF","brightness":0}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[3]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+    });
 });

test/receive.test.js

@@ -8,7 +8,7 @@ const flushPromises = () => new Promise(setImmediate);
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device receive', () => {
+describe('Receive', () => {
     let controller;
 
     beforeEach(async () => {
@@ -33,23 +33,6 @@ describe('Device receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-
-    it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', async () => {
-        settings.set(['homeassistant'], true);
-        const device = zigbeeHerdsman.devices.WXKG11LM;
-        const data = {onOff: 1}
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
     it('Should handle a zigbee message which uses ep (left)', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM;
         const data = {onOff: 1}
@@ -159,6 +142,20 @@ describe('Device receive', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
     });
 
+    it('Shouldnt republish old state', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3572
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        jest.runAllTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+    });
+
     it('Should handle a zigbee message with 1 precision', async () => {
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);

test/report.test.js

@@ -19,7 +19,7 @@ const debounce = require('debounce');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
 
-describe('Device report', () => {
+describe('Report', () => {
     let controller;
 
     function expectOnOffBrightnessColorReport(endpoint, colorXY) {

test/settings.test.js

@@ -125,6 +125,12 @@ describe('Settings', () => {
 
         settings._write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'user'], 'test123');
+        settings.set(['advanced', 'network_key'], [1,2,3, 4]);
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
     });
 
     it('Should read devices form a separate file', () => {
@@ -489,6 +495,19 @@ describe('Settings', () => {
         }).toThrow(new Error("Device '0x123' already exists"));
     });
 
+    it('Should not allow any string values for network_key', () => {
+        write(configurationFile, {
+            advanced: {network_key: 'NOT_GENERATE'},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+    });
+
+
     it('Should not allow retention configuration without MQTT v5', () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},

test/stub/data.js

@@ -214,6 +214,7 @@ writeDefaultState();
 
 module.exports = {
     mockDir,
+    read: () => yaml.read(path.join(mockDir, 'configuration.yaml')),
     writeDefaultConfiguration,
     writeDefaultState,
     removeState,