1.13.1

'PM-S140-ZB': [cfg.switch],
    'PM-S240-ZB': [switchEndpoint('top'), switchEndpoint('bottom')],
    'PM-S340-ZB': [switchEndpoint('top'), switchEndpoint('center'), switchEndpoint('bottom')],

.github/ISSUE_TEMPLATE.md

@@ -1,23 +1,38 @@
 <!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2mqtt version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
 Did you read the FAQ?
-https://www.zigbee2mqtt.io/information/FAQ.html
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2mqtt fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
 
 Unsupported device?
-https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
 
-A question?
-Read the documentation: https://www.zigbee2mqtt.io/
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
 
-Bug report:
-- First make sure you are on the latest version of zigbee2mqtt.
-- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
-- Provide a clear description of the problem.
+Bug report?
 - If applicable, provide steps how to reproduce the problem.
-- Provide a log with `log_level` set to `debug`, see https://www.zigbee2mqtt.io/configuration/configuration.html
-- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
 -->
 
 <!--Start your bug report below this line-->
+
 # Bug Report
 ## What happened
 
@@ -26,5 +41,6 @@ Bug report:
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-zigbee2mqtt version:
-CC253X firmware version:
+Zigbee2mqtt version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.gitignore

@@ -67,6 +67,7 @@ data-backup/
 data/coordinator_backup.json
 data/.storage
 data/database.db.backup
+data/extension
 
 # MacOS indexing file
 .DS_Store

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "*.yaml": "home-assistant"
+    }
+}

azure-pipelines.yaml

@@ -76,15 +76,13 @@ steps:
   displayName: 'Docker build release'
   condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
 
-# Trigger downstream
+# Trigger Home Assistant Zigbee2mqtt addon build
 - bash: |
-    BODY="{\"request\": {\"branch\": \"master\"}}"
-    curl -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "Accept: application/json" \
-        -H "Travis-API-Version: 3" \
-        -H "Authorization: token $(TRAVIS_TOKEN)" \
-        -d "$BODY" \
-        "https://api.travis-ci.org/repo/danielwelch%2Fhassio-zigbee2mqtt/requests"
+    curl -s \
+        --request POST \
+        --header "Content-Type: application/json" \
+        --data '{"definition": {"id": 1}}' \
+        -u "$(HASSIO_TRIGGER_SECRET)" \
+        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
   displayName: 'Trigger Hass.io build'
   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

lib/controller.js

@@ -6,21 +6,25 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const fs = require('fs');
+const data = require('./util/data');
+const path = require('path');
 
 // Extensions
-const ExtensionEntityPublish = require('./extension/entityPublish');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/softReset');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
-const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionBridgeConfig = require('./extension/bridgeConfig');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+// const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
-const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
-const ExtensionDeviceBind = require('./extension/deviceBind');
-const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionDeviceEvent = require('./extension/deviceEvent');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/report');
+const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
 
 class Controller {
@@ -28,50 +32,55 @@ class Controller {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
         this.eventBus = new EventBus();
-        this.state = new State();
+        this.state = new State(this.eventBus);
 
         this.publishEntityState = this.publishEntityState.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
 
         // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionEntityPublish(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceReceive(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceGroupMembership(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ),
-            new ExtensionDeviceConfigure(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceEvent(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionOTAUpdate(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
+            new ExtensionOTAUpdate(...args),
+            // new ExtensionBridge(...args),
         ];
 
+        /* istanbul ignore else */
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
+
         if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionReport(...args));
         }
 
         if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
         /* istanbul ignore next */
         if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionSoftReset(...args));
         }
 
         if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionAvailability(...args));
+        }
+
+        const extensionPath = data.joinPath('extension');
+        if (fs.existsSync(extensionPath)) {
+            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
+            for (const extension of extensions) {
+                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
+                this.extensions.push(new Extension(...args, settings, logger));
+            }
         }
     }
 
@@ -114,8 +123,8 @@ class Controller {
             logger.info(
                 (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
                 ` (${entity.device.ieeeAddr}): ` +
-                (entity.mapped ?
-                    `${entity.mapped.model} - ${entity.mapped.vendor} ${entity.mapped.description} ` :
+                (entity.definition ?
+                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
                     'Not supported ') +
                 `(${entity.device.type})`,
             );
@@ -143,6 +152,13 @@ class Controller {
             }
         }
 
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClients()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
+
         // Call extensions
         await this.callExtensionMethod('onMQTTConnected', []);
     }
@@ -170,13 +186,13 @@ class Controller {
     }
 
     async onZigbeeEvent(type, data) {
-        const entity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !entity.settings) {
+        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            entity.settings = settings.addDevice(data.device.ieeeAddr);
+            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
         }
 
-        const name = entity && entity.settings ? entity.settings.friendlyName : null;
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
 
         if (type === 'message') {
             logger.debug(
@@ -186,52 +202,38 @@ class Controller {
             );
         } else if (type === 'deviceJoined') {
             logger.info(`Device '${name}' joined`);
-            this.mqtt.log('device_connected', {friendly_name: name});
         } else if (type === 'deviceInterview') {
             if (data.status === 'successful') {
                 logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
 
-                if (entity.mapped) {
-                    const {vendor, description, model} = entity.mapped;
-                    logger.info(
-                        `Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`,
-                    );
-
-                    const log = {friendly_name: name, model, vendor, description, supported: true};
-                    this.mqtt.log('pairing', 'interview_successful', log);
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
                         `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
-                    this.mqtt.log('pairing', 'interview_successful', {friendly_name: name, supported: false});
                 }
             } else if (data.status === 'failed') {
                 logger.error(`Failed to interview '${name}', device has not successfully been paired`);
-                this.mqtt.log('pairing', 'interview_failed', {friendly_name: name});
             } else {
                 /* istanbul ignore else */
                 if (data.status === 'started') {
                     logger.info(`Starting interview of '${name}'`);
-                    this.mqtt.log('pairing', 'interview_started', {friendly_name: name});
                 }
             }
         } else if (type === 'deviceAnnounce') {
             logger.debug(`Device '${name}' announced itself`);
-            this.mqtt.log('device_announced', 'announce', {friendly_name: name});
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
                 logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
-                this.mqtt.log('device_removed', 'left_network', {friendly_name: name || data.ieeeAddr});
             }
         }
 
         // Call extensions
-        this.callExtensionMethod(
-            'onZigbeeEvent',
-            [type, data, entity ? entity.mapped : null, entity ? entity.settings : null],
-        );
+        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
     }
 
     onMQTTMessage(payload) {
@@ -243,22 +245,19 @@ class Controller {
     }
 
     async publishEntityState(IDorName, payload, stateChangeReason=null) {
-        const entity = this.zigbee.resolveEntity(IDorName);
-        if (!entity || !entity.settings) {
+        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
             logger.error(`'${IDorName}' does not exist, skipping publish`);
             return;
         }
 
-        if (entity.type === 'device' && settings.get().advanced.last_seen !== 'disable' && entity.device.lastSeen) {
-            payload.last_seen = utils.formatDate(entity.device.lastSeen, settings.get().advanced.last_seen);
-        }
-
         let messagePayload = {...payload};
-        const currentState = this.state.exists(entity.settings.ID) ? this.state.get(entity.settings.ID) : {};
+        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
+            this.state.get(resolvedEntity.settings.ID) : {};
         const newState = objectAssignDeep.noMutate(currentState, payload);
 
         // Update state cache with new state.
-        this.state.set(entity.settings.ID, newState, stateChangeReason);
+        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
@@ -267,53 +266,58 @@ class Controller {
 
         const deviceOptions = settings.get().device_options;
         const options = {
-            retain: utils.getObjectsProperty([entity.settings, deviceOptions], 'retain', false),
-            qos: utils.getObjectsProperty([entity.settings, deviceOptions], 'qos', 0),
+            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
+            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
         };
 
-        const retention = utils.getObjectsProperty([entity.settings, deviceOptions], 'retention', false);
+        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
         if (retention !== false) {
             options.properties = {messageExpiryInterval: retention};
         }
 
-        if (entity.type === 'device' && settings.get().mqtt.include_device_information) {
-            const device = this.zigbee.getDeviceByIeeeAddr(entity.device.ieeeAddr);
+        const isDevice = resolvedEntity.type === 'device';
+        if (isDevice && settings.get().mqtt.include_device_information) {
             const attributes = [
                 'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
                 'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
             ];
 
             messagePayload.device = {
-                friendlyName: entity.name,
-                model: entity.mapped ? entity.mapped.model : 'unknown',
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
             };
 
-            attributes.forEach((a) => messagePayload.device[a] = device[a]);
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
         }
 
         // filter mqtt message attributes
         if (deviceOptions.filtered_attributes) {
             deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
-        if (entity.settings.filtered_attributes) {
-            entity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity});
-
         if (Object.entries(messagePayload).length) {
             if (settings.get().experimental.output === 'attribute_and_json') {
-                await this.mqtt.publish(entity.name, JSON.stringify(messagePayload), options);
-                await this.iteratePayloadAttributeOutput(`${entity.name}/`, messagePayload, options);
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
             } else if (settings.get().experimental.output === 'json') {
-                await this.mqtt.publish(entity.name, JSON.stringify(messagePayload), options);
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
             } else {
                 /* istanbul ignore else */
                 if (settings.get().experimental.output === 'attribute') {
-                    await this.iteratePayloadAttributeOutput(`${entity.name}/`, messagePayload, options);
+                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
                 }
             }
         }
+
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {

lib/eventBus.js

@@ -6,6 +6,7 @@ const allowedEvents = [
     'deviceRenamed', // Device has been renamed
     'groupRenamed', // Group has been renamed
     'publishEntityState', // Entity state will be published
+    'stateChange', // Entity changes its state
 ];
 
 class EventBus extends events.EventEmitter {

lib/extension/availability.js

@@ -2,26 +2,21 @@ const logger = require('../util/logger');
 const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-// Some EndDevices should be pinged
+// Pingable end devices, some end devices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableModels = [
-    'E11-G13',
-    '53170161',
+const pingableEndDevices = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
 ];
 
-const forcedPingable =
-    zigbeeHerdsmanConverters.devices.filter((d) => pingableModels.includes(d.model));
-
-const toZigbeeCandidates = ['state', 'brightness', 'color', 'color_temp'];
-
 const Hours25 = 1000 * 60 * 60 * 25;
 
 /**
  * This extensions pings devices to check if they are online.
  */
-class DeviceAvailability extends BaseExtension {
+class Availability extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -29,22 +24,14 @@ class DeviceAvailability extends BaseExtension {
         this.timers = {};
         this.state = {};
 
-        // Initialize blacklist
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
-
-        // Initialize whitelist
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
+        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
+        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
     }
 
-    isAllowed(device) {
+    inWhitelistOrNotInBlacklist(device) {
         const ieeeAddr = device.ieeeAddr;
-
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : null;
+        const name = deviceSettings.friendlyName;
 
         // Whitelist is not empty and device is in it, enable availability
         if (this.whitelist.length > 0) {
@@ -60,15 +47,12 @@ class DeviceAvailability extends BaseExtension {
     }
 
     isPingable(device) {
-        // Device is on forcedPingable-list, enable availability
-        if (forcedPingable.find((d) => d.zigbeeModel.includes(device.modelID))) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
             return true;
         }
 
         // Device is a mains powered router
-        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
-
-        return result;
+        return utils.isRouter(device) && !utils.isBatteryPowered(device);
     }
 
     onMQTTConnected() {
@@ -76,7 +60,7 @@ class DeviceAvailability extends BaseExtension {
             // Mark all devices as online on start
             this.publishAvailability(device, true);
 
-            if (this.isAllowed(device)) {
+            if (this.inWhitelistOrNotInBlacklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
@@ -90,21 +74,20 @@ class DeviceAvailability extends BaseExtension {
 
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity) {
             logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
             return;
         }
 
-        const ieeeAddr = device.ieeeAddr;
-        const level = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
         try {
             await device.ping();
             this.publishAvailability(device, true);
-            logger.debug(`Successfully pinged '${entity.name}'`);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
         } catch (error) {
             this.publishAvailability(device, false);
-            logger[level](`Failed to ping '${entity.name}'`);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
         } finally {
             this.setTimerPingable(device);
         }
@@ -112,12 +95,12 @@ class DeviceAvailability extends BaseExtension {
 
     async handleIntervalNotPingable(device) {
         const ago = Date.now() - device.lastSeen;
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity || !device.lastSeen) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
             return;
         }
 
-        logger.debug(`Non-pingable device '${entity.name}' was last seen '${ago / 1000}' seconds ago.`);
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
 
         if (ago > Hours25) {
             this.publishAvailability(device, false);
@@ -143,23 +126,19 @@ class DeviceAvailability extends BaseExtension {
     }
 
     async onReconnect(device) {
-        if (device && device.modelID) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-
-            if (mappedDevice) {
-                const used = [];
-                try {
-                    for (const key of toZigbeeCandidates) {
-                        const converter = mappedDevice.toZigbee.find((tz) => tz.key.includes(key));
-                        if (converter && !used.includes(converter)) {
-                            await converter.convertGet(device.endpoints[0], key, {});
-                            used.push(converter);
-                        }
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            const used = [];
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter && !used.includes(converter)) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                        used.push(converter);
                     }
-                } catch (error) {
-                    const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-                    logger.error(`Failed to read state of '${entity.name}' after reconnect`);
                 }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
             }
         }
     }
@@ -171,7 +150,7 @@ class DeviceAvailability extends BaseExtension {
         }
 
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
+        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
         const topic = `${name}/availability`;
         const payload = available ? 'online' : 'offline';
         if (this.state[ieeeAddr] !== available) {
@@ -180,13 +159,13 @@ class DeviceAvailability extends BaseExtension {
         }
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
+    onZigbeeEvent(type, data, resolvedEntity) {
         const device = data.device;
         if (!device) {
             return;
         }
 
-        if (this.isAllowed(device)) {
+        if (this.inWhitelistOrNotInBlacklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {
@@ -212,4 +191,4 @@ class DeviceAvailability extends BaseExtension {
     }
 }
 
-module.exports = DeviceAvailability;
+module.exports = Availability;

lib/extension/bind.js

@@ -1,30 +1,46 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
 const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-const BaseExtension = require('./baseExtension');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const Extension = require('./extension');
 
 const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
 
 // See zigbee-herdsman-converters devices.js
 const defaultBindGroup = {type: 'group_number', ID: 901};
 
-class DeviceBind extends BaseExtension {
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        }
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        }
+
+        return {type, sourceKey, targetKey};
     }
 
     async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Parse topic, retrieve type (bind or unbind) and source
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-        const type = topic.split('/')[0];
-        const sourceKey = topic.replace(`${type}/`, '');
-        const targetKey = message;
+        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
 
         // Find source; can only be a device and target
         const source = this.zigbee.resolveEntity(sourceKey);
@@ -34,6 +50,7 @@ class DeviceBind extends BaseExtension {
 
         const sourceName = source.settings.friendlyName;
         const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+        let attemptedToBindSomething = false;
 
         // Find which clusters are supported by both the source and target.
         // Groups are assumed to support all clusters.
@@ -43,6 +60,7 @@ class DeviceBind extends BaseExtension {
 
             if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
                 logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                attemptedToBindSomething = true;
                 try {
                     let bindTarget = null;
                     if (target.type === 'group') bindTarget = target.group;
@@ -59,23 +77,45 @@ class DeviceBind extends BaseExtension {
                         `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
                         `'${sourceName}' to '${targetName}'`,
                     );
-                    this.mqtt.log(
-                        `device_${type}`,
-                        {from: sourceName, to: targetName, cluster},
-                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}`,
+                                message: {from: sourceName, to: targetName, cluster}}),
+                        );
+                    }
                 } catch (error) {
                     logger.error(
                         `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
                         `'${targetName}' (${error})`,
                     );
-                    this.mqtt.log(
-                        `device_${type}_failed`,
-                        {from: sourceName, to: targetName, cluster},
-                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}_failed`,
+                                message: {from: sourceName, to: targetName, cluster}}),
+                        );
+                    }
                 }
             }
         }
+
+        if (!attemptedToBindSomething) {
+            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                );
+            }
+        }
     }
 }
 
-module.exports = DeviceBind;
+module.exports = Bind;

lib/extension/bridge.js

@@ -0,0 +1,212 @@
+/* istanbul ignore file newApi */
+
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class BridgeLegacy extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+
+        this.requestLookup = {
+            'permitjoin': this.requestPermitJoin.bind(this),
+            'device/remove': this.deviceRemove.bind(this),
+            'device/forceremove': this.deviceForceRemove.bind(this),
+            'device/ban': this.deviceBan.bind(this),
+            'group/remove': this.groupRemove.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/+`);
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = JSON.parse(message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') payload = {friendlyName: resolvedEntity.settings.friendlyName, ieeeAddress};
+            else if (type === 'deviceInterview') {
+                payload = {friendlyName: resolvedEntity.settings.friendlyName, status: data.status, ieeeAddress};
+                if (data.status === 'successful') {
+                    payload.supported = !!resolvedEntity.definition;
+                    payload.definition = resolvedEntity.definition ? {
+                        model: resolvedEntity.definition.model,
+                        vendor: resolvedEntity.definition.vendor,
+                        description: resolvedEntity.definition.description,
+                        supports: resolvedEntity.definition.supports,
+                    } : null;
+                }
+            } else payload = {ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish('bridge/event', JSON.stringify({type, data: payload}), {retain: false, qos: 0});
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    async deviceRemove(message) {
+        return this.removeForceRemoveOrBanEntity('remove', 'device', message);
+    }
+
+    async deviceForceRemove(message) {
+        return this.removeForceRemoveOrBanEntity('force_remove', 'device', message);
+    }
+
+    async deviceBan(message) {
+        return this.removeForceRemoveOrBanEntity('ban', 'device', message);
+    }
+
+    async groupRemove(message) {
+        return this.removeForceRemoveOrBanEntity('remove', 'group', message);
+    }
+
+    async requestPermitJoin(message) {
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.permitJoin(value);
+        await this.publishInfo();
+        return utils.getResponse(message, {value: value}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    async removeForceRemoveOrBanEntity(action, entityType, message) {
+        const ID = typeof message === 'object' ? message.ID : message.trim();
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.type !== entityType) {
+            throw new Error(`${ID} is not a ${entityType}`);
+        }
+
+        const lookup = {
+            ban: ['banned', 'Banning', 'ban'],
+            force_remove: ['force_removed', 'Force removing', 'force remove'],
+            remove: ['removed', 'Removing', 'remove'],
+        };
+
+        try {
+            logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+            if (entity.type === 'device') {
+                if (action === 'ban') {
+                    settings.banDevice(entity.settings.ID);
+                }
+
+                action === 'force_remove' ?
+                    await entity.device.removeFromDatabase() : await entity.device.removeFromNetwork();
+            } else {
+                await entity.group.removeFromDatabase();
+            }
+
+            // Fire event
+            if (entity.type === 'device') {
+                this.eventBus.emit('deviceRemoved', {device: entity.device});
+            }
+
+            // Remove from configuration.yaml
+            entity.type === 'device' ?
+                settings.removeDevice(entity.settings.ID) : settings.removeGroup(entity.settings.ID);
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+            entity.type === 'device' ? this.publishDevices() : this.publishGroups();
+            return utils.getResponse(message, {ID}, null);
+        } catch (error) {
+            throw new Error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+        }
+    }
+
+    async publishInfo() {
+        const info = await utils.getZigbee2mqttVersion();
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const payload = {
+            version: info.version,
+            commit: info.commitHash,
+            coordinator,
+            logLevel: logger.getLevel(),
+            permitJoin: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    async publishDevices(topic, message) {
+        const devices = this.zigbee.getClients().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const definitionPayload = definition ? {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                supports: definition.supports,
+            } : null;
+
+            return {
+                ieeeAddress: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+                supported: !!definition,
+                friendlyName: resolved.settings.friendlyName,
+                definition: definitionPayload,
+                powerSource: device.powerSource,
+                softwareBuildID: device.softwareBuildID,
+                dateCode: device.dateCode,
+                interviewing: device.interviewing,
+                interviewCompleted: device.interviewCompleted,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+    }
+
+    async publishGroups(topic, message) {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                ID: group.groupID,
+                friendlyName: resolved.settings.friendlyName,
+                members: group.members.map((m) => {
+                    return {
+                        ieeeAddress: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+    }
+}
+
+module.exports = BridgeLegacy;

lib/extension/configure.js

@@ -0,0 +1,120 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.configuring = new Set();
+        this.attempts = {};
+
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+    }
+
+    shouldConfigure(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            return false;
+        }
+
+        if (resolvedEntity.device.meta &&
+            resolvedEntity.device.meta.hasOwnProperty('configured') &&
+            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            return false;
+        }
+
+        if (resolvedEntity.device.interviewing === true) {
+            return false;
+        }
+
+        return true;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (topic !== this.legacyTopic) {
+                return;
+            }
+
+            const resolvedEntity = this.zigbee.resolveEntity(message);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                logger.error(`Device '${message}' does not exist`);
+                return;
+            }
+
+            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(resolvedEntity, true);
+        }
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldConfigure(resolvedEntity)) {
+                await this.configure(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+
+        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
+    }
+
+    async configure(resolvedEntity, force=false) {
+        const device = resolvedEntity.device;
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return false;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${resolvedEntity.name}'`);
+        try {
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            logger.info(`Successfully configured '${resolvedEntity.name}'`);
+            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.save();
+        } catch (error) {
+            logger.error(
+                `Failed to configure '${resolvedEntity.name}', ` +
+                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
+            );
+            this.attempts[device.ieeeAddr]++;
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/deviceConfigure.js

@@ -1,112 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
-
-class DeviceConfigure extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        this.configuring = new Set();
-        this.attempts = {};
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-    }
-
-    shouldConfigure(device, mappedDevice) {
-        if (!device || !mappedDevice || !mappedDevice.configure) {
-            return false;
-        }
-
-        if (device.meta &&
-            device.meta.hasOwnProperty('configured') &&
-            device.meta.configured === mappedDevice.meta.configureKey) {
-            return false;
-        }
-
-        if (device.interviewing === true) {
-            return false;
-        }
-
-        return true;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (topic !== this.topic) {
-            return;
-        }
-
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || entity.type !== 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            return;
-        }
-
-        const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(entity.device.modelID);
-        if (!mappedDevice.configure) {
-            logger.warn(`Skipping configure of '${entity.name}', device does not require this.`);
-            return;
-        }
-
-        this.configure(entity.device, mappedDevice, entity.settings, true);
-    }
-
-    async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
-
-        for (const device of this.zigbee.getClients()) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-            const settingsDevice = settings.getDevice(device.ieeeAddr);
-            if (this.shouldConfigure(device, mappedDevice)) {
-                await this.configure(device, mappedDevice, settingsDevice);
-            }
-        }
-    }
-
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        const device = data.device;
-
-        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(device, mappedDevice)) {
-            this.configure(device, mappedDevice, settingsDevice);
-        }
-    }
-
-    async configure(device, mappedDevice, settingsDevice, force=false) {
-        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
-            return false;
-        }
-
-        this.configuring.add(device.ieeeAddr);
-
-        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
-            this.attempts[device.ieeeAddr] = 0;
-        }
-
-        logger.info(`Configuring '${settingsDevice.friendlyName}'`);
-        try {
-            await mappedDevice.configure(device, this.coordinatorEndpoint);
-            logger.info(`Successfully configured '${settingsDevice.friendlyName}'`);
-            // eslint-disable-next-line
-            device.meta.configured = mappedDevice.meta.configureKey;
-            device.save();
-        } catch (error) {
-            logger.error(
-                `Failed to configure '${settingsDevice.friendlyName}', ` +
-                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
-            );
-            this.attempts[device.ieeeAddr]++;
-        }
-
-        this.configuring.delete(device.ieeeAddr);
-    }
-}
-
-module.exports = DeviceConfigure;

lib/extension/deviceEvent.js

@@ -1,36 +0,0 @@
-const BaseExtension = require('./baseExtension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-
-class DeviceEvent extends BaseExtension {
-    async onZigbeeStarted() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'start', {});
-        }
-    }
-
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (data.device) {
-            this.callOnEvent(data.device, type, data, mappedDevice);
-        }
-    }
-
-    async stop() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'stop', {});
-        }
-    }
-
-    callOnEvent(device, type, data, mappedDevice) {
-        zigbeeHerdsmanConverters.onEvent(type, data, device);
-
-        if (!mappedDevice) {
-            mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-        }
-
-        if (mappedDevice && mappedDevice.onEvent) {
-            mappedDevice.onEvent(type, data, device);
-        }
-    }
-}
-
-module.exports = DeviceEvent;

lib/extension/extension.js

@@ -1,4 +1,4 @@
-class BaseExtension {
+class Extension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -30,10 +30,10 @@ class BaseExtension {
      * Is called when a Zigbee message from a device is received.
      * @param {string} type Type of the message
      * @param {Object} data Data of the message
-     * @param {Object?} mappedDevice The mapped device
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
      * @param {Object?} settingsDevice Device settings
      */
-    // onZigbeeEvent(type, data, mappedDevice, settingsDevice) {}
+    // onZigbeeEvent(type, data, resolvedEntity) {}
 
     /**
      * Is called when a MQTT message is received
@@ -49,4 +49,4 @@ class BaseExtension {
     // stop() {}
 }
 
-module.exports = BaseExtension;
+module.exports = Extension;

lib/extension/groups.js

@@ -1,30 +1,36 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const postfixes = utils.getEndpointNames();
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
-const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups extends BaseExtension {
+class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
 
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}remove`);
-            this.mqtt.subscribe(`${topic}add`);
-            this.mqtt.subscribe(`${topic}remove_all`); // DEPRECATED
+            for (let step = 1; step < 20; step++) {
+                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
+                this.mqtt.subscribe(`${topic}remove`);
+                this.mqtt.subscribe(`${topic}add`);
+                this.mqtt.subscribe(`${topic}remove_all`);
+            }
         }
     }
 
     async onZigbeeStarted() {
         await this.syncGroupsWithSettings();
-        this.state.on('stateChange', this.onStateChange);
+        this.eventBus.on('stateChange', this.onStateChange);
     }
 
     async syncGroupsWithSettings() {
@@ -85,33 +91,33 @@ class Groups extends BaseExtension {
         });
 
         if (Object.keys(payload).length) {
-            const entity = this.zigbee.resolveEntity(data.ID);
+            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
                 return settingsGroup && settingsGroup.optimistic;
             });
 
-            if (entity.type === 'device') {
+            if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(entity.endpoint)) {
-                        if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }
                 }
             } else {
                 const groupIDsToPublish = new Set();
-                for (const member of entity.group.members) {
+                for (const member of resolvedEntity.group.members) {
                     await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
                     for (const zigbeeGroup of zigbeeGroups) {
                         if (zigbeeGroup.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                                 groupIDsToPublish.add(zigbeeGroup.groupID);
                             }
                         }
                     }
                 }
-                groupIDsToPublish.delete(entity.group.groupID);
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
                 for (const groupID of groupIDsToPublish) {
                     await this.publishEntityState(groupID, payload, reason);
                 }
@@ -119,7 +125,7 @@ class Groups extends BaseExtension {
         }
     }
 
-    allMembersOff(zigbeeGroup) {
+    areAllMembersOff(zigbeeGroup) {
         for (const member of zigbeeGroup.members) {
             const device = member.getDevice();
             if (this.state.exists(device.ieeeAddr)) {
@@ -132,67 +138,126 @@ class Groups extends BaseExtension {
         return true;
     }
 
-    async onMQTTMessage(topic, message) {
-        let type;
-        let group;
-        const topicMatch = topic.match(topicRegex);
-        if (topicMatch) {
-            group = this.zigbee.resolveEntity(topicMatch[1]);
-            type = topicMatch[2];
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
 
-            if (!group || group.type !== 'group') {
-                logger.error(`Group '${topicMatch[1]}' does not exist`);
-                this.mqtt.log(
-                    `device_group_${type}_failed`,
-                    {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'});
-                return;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            const topicMatch = topic.match(legacyTopicRegex);
+            if (topicMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
+                type = topicMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else if (topic.match(legacyTopicRegexRemoveAll)) {
+                type = 'remove_all';
+            } else {
+                return {};
             }
-        } else if (topic.match(topicRegexRemoveAll)) {
-            type = 'remove_all';
-        } else {
-            return;
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return {};
+            }
+
+            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
         }
 
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || !entity.type === 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            this.mqtt.log(
-                `device_group_${type}_failed`,
-                {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'});
-            return;
-        }
+        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
+            this.parseMQTTMessage(topic, message);
+        if (!type) return;
 
         const keys = [
-            `${entity.device.ieeeAddr}/${entity.endpoint.ID}`,
-            `${entity.name}/${entity.endpoint.ID}`,
+            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
         ];
 
-        if (entity.endpointName) {
-            keys.push(`${entity.device.ieeeAddr}/${entity.endpointName}`);
-            keys.push(`${entity.name}/${entity.endpointName}`);
+        const definition = resolvedEntityDevice.definition;
+        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
+        const endpointName = endpoints ?
+            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+
+        if (endpointName) {
+            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
         }
 
-        if (entity.isDefaultEndpoint) {
-            keys.push(entity.name);
-            keys.push(entity.device.ieeeAddr);
+        if (!hasEndpointName) {
+            keys.push(resolvedEntityDevice.name);
+            keys.push(resolvedEntityDevice.device.ieeeAddr);
         }
 
         if (type === 'add') {
-            logger.info(`Adding '${entity.name}' to '${group.name}'`);
-            await entity.endpoint.addToGroup(group.group);
-            settings.addDeviceToGroup(group.settings.ID, keys);
-            this.mqtt.log('device_group_add', {friendly_name: entity.name, group: group.name});
+            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_add`, message: payload}),
+                );
+            }
         } else if (type === 'remove') {
-            logger.info(`Removing '${entity.name}' from '${group.name}'`);
-            await entity.endpoint.removeFromGroup(group.group);
-            settings.removeDeviceFromGroup(group.settings.ID, keys);
-            this.mqtt.log('device_group_remove', {friendly_name: entity.name, group: group.name});
+            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_remove`, message: payload}),
+                );
+            }
         } else { // remove_all
-            logger.info(`Removing '${entity.name}' from all groups`);
-            await entity.endpoint.removeFromAllGroups();
+            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+            await resolvedEntityDevice.endpoint.removeFromAllGroups();
             for (const settingsGroup of settings.getGroups()) {
                 settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-                this.mqtt.log('device_group_remove_all', {friendly_name: entity.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: resolvedEntityDevice.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                    );
+                }
             }
         }
     }

lib/extension/legacy/bridgeLegacy.js

@@ -1,15 +1,15 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const utils = require('../util/utils');
+const utils = require('../../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('../extension');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
 const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
 
-class BridgeConfig extends BaseExtension {
+class BridgeLegacy extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -63,7 +63,10 @@ class BridgeConfig extends BaseExtension {
             assert(entity, `Entity '${message}' does not exist`);
             settings.whitelistDevice(entity.ID);
             logger.info(`Whitelisted '${entity.friendlyName}'`);
-            this.mqtt.log('device_whitelisted', {friendly_name: entity.friendlyName});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
         }
@@ -147,11 +150,11 @@ class BridgeConfig extends BaseExtension {
             };
 
             if (device.type !== 'Coordinator') {
-                const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
                 const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = mappedDevice ? mappedDevice.model : device.modelID;
-                payload.vendor = mappedDevice ? mappedDevice.vendor : '-';
-                payload.description = mappedDevice ? mappedDevice.description : '-';
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
                 payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
                 payload.manufacturerID = device.manufacturerID;
                 payload.manufacturerName = device.manufacturerName;
@@ -174,20 +177,23 @@ class BridgeConfig extends BaseExtension {
         if (topic.split('/').pop() == 'get') {
             this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
         } else {
-            this.mqtt.log('devices', devices);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
         }
     }
 
     groups(topic, message) {
-        this.mqtt.log('groups', settings.getGroups().map((g) => {
+        const payload = settings.getGroups().map((g) => {
             const group = {...g};
             delete group.friendlyName;
             return group;
-        }));
+        });
+
+        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
     }
 
     rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} got ${message}`;
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
 
         let json = null;
         try {
@@ -223,7 +229,11 @@ class BridgeConfig extends BaseExtension {
             const entity = this.zigbee.resolveEntity(to);
             const eventData = isGroup ? {group: entity.group} : {device: entity.device};
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
-            this.mqtt.log(`${isGroup ? 'group' : 'device'}_renamed`, {from, to});
+
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
         } catch (error) {
             logger.error(`Failed to rename - ${from} to ${to}`);
         }
@@ -254,7 +264,7 @@ class BridgeConfig extends BaseExtension {
 
         const group = settings.addGroup(name, id);
         this.zigbee.createGroup(group.ID);
-        this.mqtt.log('group_added', name);
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
@@ -264,7 +274,7 @@ class BridgeConfig extends BaseExtension {
         assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
         settings.removeGroup(message);
         entity.group.removeFromDatabase();
-        this.mqtt.log('group_removed', message);
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }
 
@@ -291,7 +301,7 @@ class BridgeConfig extends BaseExtension {
         if (!entity) {
             logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
 
-            this.mqtt.log(`device_${lookup[action][0]}_failed`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
             return;
         }
 
@@ -306,7 +316,7 @@ class BridgeConfig extends BaseExtension {
             this.state.remove(entity.settings.ID);
 
             logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-            this.mqtt.log(`device_${lookup[action][0]}`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
@@ -323,7 +333,7 @@ class BridgeConfig extends BaseExtension {
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
 
-            this.mqtt.log(`device_${lookup[action][0]}_failed`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
         }
 
         if (action === 'ban') {
@@ -368,25 +378,86 @@ class BridgeConfig extends BaseExtension {
         await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
         if (type === 'deviceJoined') {
-            this.lastJoinedDeviceName = settingsDevice.friendlyName;
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
         }
     }
 
     async touchlinkFactoryReset() {
         logger.info('Starting touchlink factory reset...');
-        this.mqtt.log('touchlink', 'reset_started', {status: 'started'});
+        this.mqtt.publish(
+            'bridge/log',
+            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
         const result = await this.zigbee.touchlinkFactoryReset();
 
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
-            this.mqtt.log('touchlink', 'reset_success', {status: 'success'});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
         } else {
             logger.warn('Failed to factory reset device through Touchlink');
-            this.mqtt.log('touchlink', 'reset_failed', {status: 'failed'});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
         }
     }
 }
 
-module.exports = BridgeConfig;
+module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -1,11 +1,11 @@
 /* istanbul ignore file */
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
 
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
-class DeviceGroupMembership extends BaseExtension {
+class DeviceGroupMembership extends Extension {
     onMQTTConnected() {
         for (let step = 1; step < 20; step++) {
             const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;

lib/extension/legacy/softReset.js

@@ -1,14 +1,14 @@
 /* istanbul ignore file */
 // DEPRECATED
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
 
 /**
  * This extensions soft resets the ZNP after a certain timeout.
  */
-class SoftReset extends BaseExtension {
+class SoftReset extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;

lib/extension/networkMap.js

@@ -2,15 +2,18 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class NetworkMap extends BaseExtension {
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
-        // Subscribe to topic.
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.topicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
 
         // Bind
         this.raw = this.raw.bind(this);
@@ -24,16 +27,23 @@ class NetworkMap extends BaseExtension {
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-        this.mqtt.subscribe(this.topicRoutes);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+            this.mqtt.subscribe(this.legacyTopicRoutes);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        if ((topic === this.topic || topic === this.topicRoutes) && this.supportedFormats.hasOwnProperty(message)) {
-            const includeRoutes = topic === this.topicRoutes;
-            const topology = await this.networkScan(includeRoutes);
-            const converted = this.supportedFormats[message](topology);
-            this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(message)) {
+                const includeRoutes = topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                const converted = this.supportedFormats[message](topology);
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            }
         }
     }
 
@@ -61,9 +71,9 @@ class NetworkMap extends BaseExtension {
 
             // Add the device model
             if (device.type !== 'Coordinator') {
-                const mappedModel = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                if (mappedModel) {
-                    labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                if (definition) {
+                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
                     labels.push(`${device.manufacturerName} ${device.modelID}`);
@@ -119,31 +129,31 @@ class NetworkMap extends BaseExtension {
 
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
-        const devices = this.zigbee.getDevices();
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
         const lqis = new Map();
         const routingTables = new Map();
         const failed = new Map();
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             try {
                 const result = await device.lqi();
                 lqis.set(device, result);
-                logger.debug(`LQI succeeded for '${resolved.name}'`);
+                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
                 failed.get(device).push('lqi');
-                logger.error(`Failed to execute LQI for '${resolved.name}'`);
+                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
             }
 
             if (includeRoutes) {
                 try {
                     const result = await device.routingTable();
                     routingTables.set(device, result);
-                    logger.debug(`Routing table succeeded for '${resolved.name}'`);
+                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {
                     failed.get(device).push('routingTable');
-                    logger.error(`Failed to execute routing table for '${resolved.name}'`);
+                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
                 }
             }
         }
@@ -153,9 +163,9 @@ class NetworkMap extends BaseExtension {
         const networkMap = {nodes: [], links: []};
         // Add nodes
         for (const device of devices) {
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             networkMap.nodes.push({
-                ieeeAddr: device.ieeeAddr, friendlyName: resolved.name, type: device.type,
+                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
             });

lib/extension/onEvent.js

@@ -0,0 +1,37 @@
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends Extension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && resolvedEntity.type === 'device') {
+            this.callOnEvent(resolvedEntity, type, data);
+        }
+    }
+
+    async stop() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'stop', {});
+        }
+    }
+
+    callOnEvent(resolvedEntity, type, data) {
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+
+        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.js

@@ -1,26 +1,30 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
 const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
-const BaseExtension = require('./baseExtension');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const Extension = require('./extension');
 const MINUTES_10 = 1000 * 60 * 10;
 
-class OTAUpdate extends BaseExtension {
+class OTAUpdate extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.inProgress = new Set();
         this.lastChecked = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        }
     }
 
-    async onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (data.type !== 'commandQueryNextImageRequest') return;
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
 
-        const supportsOTA = mappedDevice && mappedDevice.hasOwnProperty('ota');
+        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
         if (supportsOTA) {
             // When a device does a next image request, it will usually do it a few times after each other
             // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
@@ -30,13 +34,21 @@ class OTAUpdate extends BaseExtension {
             if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
-            const available = await mappedDevice.ota.isUpdateAvailable(data.device, logger, data.data);
+            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
             this.publishEntityState(data.device.ieeeAddr, {update_available: available});
 
             if (available) {
-                const message = `Update available for '${settingsDevice.friendly_name}'`;
+                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
                 logger.info(message);
-                this.mqtt.log('ota_update', message, {status: 'available', device: settingsDevice.friendly_name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
             }
         }
 
@@ -44,7 +56,10 @@ class OTAUpdate extends BaseExtension {
         // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
         // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
         const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
-        await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
     }
 
     async readSoftwareBuildIDAndDateCode(device, update) {
@@ -65,76 +80,140 @@ class OTAUpdate extends BaseExtension {
     }
 
     async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
+        let resolvedEntity = null;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (!topic.match(legacyTopicRegex)) {
+                return null;
+            }
+
+            resolvedEntity = this.zigbee.resolveEntity(message);
+        } else {
             return null;
         }
 
-        const device = this.zigbee.resolveEntity(message);
-        assert(device != null && device.type === 'device', 'Device not found or not a device');
-        if (!device.mapped || !device.mapped.ota) {
-            const message = `Device '${device.name}' does not support OTA updates`;
+        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
+        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
             logger.error(message);
-            this.mqtt.log('ota_update', message, {status: `not_supported`, device: device.name});
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
             return;
         }
 
-        if (this.inProgress.has(device.device.ieeeAddr)) {
-            logger.error(`Update or check already in progress for '${device.name}', skipping...`);
+        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
             return;
         }
-        this.inProgress.add(device.device.ieeeAddr);
+        this.inProgress.add(resolvedEntity.device.ieeeAddr);
 
         const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
         if (type === 'check') {
-            const message = `Checking if update available for '${device.name}'`;
+            const message = `Checking if update available for '${resolvedEntity.name}'`;
             logger.info(message);
-            this.mqtt.log('ota_update', message, {status: `checking_if_available`, device: device.name});
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
             try {
-                const available = await device.mapped.ota.isUpdateAvailable(device.device, logger);
-                const message=(available ?
-                    `Update available for '${device.name}'` : `No update available for '${device.name}'`);
+                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
+                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
                 logger.info(message);
-                const meta = {status: available ? 'available' : 'not_available', device: device.name};
-                this.mqtt.log('ota_update', message, meta);
-                this.publishEntityState(device.device.ieeeAddr, {update_available: available});
-                this.lastChecked[device.device.ieeeAddr] = Date.now();
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
+                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
             } catch (error) {
-                const message = `Failed to check if update available for '${device.name}' (${error.message})`;
+                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
                 logger.error(message);
-                this.mqtt.log('ota_update', message, {status: `check_failed`, device: device.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `check_failed`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
             }
         } else { // type === 'update'
-            const message = `Updating '${device.name}' to latest firmware`;
+            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
             logger.info(message);
-            this.mqtt.log('ota_update', message, {status: `update_in_progress`, device: device.name});
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
             try {
                 const onProgress = (progress, remaining) => {
-                    let message = `Update of '${device.name}' at ${progress}%`;
+                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
                     if (remaining) {
                         message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
                     }
 
                     logger.info(message);
-                    this.mqtt.log('ota_update', message, {status: `update_progress`, device: device.name, progress});
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    }
                 };
 
-                const from_ = await this.readSoftwareBuildIDAndDateCode(device.device, false);
-                await device.mapped.ota.updateToLatest(device.device, logger, onProgress);
-                const to = await this.readSoftwareBuildIDAndDateCode(device.device, true);
+                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
                 const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
-                const message = `Finished update of '${device.name}'` + (to ? `, from '${fromS}' to '${toS}'` : ``);
+                const message = `Finished update of '${resolvedEntity.name}'` +
+                    (to ? `, from '${fromS}' to '${toS}'` : ``);
                 logger.info(message);
-                const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
-                this.mqtt.log('ota_update', message, meta);
-                this.publishEntityState(device.device.ieeeAddr, {update_available: false});
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
             } catch (error) {
-                const message = `Update of '${device.name}' failed (${error.message})`;
+                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
                 logger.error(message);
-                this.mqtt.log('ota_update', message, {status: `update_failed`, device: device.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_failed`, device: resolvedEntity.name};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
             }
         }
 
-        this.inProgress.delete(device.device.ieeeAddr);
+        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
     }
 }
 

lib/extension/publish.js

@@ -4,10 +4,9 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
 
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
@@ -21,7 +20,7 @@ const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
 ];
 
-class EntityPublish extends BaseExtension {
+class EntityPublish extends Extension {
     onMQTTConnected() {
         // Subscribe to topics.
         const baseTopic = settings.get().mqtt.base_topic;
@@ -46,7 +45,7 @@ class EntityPublish extends BaseExtension {
             return null;
         }
 
-        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
+        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
     }
 
     async onMQTTMessage(topic, message) {
@@ -55,12 +54,20 @@ class EntityPublish extends BaseExtension {
             return false;
         }
 
-        const entityName = `${topic.ID}` + (topic.postfix ? `/${topic.postfix}` : '');
-        const entity = this.zigbee.resolveEntity(entityName);
+        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
+        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
 
-        if (!entity) {
-            this.mqtt.log('entity_not_found', {friendly_name: entityName});
-            logger.error(`Entity '${entityName}' is unknown`);
+        if (!resolvedEntity) {
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {friendly_name: entityKey};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `entity_not_found`, message}),
+                );
+            }
+
+            logger.error(`Entity '${entityKey}' is unknown`);
             return;
         }
 
@@ -69,26 +76,26 @@ class EntityPublish extends BaseExtension {
         let target = null;
         let options = {};
         let device = null;
-        let mapped = null;
+        let definition = null;
 
-        assert(entity.type === 'device' || entity.type === 'group');
-        if (entity.type === 'device') {
-            // Map device to a model
-            if (!entity.mapped) {
-                logger.warn(`Device with modelID '${entity.device.modelID}' is not supported.`);
+        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
+        if (resolvedEntity.type === 'device') {
+            if (!resolvedEntity.definition) {
+                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
                 return;
             }
 
-            device = entity.device;
-            mapped = entity.mapped;
-            target = entity.endpoint;
-            converters = entity.mapped.toZigbee;
-            options = entity.settings;
+            device = resolvedEntity.device;
+            definition = resolvedEntity.definition;
+            target = resolvedEntity.endpoint;
+            converters = resolvedEntity.definition.toZigbee;
+            options = resolvedEntity.settings;
         } else {
             converters = groupConverters;
-            target = entity.group;
-            options = entity.settings;
+            target = resolvedEntity.group;
+            options = resolvedEntity.settings;
+            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
         }
 
         // Convert the MQTT message to a Zigbee message.
@@ -104,12 +111,13 @@ class EntityPublish extends BaseExtension {
             }
         }
 
+
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(entity.settings.ID) || {};
+        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
@@ -121,25 +129,34 @@ class EntityPublish extends BaseExtension {
             }
         }
 
-        // Ensure that state and brightness are executed before other commands.
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
         const entries = Object.entries(json);
-        entries.sort((a, b) => (['state', 'brightness'].includes(a[0]) ? -1 : 1));
+        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
 
         // For each attribute call the corresponding converter
         const usedConverters = [];
         for (let [key, value] of entries) {
-            let postfix = topic.postfix;
+            let endpointName = topic.endpointName;
             let actualTarget = target;
 
-            // When the key has a postfix included (e.g. state_right), this will override the target.
-            if (entity.type === 'device' && key.includes('_')) {
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            if (resolvedEntity.type === 'device' && key.includes('_')) {
                 const underscoreIndex = key.lastIndexOf('_');
-                const possiblePostfix = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getPostfixes().includes(possiblePostfix)) {
-                    postfix = possiblePostfix;
+                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
+                if (utils.getEndpointNames().includes(possibleEndpointName)) {
+                    endpointName = possibleEndpointName;
                     key = key.substring(0, underscoreIndex);
                     const device = target.getDevice();
-                    actualTarget = device.getEndpoint(mapped.endpoint(device)[postfix]);
+                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
                 }
             }
 
@@ -157,32 +174,32 @@ class EntityPublish extends BaseExtension {
 
             // Converter didn't return a result, skip
             const meta = {
-                endpoint_name: postfix,
+                endpoint_name: endpointName,
                 options,
                 message: json,
                 logger,
                 device,
                 state: deviceState,
-                mapped,
+                mapped: definition,
             };
 
             try {
                 if (topic.type === 'set' && converter.convertSet) {
-                    logger.debug(`Publishing '${topic.type}' '${key}' to '${entity.name}'`);
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
                     if (result && result.state) {
                         const msg = result.state;
 
-                        if (postfix) {
-                            for (const key of ['state', 'brightness']) {
+                        if (endpointName) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                                 if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${postfix}`] = msg[key];
+                                    msg[`${key}_${endpointName}`] = msg[key];
                                     delete msg[key];
                                 }
                             }
                         }
 
-                        this.publishEntityState(entity.settings.ID, msg);
+                        this.publishEntityState(resolvedEntity.settings.ID, msg);
                     }
 
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
@@ -193,13 +210,13 @@ class EntityPublish extends BaseExtension {
                     // ever issue a read here, as we assume the device will properly report changes.
                     // Only do this when the retrieve_state option is enabled for this device.
                     if (
-                        entity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
-                        entity.settings.retrieve_state
+                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        resolvedEntity.settings.retrieve_state
                     ) {
                         setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
                     }
                 } else if (topic.type === 'get' && converter.convertGet) {
-                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${entity.name}'`);
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     await converter.convertGet(actualTarget, key, meta);
                 } else {
                     logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
@@ -207,10 +224,18 @@ class EntityPublish extends BaseExtension {
                 }
             } catch (error) {
                 const message =
-                    `Publish '${topic.type}' '${key}' to '${entity.name}' failed: '${error}'`;
+                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
                 logger.error(message);
                 logger.debug(error.stack);
-                this.mqtt.log('zigbee_publish_error', message, {friendly_name: entity.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                    );
+                }
             }
 
             usedConverters.push(converter);

lib/extension/receive.js

@@ -2,12 +2,11 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class DeviceReceive extends BaseExtension {
+class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.coordinator = null;
         this.elapsed = {};
         this.debouncers = {};
     }
@@ -54,8 +53,8 @@ class DeviceReceive extends BaseExtension {
         return result;
     }
 
-    canHandleEvent(type, data, mappedDevice) {
-        if (type !== 'message') {
+    shouldProcess(type, data, resolvedEntity) {
+        if (type !== 'message' || !resolvedEntity) {
             return false;
         }
 
@@ -83,12 +82,12 @@ class DeviceReceive extends BaseExtension {
             return false;
         }
 
-        if (data.device.modelID === null && data.device.interviewing) {
+        if (!data.device.modelID && data.device.interviewing) {
             logger.debug(`Skipping message, modelID is undefined and still interviewing`);
             return false;
         }
 
-        if (!mappedDevice) {
+        if (!resolvedEntity.definition) {
             logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
             logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
             return false;
@@ -97,27 +96,27 @@ class DeviceReceive extends BaseExtension {
         return true;
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (!this.canHandleEvent(type, data, mappedDevice)) {
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
 
-        const converters = mappedDevice.fromZigbee.filter((c) => {
+        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
             const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
             return c.cluster === data.cluster && type;
         });
 
-        // Check if there is an available converter
-        if (!converters.length) {
+        // Check if there is an available converter, genOta messages are not interesting.
+        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
             logger.debug(
-                `No converter available for '${mappedDevice.model}' with cluster '${data.cluster}' ` +
+                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
                 `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
             );
             return;
         }
 
-        const debounce = settingsDevice.debounce || settings.get().device_options.debounce;
-        const debounceIgnore = settingsDevice.debounce_ignore || settings.get().device_options.debounce_ignore;
+        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
+        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
 
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
@@ -144,21 +143,6 @@ class DeviceReceive extends BaseExtension {
                 this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
-
-                if (settings.get().homeassistant && settings.get().advanced.homeassistant_legacy_triggers) {
-                    /**
-                     * Publish an empty value for click and action payload, in this way Home Assistant
-                     * can use Home Assistant entities in automations.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-                     */
-                    Object.keys(payload).forEach((key) => {
-                        if (['action', 'click'].includes(key)) {
-                            const counterPayload = {};
-                            counterPayload[key] = '';
-                            this.publishEntityState(data.device.ieeeAddr, counterPayload);
-                        }
-                    });
-                }
             }
         };
 
@@ -166,7 +150,7 @@ class DeviceReceive extends BaseExtension {
         let payload = {};
         converters.forEach((converter) => {
             const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
-            const converted = converter.convert(mappedDevice, data, publish, options, meta);
+            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
             if (converted) {
                 payload = {...payload, ...converted};
             }
@@ -178,4 +162,4 @@ class DeviceReceive extends BaseExtension {
     }
 }
 
-module.exports = DeviceReceive;
+module.exports = Receive;

lib/extension/report.js

@@ -1,10 +1,8 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const CC2530Router = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
-const BASICZBR3 = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3');
 const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
 const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 const debounce = require('debounce');
 const ZigbeeHerdsman = require('zigbee-herdsman');
 
@@ -12,10 +10,27 @@ const defaultConfiguration = {
     minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
 };
 
-const devicesNotSupportingReporting = [CC2530Router, BASICZBR3];
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+];
 
 const reportKey = 1;
 
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
 const clusters = {
     'genOnOff': [
         {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
@@ -24,9 +39,18 @@ const clusters = {
         {attribute: 'currentLevel', ...defaultConfiguration},
     ],
     'lightingColorCtrl': [
-        {attribute: 'colorTemperature', ...defaultConfiguration},
-        {attribute: 'currentX', ...defaultConfiguration},
-        {attribute: 'currentY', ...defaultConfiguration},
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
     ],
     'closuresWindowCovering': [
         {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
@@ -76,16 +100,16 @@ const pollOnMessage = [
     },
 ];
 
-class DeviceReport extends BaseExtension {
+class Report extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.configuring = new Set();
+        this.queue = new Set();
         this.failed = new Set();
         this.pollDebouncers = {};
     }
 
-    shouldIgnoreClusterForDevice(cluster, mappedDevice) {
-        if (mappedDevice === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
             // Device announces it but doesn't support it
             // https://github.com/Koenkk/zigbee2mqtt/issues/2611
             return true;
@@ -94,17 +118,28 @@ class DeviceReport extends BaseExtension {
         return false;
     }
 
-    async setupReporting(device, mappedDevice) {
-        if (this.configuring.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
-        this.configuring.add(device.ieeeAddr);
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
+
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
 
         try {
             for (const ep of device.endpoints) {
                 for (const [cluster, configuration] of Object.entries(clusters)) {
-                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, mappedDevice)) {
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
                         logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                items.push({...entry});
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
                         await ep.bind(cluster, this.coordinatorEndpoint);
-                        await ep.configureReporting(cluster, configuration);
+                        await ep.configureReporting(cluster, items);
                         logger.info(
                             `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
                         );
@@ -123,12 +158,14 @@ class DeviceReport extends BaseExtension {
         }
 
         device.save();
-        this.configuring.delete(device.ieeeAddr);
+        this.queue.delete(device.ieeeAddr);
     }
 
-    shouldSetupReporting(mappedDevice, device, messageType) {
-        if (!device || !mappedDevice) return false;
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
 
+        const {device, definition} = resolvedEntity;
         // Handle messages of type endDeviceAnnce and devIncoming.
         // This message is typically send when a device comes online after being powered off
         // Ikea TRADFRI tend to forget their reporting after powered off.
@@ -139,7 +176,7 @@ class DeviceReport extends BaseExtension {
         if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
         if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
         // Gledopto devices don't support reporting.
-        if (devicesNotSupportingReporting.includes(mappedDevice) || mappedDevice.vendor === 'Gledopto') return false;
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
         return true;
     }
 
@@ -147,16 +184,16 @@ class DeviceReport extends BaseExtension {
         this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
 
         for (const device of this.zigbee.getClients()) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-            if (this.shouldSetupReporting(mappedDevice, device, null)) {
-                this.setupReporting(device, mappedDevice);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                this.setupReporting(resolvedEntity);
             }
         }
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (this.shouldSetupReporting(mappedDevice, data.device, type)) {
-            this.setupReporting(data.device, mappedDevice);
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            this.setupReporting(resolvedEntity);
         }
 
         if (type === 'message') {
@@ -182,7 +219,8 @@ class DeviceReport extends BaseExtension {
             let toPoll = [];
 
             // Add bound devices
-            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) => e.binds.map((e) => e))));
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
             toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
             toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
             toPoll = toPoll.map((e) => e.target);
@@ -215,4 +253,4 @@ class DeviceReport extends BaseExtension {
     }
 }
 
-module.exports = DeviceReport;
+module.exports = Report;

lib/mqtt.js

@@ -95,11 +95,15 @@ class MQTT extends events.EventEmitter {
         this.emit('message', {topic, message: message.toString()});
     }
 
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
+
     async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
-        if (!this.client || this.client.reconnecting) {
+        if (!this.isConnected()) {
             logger.error(`Not connected to MQTT server!`);
             logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
             return;
@@ -111,16 +115,6 @@ class MQTT extends events.EventEmitter {
             this.client.publish(topic, payload, options, () => resolve());
         });
     }
-
-    log(type, message, meta=null) {
-        const payload = {type, message};
-
-        if (meta) {
-            payload.meta = meta;
-        }
-
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
-    }
 }
 
 module.exports = MQTT;

lib/state.js

@@ -2,23 +2,21 @@ const logger = require('./util/logger');
 const data = require('./util/data');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
-const events = require('events');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'action_color_temperature', 'action_color',
-    'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
     'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
-class State extends events.EventEmitter {
-    constructor() {
-        super();
+class State {
+    constructor(eventBus) {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
+        this.eventBus = eventBus;
     }
 
     start() {
@@ -70,17 +68,18 @@ class State extends events.EventEmitter {
 
     set(ID, state, reason=null) {
         const toState = objectAssignDeep.noMutate(state);
-        dontCacheProperties.forEach((property) => {
-            if (toState.hasOwnProperty(property)) {
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
                 delete toState[property];
             }
-        });
+        }
 
         const fromState = this.state[ID];
 
         this.state[ID] = toState;
 
-        this.emit('stateChange', {ID, from: fromState, to: toState, reason});
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
     }
 
     remove(ID) {

lib/util/logger.js

@@ -54,22 +54,27 @@ const transportsToUse = [
 
 // Add file logger when enabled
 // NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            const {timestamp, level, message} = info;
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
 if (output.includes('file')) {
-    transportsToUse.push(new winston.transports.File({
-        filename: path.join(directory, logFilename),
-        json: false,
-        level,
-        tailable: true,
-        maxFiles: 3, // Keep last 3 files
-        maxsize: 10000000, // 10MB
-        format: winston.format.combine(
-            winston.format.timestamp({format: timestampFormat}),
-            winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
-                return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
-            }),
-        ),
-    }));
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
 }
 
 // Create logger

lib/util/settings.js

@@ -43,6 +43,8 @@ const defaults = {
         output: 'json',
     },
     advanced: {
+        legacy_api: true,
+        log_rotation: true,
         log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
         log_file: 'log.txt',
@@ -53,6 +55,7 @@ const defaults = {
         channel: 11,
         baudrate: 115200,
         rtscts: true,
+        adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
@@ -148,6 +151,7 @@ const schema = {
             properties: {
                 port: {type: ['string', 'null']},
                 disable_led: {type: 'boolean'},
+                adapter: {type: 'string', enum: ['deconz', 'zstack']},
             },
         },
         ban: {type: 'array', items: {type: 'string'}},
@@ -161,10 +165,12 @@ const schema = {
         advanced: {
             type: 'object',
             properties: {
+                legacy_api: {type: 'boolean'},
                 pan_id: {type: 'number'},
                 ext_pan_id: {type: 'array', items: {type: 'number'}},
                 channel: {type: 'number', minimum: 11, maximum: 26},
                 cache_state: {type: 'boolean'},
+                log_rotation: {type: 'boolean'},
                 log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
                 log_output: {type: 'array', items: {type: 'string'}},
                 log_directory: {type: 'string'},
@@ -172,7 +178,7 @@ const schema = {
                 baudrate: {type: 'number'},
                 rtscts: {type: 'boolean'},
                 soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: 'array', items: {type: 'number'}},
+                network_key: {type: ['array', 'string'], items: {type: 'number'}},
                 last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
                 elapsed: {type: 'boolean'},
                 availability_timeout: {type: 'number', minimum: 0},
@@ -182,6 +188,7 @@ const schema = {
                 homeassistant_discovery_topic: {type: 'string'},
                 homeassistant_status_topic: {type: 'string'},
                 timestamp_format: {type: 'string'},
+                adapter_concurrent: {type: 'number'},
             },
         },
         map_options: {
@@ -275,13 +282,16 @@ function write() {
 
     // Read settings to check if we have to split devices/groups into separate file.
     const actual = yaml.read(file);
-    if (actual.mqtt && actual.mqtt.password && actual.mqtt.user) {
-        toWrite.mqtt.user = actual.mqtt.user;
-        toWrite.mqtt.password = actual.mqtt.password;
-    }
 
-    if (actual.advanced && actual.advanced.network_key) {
-        toWrite.advanced.network_key = actual.advanced.network_key;
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
     }
 
     if (typeof actual.devices === 'string') {
@@ -303,7 +313,12 @@ function write() {
 function validate() {
     const validate = ajv.compile(schema);
     const valid = validate(_settings);
-    const postfixes = utils.getPostfixes();
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    const postfixes = utils.getEndpointNames();
 
     // Verify that all friendly names are unique
     const names = [];
@@ -311,19 +326,35 @@ function validate() {
         if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
         if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
         if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+        if (name.includes('#') || name.includes('+')) {
+            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+        }
         names.push(name);
     };
-    Object.values(_settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
-    Object.values(_settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
 
-    if (_settingsWithDefaults.mqtt.version !== 5) {
-        for (const device of Object.values(_settingsWithDefaults.devices)) {
+    const settingsWithDefaults = getWithDefaults();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
             if (device.retention) {
                 throw new Error('MQTT retention requires protocol version 5');
             }
         }
     }
 
+    const checkAvailabilityList = (list, type) => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+
     return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
 }
 
@@ -587,6 +618,9 @@ function removeDeviceFromGroup(groupIDorName, keys) {
     const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
     const settings = get();
     const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
 
     const key = groupHasDevice(group, keys);
     if (key) {
@@ -605,14 +639,7 @@ function removeGroup(groupIDorName) {
 function changeDeviceOptions(IDorName, newOptions) {
     const device = getDeviceThrowIfNotExists(IDorName);
     const settings = get();
-    const currentOptions = settings.devices[device.ID];
-
-    Object.keys(currentOptions).forEach((key) => {
-        if (newOptions[key]) {
-            currentOptions[key] = newOptions[key];
-        }
-    });
-
+    objectAssignDeep(settings.devices[device.ID], newOptions);
     write();
 }
 

lib/util/utils.js

@@ -27,11 +27,15 @@ function toLocalISOString(dDate) {
         ':' + pad(tzOffset % 60);
 }
 
-const postfixes = [
+const endpointNames = [
     'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
     'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
     'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
 function flatten(arr) {
@@ -43,7 +47,8 @@ function flatten(arr) {
 const forceEndDevice = flatten(
     ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
         .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
-        .map((mappedModel) => mappedModel.zigbeeModel));
+        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
+        .map((definition) => definition.zigbeeModel));
 
 async function getZigbee2mqttVersion() {
     return new Promise((resolve, reject) => {
@@ -119,13 +124,23 @@ function getObjectsProperty(objects, key, defaultValue) {
     return defaultValue;
 }
 
+/* istanbul ignore next newApi */
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
     objectHasProperties,
     getObjectsProperty,
-    getPostfixes: () => postfixes,
+    getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
         return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
             (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
@@ -135,4 +150,5 @@ module.exports = {
     isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     formatDate: (date, type) => formatDate(date, type),
     equalsPartial,
+    getResponse,
 };

lib/util/yaml.js

@@ -33,4 +33,12 @@ function writeIfChanged(file, content) {
     }
 }
 
-module.exports = {read, readIfExists, writeIfChanged};
+function updateIfChanged(file, key, value) {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/zigbee.js

@@ -8,26 +8,9 @@ const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
-const postfixes = utils.getPostfixes();
+const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-const herdsmanSettings = {
-    network: {
-        panID: settings.get().advanced.pan_id,
-        extendedPanID: settings.get().advanced.ext_pan_id,
-        channelList: [settings.get().advanced.channel],
-        networkKey: settings.get().advanced.network_key,
-    },
-    databasePath: data.joinPath('database.db'),
-    databaseBackupPath: data.joinPath('database.db.backup'),
-    backupPath: data.joinPath('coordinator_backup.json'),
-    serialPort: {
-        baudRate: settings.get().advanced.baudrate,
-        rtscts: settings.get().advanced.rtscts,
-        path: settings.get().serial.port,
-    },
-};
-
 class Zigbee extends events.EventEmitter {
     constructor() {
         super();
@@ -36,10 +19,37 @@ class Zigbee extends events.EventEmitter {
 
     async start() {
         logger.info(`Starting zigbee-herdsman...`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key,
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+            },
+        };
+
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
         logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
 
+        if (herdsmanSettings.network.networkKey === 'GENERATE') {
+            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+            settings.set(['advanced', 'network_key'], newKey);
+            herdsmanSettings.network.networkKey = newKey;
+        }
+
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
             this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
@@ -127,10 +137,23 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getDevicesByType(type);
     }
 
+    /**
+     * @param {string} key
+     * @return {object} {
+     *      type: device | coordinator
+     *      device|group: zigbee-herdsman entity
+     *      endpoint: selected endpoint (only if type === device)
+     *      settings: from configuration.yaml
+     *      name: name of the entity
+     *      definition: zigbee-herdsman-converters definition (only if type === device)
+     * }
+     */
     resolveEntity(key) {
+        /* istanbul ignore next newApi */
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device', `Wrong type '${typeof key}'`,
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
         );
 
         if (typeof key === 'string' || typeof key === 'number') {
@@ -149,13 +172,13 @@ class Zigbee extends events.EventEmitter {
                 };
             }
 
-            let postfix = postfixes.find((p) => key.endsWith(`/${p}`));
-            const postfixByNumber = key.match(keyEndpointByNumber);
-            if (!postfix && postfixByNumber) {
-                postfix = Number(postfixByNumber[1]);
+            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            const endpointByNumber = key.match(keyEndpointByNumber);
+            if (!endpointKey && endpointByNumber) {
+                endpointKey = Number(endpointByNumber[1]);
             }
-            if (postfix) {
-                key = key.replace(`/${postfix}`, '');
+            if (endpointKey) {
+                key = key.replace(`/${endpointKey}`, '');
             }
 
             const entity = settings.getEntity(key);
@@ -167,19 +190,17 @@ class Zigbee extends events.EventEmitter {
                     return null;
                 }
 
-                const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                const endpoints = mapped && mapped.endpoint ? mapped.endpoint(device) : null;
-                let isDefaultEndpoint = true;
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
                 let endpoint;
-                if (postfix) {
-                    isDefaultEndpoint = false;
-                    if (postfixByNumber) {
-                        endpoint = device.getEndpoint(postfix);
+                if (endpointKey) {
+                    if (endpointByNumber) {
+                        endpoint = device.getEndpoint(endpointKey);
                     } else {
-                        assert(mapped != null, `Postfix '${postfix}' is given but device is unsupported`);
-                        assert(endpoints != null, `Postfix '${postfix}' is given but device defines no endpoints`);
-                        const endpointID = endpoints[postfix];
-                        assert(endpointID, `Postfix '${postfix}' is given but device has no such endpoint`);
+                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
+                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpoints[endpointKey];
+                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
                         endpoint = device.getEndpoint(endpointID);
                     }
                 } else if (endpoints && endpoints['default']) {
@@ -188,24 +209,31 @@ class Zigbee extends events.EventEmitter {
                     endpoint = device.endpoints[0];
                 }
 
-                const endpointName = endpoints ? Object.entries(endpoints).find((e) => e[1] === endpoint.ID)[0] : null;
                 return {
-                    type: 'device', device, settings: entity, mapped, endpoint, name: entity.friendlyName,
-                    isDefaultEndpoint, endpointName,
+                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
                 return {type: 'group', group, settings: entity, name: entity.friendlyName};
             }
-        } else {
+        } /* istanbul ignore else newApi */ else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
             return {
                 type: 'device',
                 device: key,
+                endpoint: key.endpoints[0],
                 settings: setting,
-                mapped: zigbeeHerdsmanConverters.findByZigbeeModel(key.modelID),
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
+                definition: zigbeeHerdsmanConverters.findByDevice(key),
+            };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: setting,
+                name: setting.friendlyName,
             };
         }
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.12.0",
+  "version": "1.13.1",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -45,8 +45,8 @@
     "rimraf": "*",
     "semver": "*",
     "winston": "*",
-    "zigbee-herdsman": "0.12.64",
-    "zigbee-herdsman-converters": "12.0.51"
+    "zigbee-herdsman": "0.12.90",
+    "zigbee-herdsman-converters": "12.0.92"
   },
   "devDependencies": {
     "eslint": "*",

test/assets/exampleExtension.js

@@ -0,0 +1,11 @@
+class Example {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.mqtt = mqtt;
+    }
+
+    onMQTTConnected() {
+        this.mqtt.publish('example/extension', 'test')
+    }
+}
+
+module.exports = Example;

test/availability.test.js

@@ -15,11 +15,11 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device availability', () => {
+describe('Availability', () => {
     let controller;
 
     function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'DeviceAvailability');
+        return controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
     beforeEach(async () => {

test/bind.test.js

@@ -6,7 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
-describe('Device bind', () => {
+describe('Bind', () => {
     let controller;
 
     mockClear = (device) => {
@@ -54,6 +54,17 @@ describe('Device bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'button');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
+    });
+
     it('Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);

test/configure.test.js

@@ -9,7 +9,7 @@ const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device configure', () => {
+describe('Configure', () => {
     let controller;
 
     expectRemoteConfigured = () => {
@@ -153,7 +153,7 @@ describe('Device configure', () => {
     });
 
     it('Should configure max 3 times when fails', async () => {
-        controller.extensions.find((e) => e.constructor.name === 'DeviceConfigure').attempts = {};
+        controller.extensions.find((e) => e.constructor.name === 'Configure').attempts = {};
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
         const endpoint = device.getEndpoint(1);

test/controller.test.js

@@ -30,7 +30,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),"serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -85,6 +85,14 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should generate network_key when set to GENERATE', async () => {
+        settings.set(['advanced', 'network_key'], 'GENERATE');
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(data.read().advanced.network_key.length).toStrictEqual(16);
+    });
+
     it('Start controller should publish cached states', async () => {
         data.writeDefaultState();
         await controller.start();
@@ -242,9 +250,16 @@ describe('Controller', () => {
         expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1 with groupID 0`);
     });
 
+    it('Should add entities which are missing from configuration but are in database to configuration', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
+    });
+
     it('On zigbee event message from unkown device should create it', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.notInSettings;
+        settings.removeDevice(device.ieeeAddr);
         expect(settings.getDevice(device.ieeeAddr)).toBeNull();
         const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
         await zigbeeHerdsman.events.message(payload);
@@ -522,4 +537,15 @@ describe('Controller', () => {
         await flushPromises();
         expect(controller.state.state).toStrictEqual({});
     });
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
 });

test/homeassistant.test.js

@@ -32,6 +32,27 @@ describe('HomeAssistant extension', () => {
         expect(missing).toHaveLength(0);
     });
 
+    it('Should not have duplicate type/object_ids in a mapping', () => {
+        const duplicated = [];
+        const HomeAssistant = require('../lib/extension/homeassistant');
+        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
+
+        require('zigbee-herdsman-converters').devices.forEach((d) => {
+            const mapping = ha._getMapping()[d.model];
+            const cfg_type_object_ids = [];
+
+            mapping.forEach((c) => {
+                if (cfg_type_object_ids.includes(c['type'] + '/' + c['object_id'])) {
+                    duplicated.push(d.model);
+                } else {
+                    cfg_type_object_ids.push(c['type'] + '/' + c['object_id']);
+                }
+            });
+        });
+
+        expect(duplicated).toHaveLength(0);
+    });
+
     it('Should discover devices', async () => {
         controller = new Controller(false);
         await controller.start();
@@ -496,7 +517,7 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        // 1 publish is the publish from deviceReceive
+        // 1 publish is the publish from receive
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
@@ -537,7 +558,7 @@ describe('HomeAssistant extension', () => {
         );
     });
 
-    it('Shouldnt discover when message has no device yet', async () => {
+    it('Shouldnt discover when device leaves', async () => {
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -564,13 +585,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',
+            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            '{"brightness":255}',
+            '{"brightness":255,"update_available":false}',
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -918,4 +939,25 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
+
+    it('Should counter an action/click payload with an empty payload', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/click_single/config');
+        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/click');
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -1,19 +1,19 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const MQTT = require('./stub/mqtt');
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+const MQTT = require('../stub/mqtt');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
 
-describe('Bridge config', () => {
+describe('Bridge legacy', () => {
     let controller;
 
     beforeAll(async () => {
-        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
         controller = new Controller();
         await controller.start();
     })
@@ -100,6 +100,21 @@ describe('Bridge config', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
+        );
     });
 
     it('Should allow permit join', async () => {
@@ -161,8 +176,8 @@ describe('Bridge config', () => {
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/config/devices');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
         expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
-        expect(payload[0]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
-        expect(payload[1]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
         Date.now = now;
     });
 

test/logger.test.js

@@ -98,4 +98,14 @@ describe('Logger', () => {
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
     });
+
+    it('Should allow to disable log rotation', () => {
+        settings.set(['advanced', 'log_rotation'], false);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].maxFiles).toBeNull();
+        expect(pipes[1].tailable).toBeFalsy();
+        expect(pipes[1].maxsize).toBeNull();
+    });
 });

test/onEvent.test.js

@@ -13,11 +13,11 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const mockOnEvent = jest.fn();
-const mappedLivolo = zigbeeHerdsmanConverters.findByZigbeeModel(zigbeeHerdsman.devices.LIVOLO.modelID);
+const mappedLivolo = zigbeeHerdsmanConverters.findByDevice(zigbeeHerdsman.devices.LIVOLO);
 mappedLivolo.onEvent = mockOnEvent;
 zigbeeHerdsmanConverters.onEvent = jest.fn();
 
-describe('Device event', () => {
+describe('On event', () => {
     let controller;
     const device = zigbeeHerdsman.devices.LIVOLO;
 

test/otaUpdate.test.js

@@ -38,7 +38,7 @@ describe('OTA update', () => {
             count++;
             return {swBuildId: count, dateCode: '2019010' + count}
         });
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
         logger.error.mockClear();
@@ -54,8 +54,8 @@ describe('OTA update', () => {
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0%`);
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
         expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
@@ -67,7 +67,7 @@ describe('OTA update', () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
         logger.error.mockClear();
@@ -84,7 +84,7 @@ describe('OTA update', () => {
 
     it('Should be able to check if OTA update is available', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
         logger.info.mockClear();
@@ -106,7 +106,7 @@ describe('OTA update', () => {
 
     it('Should handle if OTA update check fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.error.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
@@ -127,7 +127,7 @@ describe('OTA update', () => {
     it('Should refuse to check/update when already in progress', async () => {
         jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
         logger.info.mockClear();
@@ -154,7 +154,7 @@ describe('OTA update', () => {
             return {swBuildId: 1, dateCode: '2019010'}
         });
 
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
@@ -165,7 +165,7 @@ describe('OTA update', () => {
     it('Should check for update when device requests it', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const data = {imageType: 12382};
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
@@ -201,4 +201,15 @@ describe('OTA update', () => {
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
     });
+
+    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.error.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
 });

test/publish.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -21,7 +22,7 @@ const expectNothingPublished = () => {
     })
 }
 
-describe('Entity publish', () => {
+describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
@@ -48,6 +49,8 @@ describe('Entity publish', () => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => {
             g.command.mockClear();
         });
+
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
     it('Should publish messages to zigbee devices', async () => {
@@ -249,11 +252,11 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {x: 100, y: 50}, brightness: 20, state: 'OFF'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
-        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
@@ -586,6 +589,22 @@ describe('Entity publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
     });
 
+    it('Transition parameter should not influence brightness on state ON', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3563
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 50}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON"}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 10}, {}]);
+    });
+
     it('Should use transition when color temp', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
@@ -712,7 +731,6 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("closuresWindowCovering", "stop", {}, {});
     });
 
-
     it('Should turn device on with on/off when transition is provided', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -720,7 +738,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 30}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 30}, {}]);
     });
 
     it('Should turn device on with on/off with transition when transition 0 is provided', async () => {
@@ -730,7 +748,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 0}, {}]);
     });
 
     it('Should turn device off with onOff on off with transition', async () => {
@@ -778,6 +796,32 @@ describe('Entity publish', () => {
         expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 200, linkquality: 10}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
+    it('When device is turned off with transition and turned on WITHOUT transition it should restore the brightness', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        // Set initial brightness in state
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 200}));
+        await flushPromises();
+        endpoint.command.mockClear();
+        MQTT.publish.mockClear();
+
+        // Turn off
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF', transition: 3}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+
+        // Turn on again
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 0}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
     it('Home Assistant: should set state', async () => {
         settings.set(['homeassistant'], true);
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -999,4 +1043,58 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(0);
         expect(logger.error).toHaveBeenCalledWith("No converter available for 'get' 'brightness_move' (20)");
     });
+
+    it('Should restore brightness when its turned off without transition and is turned on with', async () => {
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/1097
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.command.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 123}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 0}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "OFF"}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1.0}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 10}, {}]);
+    });
+
+    it('Shouldnt use moveToLevelWithOnOff on turn on when no transition has been used as some devices do not turn on in that case', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3332
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'brightness': 150}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'OFF'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 150, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genOnOff", "on", {}, {}]);
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON"}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"OFF","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[3]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+    });
 });

test/receive.test.js

@@ -8,7 +8,7 @@ const flushPromises = () => new Promise(setImmediate);
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device receive', () => {
+describe('Receive', () => {
     let controller;
 
     beforeEach(async () => {
@@ -33,23 +33,6 @@ describe('Device receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-
-    it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', async () => {
-        settings.set(['homeassistant'], true);
-        const device = zigbeeHerdsman.devices.WXKG11LM;
-        const data = {onOff: 1}
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
     it('Should handle a zigbee message which uses ep (left)', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM;
         const data = {onOff: 1}

test/report.test.js

@@ -19,10 +19,10 @@ const debounce = require('debounce');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
 
-describe('Device report', () => {
+describe('Report', () => {
     let controller;
 
-    function expectOnOffBrightnessColorReport(endpoint) {
+    function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
@@ -31,7 +31,11 @@ describe('Device report', () => {
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
         expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 300, "minimumReportInterval": 0, "reportableChange": 0}]);
         expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
-        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
     }
 
     mockClear = (device) => {
@@ -57,7 +61,7 @@ describe('Device report', () => {
     it('Should configure reporting on startup', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, true);
     });
 
     it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
@@ -70,7 +74,7 @@ describe('Device report', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, false);
         expect(device.save).toHaveBeenCalledTimes(1);
     });
 
@@ -130,7 +134,7 @@ describe('Device report', () => {
         const payload = {device};
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, false);
     });
 
     it('Should not configure reporting on device leave', async () => {
@@ -211,4 +215,31 @@ describe('Device report', () => {
         expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
     });
+
+    it('Should not setup colorTemperature reporting when bulb does not support it and should read colorCapabilities when its not there yet ', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        endpoint.getClusterAttributeValue = jest.fn();
+
+        let count = 0;
+        endpoint.getClusterAttributeValue.mockImplementation((d) => {
+            count++;
+            if (count === 1) return undefined;
+            return 17;
+        });
+
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
 });

test/settings.test.js

@@ -125,6 +125,12 @@ describe('Settings', () => {
 
         settings._write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'user'], 'test123');
+        settings.set(['advanced', 'network_key'], [1,2,3, 4]);
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
     });
 
     it('Should read devices form a separate file', () => {
@@ -434,6 +440,31 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Shouldnt crash when removing device from group when group has no devices', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
+                '1': {
+                    friendly_name: 'test123',
+                }
+            }
+        });
+
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
     it('Should throw when adding device to non-existing group', () => {
         write(configurationFile, {
             devices: {
@@ -464,6 +495,19 @@ describe('Settings', () => {
         }).toThrow(new Error("Device '0x123' already exists"));
     });
 
+    it('Should not allow any string values for network_key', () => {
+        write(configurationFile, {
+            advanced: {network_key: 'NOT_GENERATE'},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+    });
+
+
     it('Should not allow retention configuration without MQTT v5', () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
@@ -476,6 +520,19 @@ describe('Settings', () => {
         }).toThrowError('MQTT retention requires protocol version 5');
     });
 
+    it('Should not allow non-existing entities in availability blacklist', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
+            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+    });
+
     it('Should ban devices', () => {
         write(configurationFile, {});
         settings.banDevice('0x123');
@@ -536,6 +593,18 @@ describe('Settings', () => {
         }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
     });
 
+    it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+    });
+
     it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
@@ -545,7 +614,7 @@ describe('Settings', () => {
 
         expect(() => {
             settings.validate();
-        }).toThrowError(`Following friendly_name are not allowed: '${utils.getPostfixes()}'`);
+        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {

test/stub/data.js

@@ -151,6 +151,9 @@ function writeDefaultConfiguration() {
             '0x0017880104e43559': {
                 friendly_name: 'U202DST600ZB'
             },
+            '0x0017880104e44559': {
+                friendly_name: '3157100_thermostat',
+            }
         },
         groups: {
             '1': {
@@ -211,6 +214,7 @@ writeDefaultState();
 
 module.exports = {
     mockDir,
+    read: () => yaml.read(path.join(mockDir, 'configuration.yaml')),
     writeDefaultConfiguration,
     writeDefaultState,
     removeState,

test/stub/zigbeeHerdsman.js

@@ -100,13 +100,13 @@ class Device {
 
 const returnDevices = [];
 
-const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3')], true, "Mains (single phase)", "LLC020");
+const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
 const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 
 const devices = {
     'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
-    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
+    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
@@ -139,12 +139,13 @@ const devices = {
     'LIVOLO': new Device('Router', '0x0017880104e45560', 6541,4152, [new Endpoint(6, [0, 6], [])], true, "Mains (single phase)", 'TI0001          '),
     'tradfri_remote': new Device('EndDevice', '0x90fd9ffffe4b64ae', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64ae')], true, "Battery", "TRADFRI remote control"),
     'roller_shutter': new Device('EndDevice', '0x90fd9ffffe4b64af', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64af')], true, "Battery", "SCM-R_00.00.03.15TC"),
-    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax')], true, "Mains (single phase)", "lumi.light.aqcn02"),
+    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "lumi.light.aqcn02"),
     'SP600_OLD': new Device('Router', '0x90fd9ffffe4b64aa', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20160120'),
     'SP600_NEW': new Device('Router', '0x90fd9ffffe4b64ab', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20170220'),
     'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
     'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
     'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
 }
 
 const groups = {