1.14.0

Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -1,23 +1,38 @@
 <!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2mqtt version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
 Did you read the FAQ?
-https://www.zigbee2mqtt.io/information/FAQ.html
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2mqtt fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
 
 Unsupported device?
-https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
 
-A question?
-Read the documentation: https://www.zigbee2mqtt.io/
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
 
-Bug report:
-- First make sure you are on the latest version of zigbee2mqtt.
-- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
-- Provide a clear description of the problem.
+Bug report?
 - If applicable, provide steps how to reproduce the problem.
-- Provide a log with `log_level` set to `debug`, see https://www.zigbee2mqtt.io/configuration/configuration.html
-- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
 -->
 
 <!--Start your bug report below this line-->
+
 # Bug Report
 ## What happened
 
@@ -26,5 +41,6 @@ Bug report:
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-zigbee2mqtt version:
-CC253X firmware version:
+Zigbee2mqtt version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.github/branch-switcher.yml

@@ -0,0 +1 @@
+preferredBranch: dev

.gitignore

@@ -67,6 +67,7 @@ data-backup/
 data/coordinator_backup.json
 data/.storage
 data/database.db.backup
+data/extension
 
 # MacOS indexing file
 .DS_Store

.npmignore

@@ -0,0 +1,25 @@
+#tests
+test
+coverage
+
+#build tools
+.travis.yml
+.jenkins.yml
+.codeclimate.yml
+.github
+azure-pipelines.yaml
+
+#linters
+.jscsrc
+.jshintrc
+.eslintrc*
+.dockerignore
+.eslintignore
+
+#editor settings
+.vscode
+
+#src
+docs
+docker
+images

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "*.yaml": "home-assistant"
+    }
+}

README.md

@@ -29,6 +29,9 @@
         <a href="https://www.codacy.com/manual/Koenkk/zigbee2mqtt?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=Koenkk/zigbee2mqtt&amp;utm_campaign=Badge_Grade">
             <img src="https://api.codacy.com/project/badge/Grade/24f1e0fe39f04daa810e8a1416693d3f">
         </a>
+        <a href="https://www.npmjs.com/package/zigbee2mqtt">
+            <img src="https://img.shields.io/npm/v/zigbee2mqtt">
+        </a>
     </div>
     <h1>Zigbee2mqtt  🌉 🐝</h1>
     <p>
@@ -63,6 +66,9 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ## Architecture
 ![Architecture](images/architecture.png)
 
+### Internal Architecture
+Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

azure-pipelines.yaml

@@ -14,9 +14,9 @@ steps:
 # Prepare
 - task: NodeTool@0
   inputs:
-    versionSpec: '10.x'
+    versionSpec: '12.x'
   displayName: 'Install Node.js'
-- bash: npm install
+- bash: npm ci
   displayName: 'Install dependencies'
 
 # Verify
@@ -25,12 +25,20 @@ steps:
 - bash: npm run eslint
   displayName: 'Lint'
 
+# NPM
+- task: Npm@1
+  inputs:
+    command: publish
+    publishEndpoint: npm
+  displayName: 'Publish to npm'
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
+
 # Docker
 - task: DockerInstaller@0
   displayName: Docker install
   condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
   inputs:
-    dockerVersion: 19.03.2
+    dockerVersion: 19.03.5
     releaseType: stable
 - task: Docker@2
   displayName: Docker registry login
@@ -41,7 +49,7 @@ steps:
 - bash: |
     sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
     sudo chmod a+x /usr/local/bin/buildx
-    docker run --rm --privileged hypriot/qemu-register
+    docker run --rm --privileged hypriot/qemu-register:v2.7.0
     buildx create --use
     buildx ls
   displayName: 'Docker setup'
@@ -68,15 +76,13 @@ steps:
   displayName: 'Docker build release'
   condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
 
-# Trigger downstream
+# Trigger Home Assistant Zigbee2mqtt addon build
 - bash: |
-    BODY="{\"request\": {\"branch\": \"master\"}}"
-    curl -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "Accept: application/json" \
-        -H "Travis-API-Version: 3" \
-        -H "Authorization: token $(TRAVIS_TOKEN)" \
-        -d "$BODY" \
-        "https://api.travis-ci.org/repo/danielwelch%2Fhassio-zigbee2mqtt/requests"
+    curl -s \
+        --request POST \
+        --header "Content-Type: application/json" \
+        --data '{"definition": {"id": 1}}' \
+        -u "$(HASSIO_TRIGGER_SECRET)" \
+        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
   displayName: 'Trigger Hass.io build'
   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

docker/Dockerfile

@@ -1,21 +1,30 @@
-FROM node:10-alpine
+FROM node:12-alpine as base
 
-# Copy files
-ADD . /app
-RUN cp /app/data/configuration.yaml /app
-RUN cp /app/docker/run.sh /app
-RUN chmod +x /app/run.sh
 WORKDIR /app
+RUN apk add --no-cache tzdata eudev
+
+COPY package.json .
+
+# Dependencies
+FROM base as dependencies
+
+COPY npm-shrinkwrap.json .
+
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
+    npm ci --production && \
+    apk del .buildtools
+
+# Release
+FROM base as release
+
+COPY --from=dependencies /app/node_modules ./node_modules
+COPY lib ./lib
+COPY index.js docker/run.sh data/configuration.yaml ./
+
+RUN chmod +x /app/run.sh
+RUN mkdir /app/data
 
-# Write .hash.json
 ARG COMMIT
 RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
 
-# Install dependencies
-RUN apk add --update --no-cache tzdata udev eudev && \
-    apk add --update --no-cache --virtual .buildtools make gcc g++ python linux-headers udev git && \
-    npm install --unsafe-perm --production && \
-    apk del .buildtools
-
-# Entrypoint
 ENTRYPOINT ["./run.sh"]

lib/controller.js

@@ -1,71 +1,88 @@
 const MQTT = require('./mqtt');
 const Zigbee = require('./zigbee');
+const EventBus = require('./eventBus');
 const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const fs = require('fs');
+const data = require('./util/data');
+const path = require('path');
 
 // Extensions
-const ExtensionEntityPublish = require('./extension/entityPublish');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/softReset');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
-const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionBridgeConfig = require('./extension/bridgeConfig');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
-const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
-const ExtensionDeviceBind = require('./extension/deviceBind');
-const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionDeviceEvent = require('./extension/deviceEvent');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/report');
+const ExtensionOnEvent = require('./extension/onEvent');
+const ExtensionOTAUpdate = require('./extension/otaUpdate');
 
 class Controller {
     constructor() {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
-        this.state = new State();
+        this.eventBus = new EventBus();
+        this.state = new State(this.eventBus);
 
         this.publishEntityState = this.publishEntityState.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
 
         // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionEntityPublish(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceReceive(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceGroupMembership(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceConfigure(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceEvent(this.zigbee, this.mqtt, this.state, this.publishEntityState),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
+            new ExtensionOTAUpdate(...args),
         ];
 
+        if (settings.get().experimental.new_api) {
+            this.extensions.push(new ExtensionBridge(...args));
+        }
+
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
+
         if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState,
-            ));
+            this.extensions.push(new ExtensionReport(...args));
         }
 
         if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState,
-            ));
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
         /* istanbul ignore next */
         if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState,
-            ));
+            this.extensions.push(new ExtensionSoftReset(...args));
         }
 
         if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState,
-            ));
+            this.extensions.push(new ExtensionAvailability(...args));
+        }
+
+        const extensionPath = data.joinPath('extension');
+        if (fs.existsSync(extensionPath)) {
+            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
+            for (const extension of extensions) {
+                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
+                this.extensions.push(new Extension(...args, settings, logger));
+            }
         }
     }
 
@@ -89,9 +106,10 @@ class Controller {
 
         // Start zigbee
         try {
+            await this.zigbee.start();
+            this.callExtensionMethod('onZigbeeStarted', []);
             this.zigbee.on('event', this.onZigbeeEvent.bind(this));
             this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
-            await this.zigbee.start();
         } catch (error) {
             logger.error('Failed to start zigbee');
             logger.error('Exiting...');
@@ -107,8 +125,8 @@ class Controller {
             logger.info(
                 (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
                 ` (${entity.device.ieeeAddr}): ` +
-                (entity.mapped ?
-                    `${entity.mapped.model} - ${entity.mapped.vendor} ${entity.mapped.description} ` :
+                (entity.definition ?
+                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
                     'Not supported ') +
                 `(${entity.device.type})`,
             );
@@ -121,10 +139,7 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        this.zigbee.permitJoin(settings.get().permit_join);
-
-        // Call extensions
-        this.callExtensionMethod('onZigbeeStarted', []);
+        await this.zigbee.permitJoin(settings.get().permit_join);
 
         // MQTT
         this.mqtt.on('message', this.onMQTTMessage.bind(this));
@@ -139,6 +154,13 @@ class Controller {
             }
         }
 
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClients()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
+
         // Call extensions
         await this.callExtensionMethod('onMQTTConnected', []);
     }
@@ -166,13 +188,13 @@ class Controller {
     }
 
     async onZigbeeEvent(type, data) {
-        const entity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !entity.settings) {
+        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            entity.settings = settings.addDevice(data.device.ieeeAddr);
+            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
         }
 
-        const name = entity && entity.settings ? entity.settings.friendlyName : null;
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
 
         if (type === 'message') {
             logger.debug(
@@ -182,34 +204,25 @@ class Controller {
             );
         } else if (type === 'deviceJoined') {
             logger.info(`Device '${name}' joined`);
-            this.mqtt.log('device_connected', {friendly_name: name});
         } else if (type === 'deviceInterview') {
             if (data.status === 'successful') {
                 logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
 
-                if (entity.mapped) {
-                    const {vendor, description, model} = entity.mapped;
-                    logger.info(
-                        `Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`,
-                    );
-
-                    const log = {friendly_name: name, model, vendor, description, supported: true};
-                    this.mqtt.log('pairing', 'interview_successful', log);
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
                         `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
-                    this.mqtt.log('pairing', 'interview_successful', {friendly_name: name, supported: false});
                 }
             } else if (data.status === 'failed') {
                 logger.error(`Failed to interview '${name}', device has not successfully been paired`);
-                this.mqtt.log('pairing', 'interview_failed', {friendly_name: name});
             } else {
                 /* istanbul ignore else */
                 if (data.status === 'started') {
                     logger.info(`Starting interview of '${name}'`);
-                    this.mqtt.log('pairing', 'interview_started', {friendly_name: name});
                 }
             }
         } else if (type === 'deviceAnnounce') {
@@ -218,15 +231,11 @@ class Controller {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
                 logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
-                this.mqtt.log('device_removed', 'left_network', {friendly_name: name || data.ieeeAddr});
             }
         }
 
         // Call extensions
-        this.callExtensionMethod(
-            'onZigbeeEvent',
-            [type, data, entity ? entity.mapped : null, entity ? entity.settings : null],
-        );
+        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
     }
 
     onMQTTMessage(payload) {
@@ -238,58 +247,79 @@ class Controller {
     }
 
     async publishEntityState(IDorName, payload, stateChangeReason=null) {
-        const entity = this.zigbee.resolveEntity(IDorName);
-        if (!entity || !entity.settings) {
+        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
             logger.error(`'${IDorName}' does not exist, skipping publish`);
             return;
         }
 
-        if (entity.type === 'device' && settings.get().advanced.last_seen !== 'disable' && entity.device.lastSeen) {
-            payload.last_seen = utils.formatDate(entity.device.lastSeen, settings.get().advanced.last_seen);
-        }
-
         let messagePayload = {...payload};
-        const currentState = this.state.exists(entity.settings.ID) ? this.state.get(entity.settings.ID) : {};
+        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
+            this.state.get(resolvedEntity.settings.ID) : {};
         const newState = objectAssignDeep.noMutate(currentState, payload);
 
         // Update state cache with new state.
-        this.state.set(entity.settings.ID, newState, stateChangeReason);
+        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
             messagePayload = newState;
         }
 
+        const deviceOptions = settings.get().device_options;
         const options = {
-            retain: entity.settings.hasOwnProperty('retain') ? entity.settings.retain : false,
-            qos: entity.settings.hasOwnProperty('qos') ? entity.settings.qos : 0,
+            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
+            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
         };
 
-        if (entity.type === 'device' && settings.get().mqtt.include_device_information) {
-            const device = this.zigbee.getDeviceByIeeeAddr(entity.device.ieeeAddr);
+        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
+        if (retention !== false) {
+            options.properties = {messageExpiryInterval: retention};
+        }
+
+        const isDevice = resolvedEntity.type === 'device';
+        if (isDevice && settings.get().mqtt.include_device_information) {
             const attributes = [
                 'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
                 'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
             ];
 
             messagePayload.device = {
-                friendlyName: entity.name,
-                model: entity.mapped ? entity.mapped.model : 'unknown',
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
             };
 
-            attributes.forEach((a) => messagePayload.device[a] = device[a]);
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+        }
+
+        // filter mqtt message attributes
+        if (deviceOptions.filtered_attributes) {
+            deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        }
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
         if (Object.entries(messagePayload).length) {
-            if (settings.get().experimental.output === 'json') {
-                await this.mqtt.publish(entity.name, JSON.stringify(messagePayload), options);
+            if (settings.get().experimental.output === 'attribute_and_json') {
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
+            } else if (settings.get().experimental.output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
             } else {
                 /* istanbul ignore else */
                 if (settings.get().experimental.output === 'attribute') {
-                    await this.iteratePayloadAttributeOutput(`${entity.name}/`, messagePayload, options);
+                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
                 }
             }
         }
+
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {

lib/eventBus.js

@@ -0,0 +1,24 @@
+const events = require('events');
+const assert = require('assert');
+
+const allowedEvents = [
+    'deviceRemoved', // Device has been removed
+    'deviceRenamed', // Device has been renamed
+    'groupRenamed', // Group has been renamed
+    'publishEntityState', // Entity state will be published
+    'stateChange', // Entity changes its state
+];
+
+class EventBus extends events.EventEmitter {
+    emit(event, data) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.emit(event, data);
+    }
+
+    on(event, callback) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.on(event, callback);
+    }
+}
+
+module.exports = EventBus;

lib/extension/availability.js

@@ -2,48 +2,44 @@ const logger = require('../util/logger');
 const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-// Some EndDevices should be pinged
+// Pingable end devices, some end devices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const forcedPingable = [
+const pingableEndDevices = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
 ];
 
-const toZigbeeCandidates = ['state', 'brightness', 'color', 'color_temp'];
-
 const Hours25 = 1000 * 60 * 60 * 25;
 
 /**
  * This extensions pings devices to check if they are online.
  */
-class DeviceAvailability extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
+class Availability extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
 
         this.availability_timeout = settings.get().advanced.availability_timeout;
         this.timers = {};
         this.state = {};
 
-        // Initialize blacklist
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
-
-        // Initialize whitelist
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
+        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
+        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
     }
 
-    isAllowed(device) {
+    inWhitelistOrNotInBlacklist(device) {
+        const ieeeAddr = device.ieeeAddr;
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings.friendlyName;
+
         // Whitelist is not empty and device is in it, enable availability
         if (this.whitelist.length > 0) {
-            return this.whitelist.includes(device.ieeeAddr);
+            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
         }
 
         // Device is on blacklist, disable availability
-        if (this.blacklist.includes(device.ieeeAddr)) {
+        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
             return false;
         }
 
@@ -51,27 +47,25 @@ class DeviceAvailability extends BaseExtension {
     }
 
     isPingable(device) {
-        // Device is on forcedPingable-list, enable availability
-        if (forcedPingable.find((d) => d.zigbeeModel.includes(device.modelID))) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
             return true;
         }
 
         // Device is a mains powered router
-        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
-
-        return result;
+        return utils.isRouter(device) && !utils.isBatteryPowered(device);
     }
 
     onMQTTConnected() {
         for (const device of this.zigbee.getClients()) {
             // Mark all devices as online on start
-            this.publishAvailability(device, true);
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
 
-            if (this.isAllowed(device)) {
+            if (this.inWhitelistOrNotInBlacklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
-                    this.timers[device.ieeeAddr] = setInterval(() => {
+                    this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
                     }, utils.secondsToMilliseconds(300));
                 }
@@ -81,34 +75,33 @@ class DeviceAvailability extends BaseExtension {
 
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity) {
             logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
             return;
         }
 
-        const ieeeAddr = device.ieeeAddr;
-        const level = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
         try {
             await device.ping();
             this.publishAvailability(device, true);
-            logger.debug(`Successfully pinged '${entity.name}'`);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
         } catch (error) {
             this.publishAvailability(device, false);
-            logger[level](`Failed to ping '${entity.name}'`);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
         } finally {
             this.setTimerPingable(device);
         }
     }
 
     async handleIntervalNotPingable(device) {
-        const ago = Date.now() - device.lastSeen;
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity || !device.lastSeen) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
             return;
         }
 
-        logger.debug(`Non-pingable device '${entity.name}' was last seen '${ago / 1000}' seconds ago.`);
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
 
         if (ago > Hours25) {
             this.publishAvailability(device, false);
@@ -134,50 +127,46 @@ class DeviceAvailability extends BaseExtension {
     }
 
     async onReconnect(device) {
-        if (device && device.modelID) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-
-            if (mappedDevice) {
-                const used = [];
-                try {
-                    for (const key of toZigbeeCandidates) {
-                        const converter = mappedDevice.toZigbee.find((tz) => tz.key.includes(key));
-                        if (converter && !used.includes(converter)) {
-                            await converter.convertGet(device.endpoints[0], key, {});
-                            used.push(converter);
-                        }
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            const used = [];
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter && !used.includes(converter)) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                        used.push(converter);
                     }
-                } catch (error) {
-                    const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-                    logger.error(`Failed to read state of '${entity.name}' after reconnect`);
                 }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
             }
         }
     }
 
-    publishAvailability(device, available) {
+    publishAvailability(device, available, force=false) {
         const ieeeAddr = device.ieeeAddr;
         if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
             this.onReconnect(device);
         }
 
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
+        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
         const topic = `${name}/availability`;
         const payload = available ? 'online' : 'offline';
-        if (this.state[ieeeAddr] !== available) {
+        if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;
             this.mqtt.publish(topic, payload, {retain: true, qos: 0});
         }
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
+    onZigbeeEvent(type, data, resolvedEntity) {
         const device = data.device;
         if (!device) {
             return;
         }
 
-        if (this.isAllowed(device)) {
+        if (this.inWhitelistOrNotInBlacklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {
@@ -203,4 +192,4 @@ class DeviceAvailability extends BaseExtension {
     }
 }
 
-module.exports = DeviceAvailability;
+module.exports = Availability;

lib/extension/bind.js

@@ -0,0 +1,121 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const assert = require('assert');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const Extension = require('./extension');
+
+const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+
+// See zigbee-herdsman-converters devices.js
+const defaultBindGroup = {type: 'group_number', ID: 901};
+
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        }
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        }
+
+        return {type, sourceKey, targetKey};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
+
+        // Find source; can only be a device and target
+        const source = this.zigbee.resolveEntity(sourceKey);
+        assert(source != null && source.type === 'device', 'Source undefined or not a device');
+        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
+        assert(target != null, 'Target is unknown');
+
+        const sourceName = source.settings.friendlyName;
+        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+        let attemptedToBindSomething = false;
+
+        // Find which clusters are supported by both the source and target.
+        // Groups are assumed to support all clusters.
+        for (const cluster of clusters) {
+            const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+
+            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                attemptedToBindSomething = true;
+                try {
+                    let bindTarget = null;
+                    if (target.type === 'group') bindTarget = target.group;
+                    else if (target.type === 'group_number') bindTarget = target.ID;
+                    else bindTarget = target.endpoint;
+
+                    if (type === 'bind') {
+                        await source.endpoint.bind(cluster, bindTarget);
+                    } else {
+                        await source.endpoint.unbind(cluster, bindTarget);
+                    }
+
+                    logger.info(
+                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                        `'${sourceName}' to '${targetName}'`,
+                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}`,
+                                message: {from: sourceName, to: targetName, cluster}}),
+                        );
+                    }
+                } catch (error) {
+                    logger.error(
+                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                        `'${targetName}' (${error})`,
+                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}_failed`,
+                                message: {from: sourceName, to: targetName, cluster}}),
+                        );
+                    }
+                }
+            }
+        }
+
+        if (!attemptedToBindSomething) {
+            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                );
+            }
+        }
+    }
+}
+
+module.exports = Bind;

lib/extension/bridge.js

@@ -0,0 +1,211 @@
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+
+        this.requestLookup = {
+            'permitjoin': this.permitJoin.bind(this),
+            // 'device/remove': this.deviceRemove.bind(this),
+            // 'device/forceremove': this.deviceForceRemove.bind(this),
+            // 'device/ban': this.deviceBan.bind(this),
+            // 'group/remove': this.groupRemove.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = JSON.parse(message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') payload = {friendlyName: resolvedEntity.settings.friendlyName, ieeeAddress};
+            else if (type === 'deviceInterview') {
+                payload = {friendlyName: resolvedEntity.settings.friendlyName, status: data.status, ieeeAddress};
+                if (data.status === 'successful') {
+                    payload.supported = !!resolvedEntity.definition;
+                    payload.definition = resolvedEntity.definition ? {
+                        model: resolvedEntity.definition.model,
+                        vendor: resolvedEntity.definition.vendor,
+                        description: resolvedEntity.definition.description,
+                        supports: resolvedEntity.definition.supports,
+                    } : null;
+                }
+            } else payload = {ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish('bridge/event', JSON.stringify({type, data: payload}), {retain: false, qos: 0});
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    // async deviceRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('remove', 'device', message);
+    // }
+
+    // async deviceForceRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('force_remove', 'device', message);
+    // }
+
+    // async deviceBan(message) {
+    //     return this.removeForceRemoveOrBanEntity('ban', 'device', message);
+    // }
+
+    // async groupRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('remove', 'group', message);
+    // }
+
+    async permitJoin(message) {
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.permitJoin(value);
+        await this.publishInfo();
+        return utils.getResponse(message, {value: value}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    // async removeForceRemoveOrBanEntity(action, entityType, message) {
+    //     const ID = typeof message === 'object' ? message.ID : message.trim();
+    //     const entity = this.zigbee.resolveEntity(ID);
+    //     if (!entity || entity.type !== entityType) {
+    //         throw new Error(`${ID} is not a ${entityType}`);
+    //     }
+
+    //     const lookup = {
+    //         ban: ['banned', 'Banning', 'ban'],
+    //         force_remove: ['force_removed', 'Force removing', 'force remove'],
+    //         remove: ['removed', 'Removing', 'remove'],
+    //     };
+
+    //     try {
+    //         logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+    //         if (entity.type === 'device') {
+    //             if (action === 'ban') {
+    //                 settings.banDevice(entity.settings.ID);
+    //             }
+
+    //             action === 'force_remove' ?
+    //                 await entity.device.removeFromDatabase() : await entity.device.removeFromNetwork();
+    //         } else {
+    //             await entity.group.removeFromDatabase();
+    //         }
+
+    //         // Fire event
+    //         if (entity.type === 'device') {
+    //             this.eventBus.emit('deviceRemoved', {device: entity.device});
+    //         }
+
+    //         // Remove from configuration.yaml
+    //         entity.type === 'device' ?
+    //             settings.removeDevice(entity.settings.ID) : settings.removeGroup(entity.settings.ID);
+
+    //         // Remove from state
+    //         this.state.remove(entity.settings.ID);
+
+    //         logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+    //         entity.type === 'device' ? this.publishDevices() : this.publishGroups();
+    //         return utils.getResponse(message, {ID}, null);
+    //     } catch (error) {
+    //         throw new Error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+    //     }
+    // }
+
+    async publishInfo() {
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            logLevel: logger.getLevel(),
+            permitJoin: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    async publishDevices(topic, message) {
+        const devices = this.zigbee.getClients().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const definitionPayload = definition ? {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                supports: definition.supports,
+            } : null;
+
+            return {
+                ieeeAddress: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+                supported: !!definition,
+                friendlyName: resolved.name,
+                definition: definitionPayload,
+                powerSource: device.powerSource,
+                softwareBuildID: device.softwareBuildID,
+                dateCode: device.dateCode,
+                interviewing: device.interviewing,
+                interviewCompleted: device.interviewCompleted,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+    }
+
+    async publishGroups(topic, message) {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                ID: group.groupID,
+                friendlyName: resolved.name,
+                members: group.members.map((m) => {
+                    return {
+                        ieeeAddress: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+    }
+}
+
+module.exports = Bridge;

lib/extension/configure.js

@@ -0,0 +1,120 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.configuring = new Set();
+        this.attempts = {};
+
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+    }
+
+    shouldConfigure(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            return false;
+        }
+
+        if (resolvedEntity.device.meta &&
+            resolvedEntity.device.meta.hasOwnProperty('configured') &&
+            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            return false;
+        }
+
+        if (resolvedEntity.device.interviewing === true) {
+            return false;
+        }
+
+        return true;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (topic !== this.legacyTopic) {
+                return;
+            }
+
+            const resolvedEntity = this.zigbee.resolveEntity(message);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                logger.error(`Device '${message}' does not exist`);
+                return;
+            }
+
+            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(resolvedEntity, true);
+        }
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldConfigure(resolvedEntity)) {
+                await this.configure(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+
+        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
+    }
+
+    async configure(resolvedEntity, force=false) {
+        const device = resolvedEntity.device;
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return false;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${resolvedEntity.name}'`);
+        try {
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            logger.info(`Successfully configured '${resolvedEntity.name}'`);
+            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.save();
+        } catch (error) {
+            logger.error(
+                `Failed to configure '${resolvedEntity.name}', ` +
+                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
+            );
+            this.attempts[device.ieeeAddr]++;
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/deviceBind.js

@@ -1,72 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-const BaseExtension = require('./baseExtension');
-
-const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
-
-class DeviceBind extends BaseExtension {
-    onMQTTConnected() {
-        for (let step = 1; step < 20; step++) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind${'/+'.repeat(step)}`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind${'/+'.repeat(step)}`);
-        }
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Parse topic, retrieve type (bind or unbind) and source
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-        const type = topic.split('/')[0];
-        const sourceKey = topic.replace(`${type}/`, '');
-        const targetKey = message;
-
-        // Find source; can only be a device and target
-        const source = this.zigbee.resolveEntity(sourceKey);
-        assert(source != null && source.type === 'device', 'Source undefined or not a device');
-        const target = this.zigbee.resolveEntity(targetKey);
-        assert(target != null, 'Target is unknown');
-
-        const sourceName = source.settings.friendlyName;
-        const targetName = target.settings.friendlyName;
-
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters.
-        for (const cluster of clusters) {
-            const targetValid = target.type === 'group' ||
-                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
-
-            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
-                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
-                try {
-                    const bindTarget = target.type === 'group' ? target.group : target.endpoint;
-                    if (type === 'bind') {
-                        await source.endpoint.bind(cluster, bindTarget);
-                    } else {
-                        await source.endpoint.unbind(cluster, bindTarget);
-                    }
-
-                    logger.info(
-                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceName}' to '${targetName}'`,
-                    );
-                    this.mqtt.log(
-                        `device_${type}`,
-                        {from: sourceName, to: targetName, cluster},
-                    );
-                } catch (error) {
-                    logger.error(
-                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
-                        `'${targetName}' (${error})`,
-                    );
-                }
-            }
-        }
-    }
-}
-
-module.exports = DeviceBind;

lib/extension/deviceConfigure.js

@@ -1,109 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
-
-class DeviceConfigure extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
-
-        this.configuring = new Set();
-        this.attempts = {};
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-    }
-
-    shouldConfigure(device, mappedDevice) {
-        if (!device) {
-            return false;
-        }
-
-        if (device.meta.hasOwnProperty('configured') && device.meta.configured === mappedDevice.meta.configureKey) {
-            return false;
-        }
-
-        if (!mappedDevice || !mappedDevice.configure) {
-            return false;
-        }
-
-        if (device.interviewing === true) {
-            return false;
-        }
-
-        return true;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (topic !== this.topic) {
-            return;
-        }
-
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || !entity.type === 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            return;
-        }
-
-        const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(entity.device.modelID);
-        this.configure(entity.device, mappedDevice, entity.settings, true);
-    }
-
-    async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
-
-        for (const device of this.zigbee.getClients()) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-            const settingsDevice = settings.getDevice(device.ieeeAddr);
-            if (this.shouldConfigure(device, mappedDevice)) {
-                await this.configure(device, mappedDevice, settingsDevice);
-            }
-        }
-    }
-
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        const device = data.device;
-
-        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(device, mappedDevice)) {
-            this.configure(device, mappedDevice, settingsDevice);
-        }
-    }
-
-    async configure(device, mappedDevice, settingsDevice, force=false) {
-        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
-            return false;
-        }
-
-        this.configuring.add(device.ieeeAddr);
-
-        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
-            this.attempts[device.ieeeAddr] = 0;
-        }
-
-        logger.info(`Configuring '${settingsDevice.friendlyName}'`);
-        try {
-            await mappedDevice.configure(device, this.coordinatorEndpoint);
-            logger.info(`Succesfully configured '${settingsDevice.friendlyName}'`);
-            // eslint-disable-next-line
-            device.meta.configured = mappedDevice.meta.configureKey;
-            device.save();
-        } catch (error) {
-            logger.error(
-                `Failed to configure '${settingsDevice.friendlyName}', ` +
-                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
-            );
-            this.attempts[device.ieeeAddr]++;
-        }
-
-        this.configuring.delete(device.ieeeAddr);
-    }
-}
-
-module.exports = DeviceConfigure;

lib/extension/deviceEvent.js

@@ -1,34 +0,0 @@
-const BaseExtension = require('./baseExtension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-
-class DeviceEvent extends BaseExtension {
-    async onZigbeeStarted() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'start', {});
-        }
-    }
-
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (data.device) {
-            this.callOnEvent(data.device, type, data, mappedDevice);
-        }
-    }
-
-    async stop() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'stop', {});
-        }
-    }
-
-    callOnEvent(device, type, data, mappedDevice) {
-        if (!mappedDevice) {
-            mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-        }
-
-        if (mappedDevice && mappedDevice.onEvent) {
-            mappedDevice.onEvent(type, data, device);
-        }
-    }
-}
-
-module.exports = DeviceEvent;

lib/extension/extension.js

@@ -1,4 +1,4 @@
-class BaseExtension {
+class Extension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -6,12 +6,14 @@ class BaseExtension {
      * @param {MQTT} mqtt MQTT controller
      * @param {State} state State controller
      * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
      */
-    constructor(zigbee, mqtt, state, publishEntityState) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.zigbee = zigbee;
         this.mqtt = mqtt;
         this.state = state;
         this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
     }
 
     /**
@@ -28,10 +30,10 @@ class BaseExtension {
      * Is called when a Zigbee message from a device is received.
      * @param {string} type Type of the message
      * @param {Object} data Data of the message
-     * @param {Object?} mappedDevice The mapped device
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
      * @param {Object?} settingsDevice Device settings
      */
-    // onZigbeeEvent(type, data, mappedDevice, settingsDevice) {}
+    // onZigbeeEvent(type, data, resolvedEntity) {}
 
     /**
      * Is called when a MQTT message is received
@@ -47,4 +49,4 @@ class BaseExtension {
     // stop() {}
 }
 
-module.exports = BaseExtension;
+module.exports = Extension;

lib/extension/groups.js

@@ -1,30 +1,36 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const postfixes = utils.getEndpointNames();
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
-const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
+class Groups extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
 
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}remove`);
-            this.mqtt.subscribe(`${topic}add`);
-            this.mqtt.subscribe(`${topic}remove_all`); // DEPRECATED
+            for (let step = 1; step < 20; step++) {
+                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
+                this.mqtt.subscribe(`${topic}remove`);
+                this.mqtt.subscribe(`${topic}add`);
+                this.mqtt.subscribe(`${topic}remove_all`);
+            }
         }
     }
 
     async onZigbeeStarted() {
         await this.syncGroupsWithSettings();
-        this.state.on('stateChange', this.onStateChange);
+        this.eventBus.on('stateChange', this.onStateChange);
     }
 
     async syncGroupsWithSettings() {
@@ -85,33 +91,33 @@ class Groups extends BaseExtension {
         });
 
         if (Object.keys(payload).length) {
-            const entity = this.zigbee.resolveEntity(data.ID);
+            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
                 return settingsGroup && settingsGroup.optimistic;
             });
 
-            if (entity.type === 'device') {
+            if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(entity.endpoint)) {
-                        if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }
                 }
             } else {
                 const groupIDsToPublish = new Set();
-                for (const member of entity.group.members) {
+                for (const member of resolvedEntity.group.members) {
                     await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
                     for (const zigbeeGroup of zigbeeGroups) {
                         if (zigbeeGroup.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                                 groupIDsToPublish.add(zigbeeGroup.groupID);
                             }
                         }
                     }
                 }
-                groupIDsToPublish.delete(entity.group.groupID);
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
                 for (const groupID of groupIDsToPublish) {
                     await this.publishEntityState(groupID, payload, reason);
                 }
@@ -119,7 +125,7 @@ class Groups extends BaseExtension {
         }
     }
 
-    allMembersOff(zigbeeGroup) {
+    areAllMembersOff(zigbeeGroup) {
         for (const member of zigbeeGroup.members) {
             const device = member.getDevice();
             if (this.state.exists(device.ieeeAddr)) {
@@ -132,61 +138,126 @@ class Groups extends BaseExtension {
         return true;
     }
 
-    async onMQTTMessage(topic, message) {
-        let type;
-        let group;
-        const topicMatch = topic.match(topicRegex);
-        if (topicMatch) {
-            group = this.zigbee.resolveEntity(topicMatch[1]);
-            type = topicMatch[2];
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
 
-            if (!group || group.type !== 'group') {
-                logger.error(`Group '${topicMatch[1]}' does not exist`);
-                return;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            const topicMatch = topic.match(legacyTopicRegex);
+            if (topicMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
+                type = topicMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else if (topic.match(legacyTopicRegexRemoveAll)) {
+                type = 'remove_all';
+            } else {
+                return {};
             }
-        } else if (topic.match(topicRegexRemoveAll)) {
-            type = 'remove_all';
-        } else {
-            return;
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return {};
+            }
+
+            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
         }
 
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || !entity.type === 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            return;
-        }
+        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
+            this.parseMQTTMessage(topic, message);
+        if (!type) return;
 
         const keys = [
-            `${entity.device.ieeeAddr}/${entity.endpoint.ID}`,
-            `${entity.name}/${entity.endpoint.ID}`,
+            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
         ];
 
-        if (entity.endpointName) {
-            keys.push(`${entity.device.ieeeAddr}/${entity.endpointName}`);
-            keys.push(`${entity.name}/${entity.endpointName}`);
+        const definition = resolvedEntityDevice.definition;
+        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
+        const endpointName = endpoints ?
+            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+
+        if (endpointName) {
+            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
         }
 
-        if (entity.isDefaultEndpoint) {
-            keys.push(entity.name);
-            keys.push(entity.device.ieeeAddr);
+        if (!hasEndpointName) {
+            keys.push(resolvedEntityDevice.name);
+            keys.push(resolvedEntityDevice.device.ieeeAddr);
         }
 
         if (type === 'add') {
-            logger.info(`Adding '${entity.name}' to '${group.name}'`);
-            await entity.endpoint.addToGroup(group.group);
-            settings.addDeviceToGroup(group.settings.ID, keys);
-            this.mqtt.log('device_group_add', {friendly_name: entity.name, group: group.name});
+            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_add`, message: payload}),
+                );
+            }
         } else if (type === 'remove') {
-            logger.info(`Removing '${entity.name}' from '${group.name}'`);
-            await entity.endpoint.removeFromGroup(group.group);
-            settings.removeDeviceFromGroup(group.settings.ID, keys);
-            this.mqtt.log('device_group_remove', {friendly_name: entity.name, group: group.name});
+            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_remove`, message: payload}),
+                );
+            }
         } else { // remove_all
-            logger.info(`Removing '${entity.name}' from all groups`);
-            await entity.endpoint.removeFromAllGroups();
+            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+            await resolvedEntityDevice.endpoint.removeFromAllGroups();
             for (const settingsGroup of settings.getGroups()) {
                 settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-                this.mqtt.log('device_group_remove_all', {friendly_name: entity.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: resolvedEntityDevice.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                    );
+                }
             }
         }
     }

lib/extension/legacy/bridgeLegacy.js

@@ -1,17 +1,17 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const utils = require('../util/utils');
+const utils = require('../../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('../extension');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
 const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
 
-class BridgeConfig extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
+class BridgeLegacy extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
 
         // Bind functions
         this.permitJoin = this.permitJoin.bind(this);
@@ -63,7 +63,10 @@ class BridgeConfig extends BaseExtension {
             assert(entity, `Entity '${message}' does not exist`);
             settings.whitelistDevice(entity.ID);
             logger.info(`Whitelisted '${entity.friendlyName}'`);
-            this.mqtt.log('device_whitelisted', {friendly_name: entity.friendlyName});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
         }
@@ -147,9 +150,11 @@ class BridgeConfig extends BaseExtension {
             };
 
             if (device.type !== 'Coordinator') {
-                const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
                 const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = mappedDevice ? mappedDevice.model : device.modelID;
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
                 payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
                 payload.manufacturerID = device.manufacturerID;
                 payload.manufacturerName = device.manufacturerName;
@@ -172,20 +177,23 @@ class BridgeConfig extends BaseExtension {
         if (topic.split('/').pop() == 'get') {
             this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
         } else {
-            this.mqtt.log('devices', devices);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
         }
     }
 
     groups(topic, message) {
-        this.mqtt.log('groups', settings.getGroups().map((g) => {
+        const payload = settings.getGroups().map((g) => {
             const group = {...g};
             delete group.friendlyName;
             return group;
-        }));
+        });
+
+        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
     }
 
     rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} got ${message}`;
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
 
         let json = null;
         try {
@@ -215,9 +223,17 @@ class BridgeConfig extends BaseExtension {
 
     _renameInternal(from, to) {
         try {
+            const isGroup = settings.getGroup(from) !== null;
             settings.changeFriendlyName(from, to);
             logger.info(`Successfully renamed - ${from} to ${to} `);
-            this.mqtt.log('device_renamed', {from, to});
+            const entity = this.zigbee.resolveEntity(to);
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
+
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
         } catch (error) {
             logger.error(`Failed to rename - ${from} to ${to}`);
         }
@@ -248,12 +264,17 @@ class BridgeConfig extends BaseExtension {
 
         const group = settings.addGroup(name, id);
         this.zigbee.createGroup(group.ID);
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
     removeGroup(topic, message) {
         const name = message;
-        settings.removeGroup(name);
+        const entity = this.zigbee.resolveEntity(message);
+        assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+        settings.removeGroup(message);
+        entity.group.removeFromDatabase();
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }
 
@@ -279,10 +300,15 @@ class BridgeConfig extends BaseExtension {
 
         if (!entity) {
             logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
+
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
             return;
         }
 
         const cleanup = () => {
+            // Fire event
+            this.eventBus.emit('deviceRemoved', {device: entity.device});
+
             // Remove from configuration.yaml
             settings.removeDevice(entity.settings.ID);
 
@@ -290,7 +316,7 @@ class BridgeConfig extends BaseExtension {
             this.state.remove(entity.settings.ID);
 
             logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-            this.mqtt.log(`device_${lookup[action][0]}`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
@@ -306,6 +332,8 @@ class BridgeConfig extends BaseExtension {
             logger.error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
+
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
         }
 
         if (action === 'ban') {
@@ -350,22 +378,86 @@ class BridgeConfig extends BaseExtension {
         await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
         if (type === 'deviceJoined') {
-            this.lastJoinedDeviceName = settingsDevice.friendlyName;
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
         }
     }
 
     async touchlinkFactoryReset() {
         logger.info('Starting touchlink factory reset...');
+        this.mqtt.publish(
+            'bridge/log',
+            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
         const result = await this.zigbee.touchlinkFactoryReset();
 
         if (result) {
-            logger.info('Succesfully factory reset device through Touchlink');
+            logger.info('Successfully factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
         } else {
             logger.warn('Failed to factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
         }
     }
 }
 
-module.exports = BridgeConfig;
+module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -1,13 +1,16 @@
 /* istanbul ignore file */
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
 
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
-class DeviceGroupMembership extends BaseExtension {
+class DeviceGroupMembership extends Extension {
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/device/+/get_group_membership`);
+        for (let step = 1; step < 20; step++) {
+            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
+            this.mqtt.subscribe(topic);
+        }
     }
 
     async onMQTTMessage(topic, message) {

lib/extension/legacy/softReset.js

@@ -1,16 +1,16 @@
 /* istanbul ignore file */
 // DEPRECATED
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
 
 /**
  * This extensions soft resets the ZNP after a certain timeout.
  */
-class SoftReset extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
+class SoftReset extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;
         this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
     }

lib/extension/networkMap.js

@@ -1,16 +1,18 @@
 const settings = require('../util/settings');
 const utils = require('../util/utils');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class NetworkMap extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
 
-        // Subscribe to topic.
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.topicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
 
         // Bind
         this.raw = this.raw.bind(this);
@@ -24,16 +26,23 @@ class NetworkMap extends BaseExtension {
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-        this.mqtt.subscribe(this.topicRoutes);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+            this.mqtt.subscribe(this.legacyTopicRoutes);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        if ((topic === this.topic || topic === this.topicRoutes) && this.supportedFormats.hasOwnProperty(message)) {
-            const includeRoutes = topic === this.topicRoutes;
-            const topology = await this.networkScan(includeRoutes);
-            const converted = this.supportedFormats[message](topology);
-            this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(message)) {
+                const includeRoutes = topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                const converted = this.supportedFormats[message](topology);
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            }
         }
     }
 
@@ -47,32 +56,32 @@ class NetworkMap extends BaseExtension {
         let text = 'digraph G {\nnode[shape=record];\n';
         let style = '';
 
-        topology.nodes.forEach((device) => {
+        topology.nodes.forEach((node) => {
             const labels = [];
 
             // Add friendly name
-            labels.push(`${device.friendlyName}`);
+            labels.push(`${node.friendlyName}`);
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${device.ieeeAddr} (${device.networkAddress})` +
-                ((device.failed && device.failed.length) ? `failed: ${device.failed.join(',')}` : ''),
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
-            if (device.type !== 'Coordinator') {
-                const mappedModel = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                if (mappedModel) {
-                    labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
+            if (node.type !== 'Coordinator') {
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
-                    labels.push(`${device.manufacturerName} ${device.modelID}`);
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
                 }
             }
 
             // Add the device last_seen timestamp
             let lastSeen = 'unknown';
-            const date = device.type === 'Coordinator' ? Date.now() : device.lastSeen;
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
                 lastSeen = utils.formatDate(date, 'ISO_8601_local');
             }
@@ -80,10 +89,10 @@ class NetworkMap extends BaseExtension {
             labels.push(lastSeen);
 
             // Shape the record according to device type
-            if (device.type == 'Coordinator') {
+            if (node.type == 'Coordinator') {
                 style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
                     `fontcolor="${colors.font.coordinator}"`;
-            } else if (device.type == 'Router') {
+            } else if (node.type == 'Router') {
                 style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
                     `fontcolor="${colors.font.router}"`;
             } else {
@@ -92,22 +101,22 @@ class NetworkMap extends BaseExtension {
             }
 
             // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
 
             /**
              * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.links.filter((e) => (e.source.ieeeAddr === device.ieeeAddr)).forEach((e) => {
-                const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, ' :
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
                 const textRoutes = e.routes.map((r) => r.destinationAddress);
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
-                text += `  "${device.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
                 text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
             });
         });
@@ -119,31 +128,31 @@ class NetworkMap extends BaseExtension {
 
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
-        const devices = this.zigbee.getDevices();
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
         const lqis = new Map();
         const routingTables = new Map();
         const failed = new Map();
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             try {
                 const result = await device.lqi();
                 lqis.set(device, result);
-                logger.debug(`LQI succeeded for '${resolved.name}'`);
+                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
                 failed.get(device).push('lqi');
-                logger.error(`Failed to execute LQI for '${resolved.name}'`);
+                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
             }
 
             if (includeRoutes) {
                 try {
                     const result = await device.routingTable();
                     routingTables.set(device, result);
-                    logger.debug(`Routing table succeeded for '${resolved.name}'`);
+                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {
                     failed.get(device).push('routingTable');
-                    logger.error(`Failed to execute routing table for '${resolved.name}'`);
+                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
                 }
             }
         }
@@ -153,9 +162,9 @@ class NetworkMap extends BaseExtension {
         const networkMap = {nodes: [], links: []};
         // Add nodes
         for (const device of devices) {
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             networkMap.nodes.push({
-                ieeeAddr: device.ieeeAddr, friendlyName: resolved.name, type: device.type,
+                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
             });

lib/extension/onEvent.js

@@ -0,0 +1,37 @@
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends Extension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && resolvedEntity.type === 'device') {
+            this.callOnEvent(resolvedEntity, type, data);
+        }
+    }
+
+    async stop() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'stop', {});
+        }
+    }
+
+    callOnEvent(resolvedEntity, type, data) {
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+
+        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.js

@@ -0,0 +1,220 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const assert = require('assert');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const Extension = require('./extension');
+const MINUTES_10 = 1000 * 60 * 10;
+
+class OTAUpdate extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.inProgress = new Set();
+        this.lastChecked = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+
+        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
+        if (supportsOTA) {
+            // When a device does a next image request, it will usually do it a few times after each other
+            // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
+            // each time.
+            const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > MINUTES_10 : true;
+            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+
+            this.lastChecked[data.device.ieeeAddr] = Date.now();
+            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
+            this.publishEntityState(data.device.ieeeAddr, {update_available: available});
+
+            if (available) {
+                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        }
+
+        // Respond to the OTA request:
+        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
+        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
+    }
+
+    async readSoftwareBuildIDAndDateCode(device, update) {
+        try {
+            const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
+
+            if (update) {
+                device.softwareBuildID = result.swBuildId;
+                device.dateCode = result.dateCode;
+                device.save();
+            }
+
+            return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
+        } catch (e) {
+            return null;
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        let resolvedEntity = null;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (!topic.match(legacyTopicRegex)) {
+                return null;
+            }
+
+            resolvedEntity = this.zigbee.resolveEntity(message);
+        } else {
+            return null;
+        }
+
+        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
+        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
+            logger.error(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            return;
+        }
+
+        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
+            return;
+        }
+        this.inProgress.add(resolvedEntity.device.ieeeAddr);
+
+        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
+        if (type === 'check') {
+            const message = `Checking if update available for '${resolvedEntity.name}'`;
+            logger.info(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            try {
+                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
+                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
+                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+            } catch (error) {
+                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
+                logger.error(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `check_failed`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        } else { // type === 'update'
+            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
+            logger.info(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            try {
+                const onProgress = (progress, remaining) => {
+                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                    if (remaining) {
+                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                    }
+
+                    logger.info(message);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    }
+                };
+
+                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
+                const message = `Finished update of '${resolvedEntity.name}'` +
+                    (to ? `, from '${fromS}' to '${toS}'` : ``);
+                logger.info(message);
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
+            } catch (error) {
+                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
+                logger.error(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_failed`, device: resolvedEntity.name};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
+            }
+        }
+
+        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+    }
+}
+
+module.exports = OTAUpdate;

lib/extension/publish.js

@@ -4,10 +4,9 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
 
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
@@ -17,9 +16,12 @@ const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
     zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
 ];
 
-class EntityPublish extends BaseExtension {
+class EntityPublish extends Extension {
     onMQTTConnected() {
         // Subscribe to topics.
         const baseTopic = settings.get().mqtt.base_topic;
@@ -44,7 +46,7 @@ class EntityPublish extends BaseExtension {
             return null;
         }
 
-        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
+        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
     }
 
     async onMQTTMessage(topic, message) {
@@ -53,12 +55,20 @@ class EntityPublish extends BaseExtension {
             return false;
         }
 
-        const entityName = `${topic.ID}` + (topic.postfix ? `/${topic.postfix}` : '');
-        const entity = this.zigbee.resolveEntity(entityName);
+        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
+        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
 
-        if (!entity) {
-            this.mqtt.log('entity_not_found', {friendly_name: entityName});
-            logger.error(`Entity '${entityName}' is unknown`);
+        if (!resolvedEntity) {
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {friendly_name: entityKey};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `entity_not_found`, message}),
+                );
+            }
+
+            logger.error(`Entity '${entityKey}' is unknown`);
             return;
         }
 
@@ -67,27 +77,26 @@ class EntityPublish extends BaseExtension {
         let target = null;
         let options = {};
         let device = null;
-        let mapped = null;
+        let definition = null;
 
-        assert(entity.type === 'device' || entity.type === 'group');
-        if (entity.type === 'device') {
-            // Map device to a model
-            if (!entity.mapped) {
-                logger.warn(`Device with modelID '${entity.device.modelID}' is not supported.`);
+        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
+        if (resolvedEntity.type === 'device') {
+            if (!resolvedEntity.definition) {
+                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
                 return;
             }
 
-            device = entity.device;
-            mapped = entity.mapped;
-            target = entity.endpoint;
-            converters = entity.mapped.toZigbee;
-            options = entity.mapped.meta && entity.mapped.meta.options ? entity.mapped.meta.options : {};
-            options = {...options, ...entity.settings};
+            device = resolvedEntity.device;
+            definition = resolvedEntity.definition;
+            target = resolvedEntity.endpoint;
+            converters = resolvedEntity.definition.toZigbee;
+            options = resolvedEntity.settings;
         } else {
             converters = groupConverters;
-            target = entity.group;
-            options = entity.settings;
+            target = resolvedEntity.group;
+            options = resolvedEntity.settings;
+            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
         }
 
         // Convert the MQTT message to a Zigbee message.
@@ -103,12 +112,13 @@ class EntityPublish extends BaseExtension {
             }
         }
 
+
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(entity.settings.ID) || {};
+        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
@@ -120,31 +130,47 @@ class EntityPublish extends BaseExtension {
             }
         }
 
-        // Ensure that state and brightness are executed before other commands.
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
         const entries = Object.entries(json);
-        entries.sort((a, b) => (['state', 'brightness'].includes(a[0]) ? -1 : 1));
+        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
 
         // For each attribute call the corresponding converter
-        const usedConverters = [];
+        const usedConverters = {};
         for (let [key, value] of entries) {
-            let postfix = topic.postfix;
+            let endpointName = topic.endpointName;
             let actualTarget = target;
 
-            // When the key has a postfix included (e.g. state_right), this will override the target.
-            if (entity.type === 'device' && key.includes('_')) {
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            if (resolvedEntity.type === 'device' && key.includes('_')) {
                 const underscoreIndex = key.lastIndexOf('_');
-                const possiblePostfix = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getPostfixes().includes(possiblePostfix)) {
-                    postfix = possiblePostfix;
+                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
+                if (utils.getEndpointNames().includes(possibleEndpointName)) {
+                    endpointName = possibleEndpointName;
                     key = key.substring(0, underscoreIndex);
                     const device = target.getDevice();
-                    actualTarget = device.getEndpoint(mapped.endpoint(device)[postfix]);
+                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+
+                    if (!actualTarget) {
+                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
+                        continue;
+                    }
                 }
             }
 
+            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters.includes(converter)) {
+            if (usedConverters[endpointOrGroupID].includes(converter)) {
                 // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
@@ -156,28 +182,32 @@ class EntityPublish extends BaseExtension {
 
             // Converter didn't return a result, skip
             const meta = {
-                endpoint_name: postfix,
+                endpoint_name: endpointName,
                 options,
                 message: json,
                 logger,
                 device,
                 state: deviceState,
-                mapped,
+                mapped: definition,
             };
 
             try {
-                if (topic.type === 'set' && converter && converter.convertSet) {
-                    logger.debug(`Publishing '${topic.type}' '${key}' to '${entity.name}'`);
+                if (topic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
                     if (result && result.state) {
                         const msg = result.state;
 
-                        if (postfix) {
-                            msg[`state_${postfix}`] = msg.state;
-                            delete msg.state;
+                        if (endpointName) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                                if (msg.hasOwnProperty(key)) {
+                                    msg[`${key}_${endpointName}`] = msg[key];
+                                    delete msg[key];
+                                }
+                            }
                         }
 
-                        this.publishEntityState(entity.settings.ID, msg);
+                        this.publishEntityState(resolvedEntity.settings.ID, msg);
                     }
 
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
@@ -188,13 +218,13 @@ class EntityPublish extends BaseExtension {
                     // ever issue a read here, as we assume the device will properly report changes.
                     // Only do this when the retrieve_state option is enabled for this device.
                     if (
-                        entity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
-                        entity.settings.retrieve_state
+                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        resolvedEntity.settings.retrieve_state
                     ) {
                         setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
                     }
-                } else if (topic.type === 'get' && converter && converter.convertGet) {
-                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${entity.name}'`);
+                } else if (topic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     await converter.convertGet(actualTarget, key, meta);
                 } else {
                     logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
@@ -202,13 +232,21 @@ class EntityPublish extends BaseExtension {
                 }
             } catch (error) {
                 const message =
-                    `Publish '${topic.type}' '${key}' to '${entity.name}' failed: '${error}'`;
+                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
                 logger.error(message);
                 logger.debug(error.stack);
-                this.mqtt.log('zigbee_publish_error', message, {friendly_name: entity.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                    );
+                }
             }
 
-            usedConverters.push(converter);
+            usedConverters[endpointOrGroupID].push(converter);
         }
 
         return true;

lib/extension/receive.js

@@ -2,26 +2,40 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class DeviceReceive extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
-        this.coordinator = null;
+class Receive extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.elapsed = {};
         this.debouncers = {};
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
     }
 
     async onZigbeeStarted() {
         this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
     }
 
+    async onPublishEntityState(data) {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+            }
+        }
+    }
+
     publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
         if (!this.debouncers[ieeeAddr]) {
             this.debouncers[ieeeAddr] = {
                 payload: {},
                 publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload);
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
                     this.debouncers[ieeeAddr].payload = {};
                 }, time * 1000),
             };
@@ -54,8 +68,8 @@ class DeviceReceive extends BaseExtension {
         return result;
     }
 
-    canHandleEvent(type, data, mappedDevice) {
-        if (type !== 'message') {
+    shouldProcess(type, data, resolvedEntity) {
+        if (type !== 'message' || !resolvedEntity) {
             return false;
         }
 
@@ -77,18 +91,18 @@ class DeviceReceive extends BaseExtension {
          * As the same message is also received directly from the end device, it makes no sense
          * to handle these messages.
          */
-        const hasGroupID = data.hasOwnProperty('groupID') && data.groupID != 0;
+        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
         if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
             logger.debug('Skipping re-transmitted Xiaomi message');
             return false;
         }
 
-        if (data.device.modelID === null && data.device.interviewing) {
+        if (!data.device.modelID && data.device.interviewing) {
             logger.debug(`Skipping message, modelID is undefined and still interviewing`);
             return false;
         }
 
-        if (!mappedDevice) {
+        if (!resolvedEntity.definition) {
             logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
             logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
             return false;
@@ -97,27 +111,28 @@ class DeviceReceive extends BaseExtension {
         return true;
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (!this.canHandleEvent(type, data, mappedDevice)) {
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
 
-        const converters = mappedDevice.fromZigbee.filter((c) => {
+        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
             const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
             return c.cluster === data.cluster && type;
         });
 
-        // Check if there is an available converter
-        if (!converters.length) {
-            // Don't log readRsp and raw as warn messages, they are not interesting most of the time.
-            const level = ['readResponse', 'raw'].includes(data.type) ? 'debug' : 'warn';
-            logger[level](
-                `No converter available for '${mappedDevice.model}' with cluster '${data.cluster}' ` +
+        // Check if there is an available converter, genOta messages are not interesting.
+        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
+            logger.debug(
+                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
                 `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
             );
             return;
         }
 
+        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
+        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
+
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
         // - If a payload is returned publish it to the MQTT broker
@@ -139,37 +154,18 @@ class DeviceReceive extends BaseExtension {
             }
 
             // Check if we have to debounce
-            if (settingsDevice && settingsDevice.hasOwnProperty('debounce')) {
-                this.publishDebounce(
-                    data.device.ieeeAddr,
-                    payload,
-                    settingsDevice.debounce,
-                    settingsDevice.debounce_ignore,
-                );
+            if (debounce) {
+                this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
-
-                if (settings.get().homeassistant) {
-                    /**
-                     * Publish an empty value for click and action payload, in this way Home Assistant
-                     * can use Home Assistant entities in automations.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-                     */
-                    Object.keys(payload).forEach((key) => {
-                        if (['action', 'click'].includes(key)) {
-                            const counterPayload = {};
-                            counterPayload[key] = '';
-                            this.publishEntityState(data.device.ieeeAddr, counterPayload);
-                        }
-                    });
-                }
             }
         };
 
+        const meta = {device: data.device};
         let payload = {};
         converters.forEach((converter) => {
             const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
-            const converted = converter.convert(mappedDevice, data, publish, options);
+            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
             if (converted) {
                 payload = {...payload, ...converted};
             }
@@ -181,4 +177,4 @@ class DeviceReceive extends BaseExtension {
     }
 }
 
-module.exports = DeviceReceive;
+module.exports = Receive;

lib/extension/report.js

@@ -1,8 +1,8 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const CC2530Router = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
+const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
 const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 const debounce = require('debounce');
 const ZigbeeHerdsman = require('zigbee-herdsman');
 
@@ -10,8 +10,27 @@ const defaultConfiguration = {
     minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
 };
 
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+];
+
 const reportKey = 1;
 
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
 const clusters = {
     'genOnOff': [
         {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
@@ -20,9 +39,18 @@ const clusters = {
         {attribute: 'currentLevel', ...defaultConfiguration},
     ],
     'lightingColorCtrl': [
-        {attribute: 'colorTemperature', ...defaultConfiguration},
-        {attribute: 'currentX', ...defaultConfiguration},
-        {attribute: 'currentY', ...defaultConfiguration},
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
     ],
     'closuresWindowCovering': [
         {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
@@ -54,29 +82,66 @@ const pollOnMessage = [
         // When the bound devices/members of group have the following manufacturerID
         manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
     },
+    {
+        key: 2,
+        cluster: {
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+    },
 ];
 
-class DeviceReport extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        super(zigbee, mqtt, state, publishEntityState);
-        this.configuring = new Set();
+class Report extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.queue = new Set();
         this.failed = new Set();
         this.pollDebouncers = {};
     }
 
-    async setupReporting(device) {
-        if (this.configuring.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
-        this.configuring.add(device.ieeeAddr);
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+            // Device announces it but doesn't support it
+            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
+            return true;
+        }
+
+        return false;
+    }
+
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
+
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
 
         try {
-            for (const endpoint of device.endpoints) {
+            for (const ep of device.endpoints) {
                 for (const [cluster, configuration] of Object.entries(clusters)) {
-                    if (endpoint.supportsInputCluster(cluster)) {
-                        logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${endpoint.ID} - ${cluster}`);
-                        await endpoint.bind(cluster, this.coordinatorEndpoint);
-                        await endpoint.configureReporting(cluster, configuration);
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
+                        logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                items.push({...entry});
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
+                        await ep.bind(cluster, this.coordinatorEndpoint);
+                        await ep.configureReporting(cluster, items);
                         logger.info(
-                            `Succesfully setup reporting for '${device.ieeeAddr}' - ${endpoint.ID} - ${cluster}`,
+                            `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
                         );
                     }
                 }
@@ -93,12 +158,14 @@ class DeviceReport extends BaseExtension {
         }
 
         device.save();
-        this.configuring.delete(device.ieeeAddr);
+        this.queue.delete(device.ieeeAddr);
     }
 
-    shouldSetupReporting(mappedDevice, device, messageType) {
-        if (!device || !mappedDevice) return false;
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
 
+        const {device, definition} = resolvedEntity;
         // Handle messages of type endDeviceAnnce and devIncoming.
         // This message is typically send when a device comes online after being powered off
         // Ikea TRADFRI tend to forget their reporting after powered off.
@@ -109,7 +176,7 @@ class DeviceReport extends BaseExtension {
         if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
         if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
         // Gledopto devices don't support reporting.
-        if (mappedDevice === CC2530Router || mappedDevice.vendor === 'Gledopto') return false;
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
         return true;
     }
 
@@ -117,16 +184,16 @@ class DeviceReport extends BaseExtension {
         this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
 
         for (const device of this.zigbee.getClients()) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-            if (this.shouldSetupReporting(mappedDevice, device, null)) {
-                this.setupReporting(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                this.setupReporting(resolvedEntity);
             }
         }
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (this.shouldSetupReporting(mappedDevice, data.device, type)) {
-            this.setupReporting(data.device);
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            this.setupReporting(resolvedEntity);
         }
 
         if (type === 'message') {
@@ -152,7 +219,8 @@ class DeviceReport extends BaseExtension {
             let toPoll = [];
 
             // Add bound devices
-            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) => e.binds.map((e) => e))));
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
             toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
             toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
             toPoll = toPoll.map((e) => e.target);
@@ -185,4 +253,4 @@ class DeviceReport extends BaseExtension {
     }
 }
 
-module.exports = DeviceReport;
+module.exports = Report;

lib/mqtt.js

@@ -22,6 +22,15 @@ class MQTT extends events.EventEmitter {
             },
         };
 
+        if (mqttSettings.version) {
+            options.protocolVersion = mqttSettings.version;
+        }
+
+        if (mqttSettings.keepalive) {
+            logger.debug(`Using MQTT keepalive: ${mqttSettings.keepalive}`);
+            options.keepalive = mqttSettings.keepalive;
+        }
+
         if (mqttSettings.ca) {
             logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
             options.ca = fs.readFileSync(mqttSettings.ca);
@@ -86,11 +95,15 @@ class MQTT extends events.EventEmitter {
         this.emit('message', {topic, message: message.toString()});
     }
 
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
+
     async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
-        if (!this.client || this.client.reconnecting) {
+        if (!this.isConnected()) {
             logger.error(`Not connected to MQTT server!`);
             logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
             return;
@@ -102,16 +115,6 @@ class MQTT extends events.EventEmitter {
             this.client.publish(topic, payload, options, () => resolve());
         });
     }
-
-    log(type, message, meta=null) {
-        const payload = {type, message};
-
-        if (meta) {
-            payload.meta = meta;
-        }
-
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
-    }
 }
 
 module.exports = MQTT;

lib/state.js

@@ -2,23 +2,21 @@ const logger = require('./util/logger');
 const data = require('./util/data');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
-const events = require('events');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'action_color_temperature', 'action_color',
-    'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
     'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
-class State extends events.EventEmitter {
-    constructor() {
-        super();
+class State {
+    constructor(eventBus) {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
+        this.eventBus = eventBus;
     }
 
     start() {
@@ -70,17 +68,18 @@ class State extends events.EventEmitter {
 
     set(ID, state, reason=null) {
         const toState = objectAssignDeep.noMutate(state);
-        dontCacheProperties.forEach((property) => {
-            if (toState.hasOwnProperty(property)) {
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
                 delete toState[property];
             }
-        });
+        }
 
         const fromState = this.state[ID];
 
         this.state[ID] = toState;
 
-        this.emit('stateChange', {ID, from: fromState, to: toState, reason});
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
     }
 
     remove(ID) {

lib/util/logger.js

@@ -6,6 +6,7 @@ const fs = require('fs');
 const fx = require('mkdir-recursive');
 const rimraf = require('rimraf');
 const colorizer = winston.format.colorize();
+const assert = require('assert');
 
 // What transports to enable
 const output = settings.get().advanced.log_output;
@@ -13,6 +14,7 @@ const output = settings.get().advanced.log_output;
 // Directory to log to
 const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
 const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
 
 // Make sure that log directoy exsists when not logging to stdout only
 if (output.includes('file')) {
@@ -21,6 +23,8 @@ if (output.includes('file')) {
 
 // Determine the log level.
 const level = settings.get().advanced.log_level;
+const validLevels = ['info', 'error', 'warn', 'debug'];
+assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
 
 const levelWithCompensatedLength = {
     'info': 'info ',
@@ -40,7 +44,8 @@ const transportsToUse = [
         format: winston.format.combine(
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
+                let {timestamp, level, message} = info;
+                level = level === 'warning' ? 'warn' : level;
                 const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
@@ -50,25 +55,39 @@ const transportsToUse = [
 
 // Add file logger when enabled
 // NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            let {timestamp, level, message} = info;
+            level = level === 'warning' ? 'warn' : level;
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
 if (output.includes('file')) {
-    transportsToUse.push(new winston.transports.File({
-        filename: path.join(directory, 'log.txt'),
-        json: false,
-        level,
-        maxFiles: 3, // Keep last 3 files
-        maxsize: 10000000, // 10MB
-        format: winston.format.combine(
-            winston.format.timestamp({format: timestampFormat}),
-            winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
-                return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
-            }),
-        ),
-    }));
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
+
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    transportsToUse.push(new winston.transports.Syslog(options));
 }
 
 // Create logger
-const logger = winston.createLogger({transports: transportsToUse});
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
 
 // Cleanup any old log directory.
 function cleanup() {
@@ -98,13 +117,17 @@ logger.setLevel = (level) => {
 // Print to user what logging is enabled
 if (output.includes('file')) {
     if (output.includes('console')) {
-        logger.info(`Logging to console and directory: '${directory}'`);
+        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
     } else {
-        logger.info(`Logging to directory: '${directory}'`);
+        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
     }
     logger.cleanup();
 } else if (output.includes('console')) {
     logger.info(`Logging to console only'`);
 }
 
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
+
 module.exports = logger;

lib/util/settings.js

@@ -1,6 +1,6 @@
 const data = require('./data');
 const utils = require('./utils');
-const file = data.joinPath('configuration.yaml');
+const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
 const objectAssignDeep = require(`object-assign-deep`);
 const path = require('path');
 const yaml = require('./yaml');
@@ -39,19 +39,25 @@ const defaults = {
         },
     },
     experimental: {
-        // json or attribute
+        // json or attribute or attribute_and_json
         output: 'json',
+        new_api: false,
     },
     advanced: {
+        legacy_api: true,
+        log_rotation: true,
         log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
+        log_file: 'log.txt',
         log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
         soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         baudrate: 115200,
         rtscts: true,
+        adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
@@ -103,6 +109,13 @@ const defaults = {
          */
         homeassistant_status_topic: 'hass/status',
 
+        /**
+         * Home Assistant legacy triggers, when enabled:
+         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
+         * - A 'sensor_action' and 'sensor_click' will be discoverd
+         */
+        homeassistant_legacy_triggers: true,
+
         /**
          * Configurable timestampFormat
          * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
@@ -114,6 +127,7 @@ const defaults = {
 const schema = {
     type: 'object',
     properties: {
+        device_options: {type: 'object'},
         homeassistant: {type: 'boolean'},
         permit_join: {type: 'boolean'},
         mqtt: {
@@ -121,6 +135,7 @@ const schema = {
             properties: {
                 base_topic: {type: 'string'},
                 server: {type: 'string'},
+                keepalive: {type: 'number'},
                 ca: {type: 'string'},
                 key: {type: 'string'},
                 cert: {type: 'string'},
@@ -129,6 +144,7 @@ const schema = {
                 client_id: {type: 'string'},
                 reject_unauthorized: {type: 'boolean'},
                 include_device_information: {type: 'boolean'},
+                version: {type: 'number'},
             },
             required: ['base_topic', 'server'],
         },
@@ -137,6 +153,7 @@ const schema = {
             properties: {
                 port: {type: ['string', 'null']},
                 disable_led: {type: 'boolean'},
+                adapter: {type: 'string', enum: ['deconz', 'zstack']},
             },
         },
         ban: {type: 'array', items: {type: 'string'}},
@@ -150,17 +167,20 @@ const schema = {
         advanced: {
             type: 'object',
             properties: {
+                legacy_api: {type: 'boolean'},
                 pan_id: {type: 'number'},
                 ext_pan_id: {type: 'array', items: {type: 'number'}},
                 channel: {type: 'number', minimum: 11, maximum: 26},
                 cache_state: {type: 'boolean'},
+                log_rotation: {type: 'boolean'},
                 log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
                 log_output: {type: 'array', items: {type: 'string'}},
                 log_directory: {type: 'string'},
+                log_file: {type: 'string'},
                 baudrate: {type: 'number'},
                 rtscts: {type: 'boolean'},
                 soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: 'array', items: {type: 'number'}},
+                network_key: {type: ['array', 'string'], items: {type: 'number'}},
                 last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
                 elapsed: {type: 'boolean'},
                 availability_timeout: {type: 'number', minimum: 0},
@@ -170,6 +190,7 @@ const schema = {
                 homeassistant_discovery_topic: {type: 'string'},
                 homeassistant_status_topic: {type: 'string'},
                 timestamp_format: {type: 'string'},
+                adapter_concurrent: {type: 'number'},
             },
         },
         map_options: {
@@ -221,7 +242,9 @@ const schema = {
                     properties: {
                         friendly_name: {type: 'string'},
                         retain: {type: 'boolean'},
+                        retention: {type: 'number'},
                         qos: {type: 'number'},
+                        filtered_attributes: {type: 'array', items: {type: 'string'}},
                     },
                     required: ['friendly_name'],
                 },
@@ -241,6 +264,7 @@ const schema = {
                         devices: {type: 'array', items: {type: 'string'}},
                         optimistic: {type: 'boolean'},
                         qos: {type: 'number'},
+                        filtered_attributes: {type: 'array', items: {type: 'string'}},
                     },
                     required: ['friendly_name'],
                 },
@@ -260,13 +284,16 @@ function write() {
 
     // Read settings to check if we have to split devices/groups into separate file.
     const actual = yaml.read(file);
-    if (actual.mqtt && actual.mqtt.password && actual.mqtt.user) {
-        toWrite.mqtt.user = actual.mqtt.user;
-        toWrite.mqtt.password = actual.mqtt.password;
-    }
 
-    if (actual.advanced && actual.advanced.network_key) {
-        toWrite.advanced.network_key = actual.advanced.network_key;
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
     }
 
     if (typeof actual.devices === 'string') {
@@ -288,17 +315,47 @@ function write() {
 function validate() {
     const validate = ajv.compile(schema);
     const valid = validate(_settings);
-    const postfixes = utils.getPostfixes();
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    const postfixes = utils.getEndpointNames();
 
     // Verify that all friendly names are unique
     const names = [];
     const check = (name) => {
         if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
         if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
+        if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+        if (name.includes('#') || name.includes('+')) {
+            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+        }
         names.push(name);
     };
-    Object.values(_settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
-    Object.values(_settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    const settingsWithDefaults = getWithDefaults();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
+            if (device.retention) {
+                throw new Error('MQTT retention requires protocol version 5');
+            }
+        }
+    }
+
+    const checkAvailabilityList = (list, type) => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
 
     return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
 }
@@ -563,6 +620,9 @@ function removeDeviceFromGroup(groupIDorName, keys) {
     const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
     const settings = get();
     const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
 
     const key = groupHasDevice(group, keys);
     if (key) {
@@ -581,14 +641,7 @@ function removeGroup(groupIDorName) {
 function changeDeviceOptions(IDorName, newOptions) {
     const device = getDeviceThrowIfNotExists(IDorName);
     const settings = get();
-    const currentOptions = settings.devices[device.ID];
-
-    Object.keys(currentOptions).forEach((key) => {
-        if (newOptions[key]) {
-            currentOptions[key] = newOptions[key];
-        }
-    });
-
+    objectAssignDeep(settings.devices[device.ID], newOptions);
     write();
 }
 
@@ -597,9 +650,15 @@ function changeFriendlyName(IDorName, newName) {
         throw new Error(`friendly_name '${newName}' is already in use`);
     }
 
-    const device = getDeviceThrowIfNotExists(IDorName);
     const settings = get();
-    settings.devices[device.ID].friendly_name = newName;
+    if (getDevice(IDorName)) {
+        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
+    } else if (getGroup(IDorName)) {
+        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
     write();
 }
 

lib/util/utils.js

@@ -27,10 +27,15 @@ function toLocalISOString(dDate) {
         ':' + pad(tzOffset % 60);
 }
 
-const postfixes = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'l1', 'l2', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
-    'ep1', 'ep2',
+const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
 function flatten(arr) {
@@ -42,7 +47,8 @@ function flatten(arr) {
 const forceEndDevice = flatten(
     ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
         .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
-        .map((mappedModel) => mappedModel.zigbeeModel));
+        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
+        .map((definition) => definition.zigbeeModel));
 
 async function getZigbee2mqttVersion() {
     return new Promise((resolve, reject) => {
@@ -108,12 +114,32 @@ function equalsPartial(object, expected) {
     return true;
 }
 
+function getObjectsProperty(objects, key, defaultValue) {
+    for (const object of objects) {
+        if (object.hasOwnProperty(key)) {
+            return object[key];
+        }
+    }
+
+    return defaultValue;
+}
+
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
     objectHasProperties,
-    getPostfixes: () => postfixes,
+    getObjectsProperty,
+    getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
         return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
             (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
@@ -123,4 +149,5 @@ module.exports = {
     isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     formatDate: (date, type) => formatDate(date, type),
     equalsPartial,
+    getResponse,
 };

lib/util/yaml.js

@@ -33,4 +33,12 @@ function writeIfChanged(file, content) {
     }
 }
 
-module.exports = {read, readIfExists, writeIfChanged};
+function updateIfChanged(file, key, value) {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/zigbee.js

@@ -8,26 +8,9 @@ const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
-const postfixes = utils.getPostfixes();
+const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-const herdsmanSettings = {
-    network: {
-        panID: settings.get().advanced.pan_id,
-        extendedPanID: settings.get().advanced.ext_pan_id,
-        channelList: [settings.get().advanced.channel],
-        networkKey: settings.get().advanced.network_key,
-    },
-    databasePath: data.joinPath('database.db'),
-    databaseBackupPath: data.joinPath('database.db.backup'),
-    backupPath: data.joinPath('coordinator_backup.json'),
-    serialPort: {
-        baudRate: settings.get().advanced.baudrate,
-        rtscts: settings.get().advanced.rtscts,
-        path: settings.get().serial.port,
-    },
-};
-
 class Zigbee extends events.EventEmitter {
     constructor() {
         super();
@@ -36,10 +19,37 @@ class Zigbee extends events.EventEmitter {
 
     async start() {
         logger.info(`Starting zigbee-herdsman...`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key,
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+            },
+        };
+
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
         logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
 
+        if (herdsmanSettings.network.networkKey === 'GENERATE') {
+            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+            settings.set(['advanced', 'network_key'], newKey);
+            herdsmanSettings.network.networkKey = newKey;
+        }
+
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
             this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
@@ -127,10 +137,22 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getDevicesByType(type);
     }
 
+    /**
+     * @param {string} key
+     * @return {object} {
+     *      type: device | coordinator
+     *      device|group: zigbee-herdsman entity
+     *      endpoint: selected endpoint (only if type === device)
+     *      settings: from configuration.yaml
+     *      name: name of the entity
+     *      definition: zigbee-herdsman-converters definition (only if type === device)
+     * }
+     */
     resolveEntity(key) {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device', `Wrong type '${typeof key}'`,
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
         );
 
         if (typeof key === 'string' || typeof key === 'number') {
@@ -149,13 +171,13 @@ class Zigbee extends events.EventEmitter {
                 };
             }
 
-            let postfix = postfixes.find((p) => key.endsWith(`/${p}`));
-            const postfixByNumber = key.match(keyEndpointByNumber);
-            if (!postfix && postfixByNumber) {
-                postfix = Number(postfixByNumber[1]);
+            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            const endpointByNumber = key.match(keyEndpointByNumber);
+            if (!endpointKey && endpointByNumber) {
+                endpointKey = Number(endpointByNumber[1]);
             }
-            if (postfix) {
-                key = key.replace(`/${postfix}`, '');
+            if (endpointKey) {
+                key = key.replace(`/${endpointKey}`, '');
             }
 
             const entity = settings.getEntity(key);
@@ -167,19 +189,17 @@ class Zigbee extends events.EventEmitter {
                     return null;
                 }
 
-                const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                const endpoints = mapped && mapped.endpoint ? mapped.endpoint(device) : null;
-                let isDefaultEndpoint = true;
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
                 let endpoint;
-                if (postfix) {
-                    isDefaultEndpoint = false;
-                    if (postfixByNumber) {
-                        endpoint = device.getEndpoint(postfix);
+                if (endpointKey) {
+                    if (endpointByNumber) {
+                        endpoint = device.getEndpoint(endpointKey);
                     } else {
-                        assert(mapped != null, `Postfix '${postfix}' is given but device is unsupported`);
-                        assert(endpoints != null, `Postfix '${postfix}' is given but device defines no endpoints`);
-                        const endpointID = endpoints[postfix];
-                        assert(endpointID, `Postfix '${postfix}' is given but device has no such endpoint`);
+                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
+                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpoints[endpointKey];
+                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
                         endpoint = device.getEndpoint(endpointID);
                     }
                 } else if (endpoints && endpoints['default']) {
@@ -188,24 +208,31 @@ class Zigbee extends events.EventEmitter {
                     endpoint = device.endpoints[0];
                 }
 
-                const endpointName = endpoints ? Object.entries(endpoints).find((e) => e[1] === endpoint.ID)[0] : null;
                 return {
-                    type: 'device', device, settings: entity, mapped, endpoint, name: entity.friendlyName,
-                    isDefaultEndpoint, endpointName,
+                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
                 return {type: 'group', group, settings: entity, name: entity.friendlyName};
             }
-        } else {
+        } else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
             return {
                 type: 'device',
                 device: key,
+                endpoint: key.endpoints[0],
                 settings: setting,
-                mapped: zigbeeHerdsmanConverters.findByZigbeeModel(key.modelID),
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
+                definition: zigbeeHerdsmanConverters.findByDevice(key),
+            };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: setting,
+                name: setting ? setting.friendlyName : key.groupID,
             };
         }
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.8.0",
+  "version": "1.14.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13"
+    "node": "^10 || ^12 || ^13 || ^14"
   },
   "keywords": [
     "xiaomi",
@@ -45,8 +45,9 @@
     "rimraf": "*",
     "semver": "*",
     "winston": "*",
-    "zigbee-herdsman": "0.12.19",
-    "zigbee-herdsman-converters": "11.1.61"
+    "winston-syslog": "*",
+    "zigbee-herdsman": "0.12.94",
+    "zigbee-herdsman-converters": "12.0.110"
   },
   "devDependencies": {
     "eslint": "*",

scripts/install.sh

@@ -26,7 +26,7 @@ chown -R pi:pi /opt/zigbee2mqtt
 
 echo "Running install. This might take a while and can produce som expected errors"
 cd /opt/zigbee2mqtt || exit
-su pi -c "npm install"
+su pi -c "npm ci"
 
 echo "Creating service file zigbee2mqtt.service"
 service_path="/etc/systemd/system/zigbee2mqtt.service"
@@ -70,4 +70,4 @@ fi
 return 0
 }
 
-[[ "$_" == "$0" ]] && zigbee2mqtt-install-package 
+[[ "$_" == "$0" ]] && zigbee2mqtt-install-package

test/assets/exampleExtension.js

@@ -0,0 +1,11 @@
+class Example {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.mqtt = mqtt;
+    }
+
+    onMQTTConnected() {
+        this.mqtt.publish('example/extension', 'test')
+    }
+}
+
+module.exports = Example;

test/availability.test.js

@@ -15,11 +15,11 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device availability', () => {
+describe('Availability', () => {
     let controller;
 
     function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'DeviceAvailability');
+        return controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
     beforeEach(async () => {
@@ -197,9 +197,23 @@ describe('Device availability', () => {
         expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
     });
 
-    it('Should not ping blacklisted devices', async () => {
+    it('Should not ping devices blacklisted by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr])
+        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices blacklisted by IEEE address', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -224,9 +238,23 @@ describe('Device availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping whitelisted devices if availability_whitelist is set', async () => {
+    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr])
+        settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -252,7 +280,6 @@ describe('Device availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(0);
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device});
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
     it('Should not read when device has no modelID and reconnects', async () => {
@@ -325,4 +352,21 @@ describe('Device availability', () => {
 
         device.lastSeen = defaultLastSeen;
     });
+
+    it('Should republish existing state on MQTT connected', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        await controller.stop();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        getExtension().state[device.ieeeAddr] = false;
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
 });

test/bind.test.js

@@ -6,7 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
-describe('Device bind', () => {
+describe('Bind', () => {
     let controller;
 
     mockClear = (device) => {
@@ -15,6 +15,7 @@ describe('Device bind', () => {
             endpoint.write.mockClear();
             endpoint.configureReporting.mockClear();
             endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
         }
     }
 
@@ -29,11 +30,8 @@ describe('Device bind', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to nested topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/+');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/+/+');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/+/+/+');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/+/+/+/+/+');
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
     });
 
     it('Should bind', async () => {
@@ -56,6 +54,17 @@ describe('Device bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'button');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
+    });
+
     it('Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
@@ -171,4 +180,24 @@ describe('Device bind', () => {
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', target);
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
+    });
 });

test/bridge.test.js

@@ -0,0 +1,208 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const {coordinator, bulb, unsupported} = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported.ieeeAddr);
+
+describe('Bridge', () => {
+    let controller;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        settings.set(['advanced', 'legacy_api'], false);
+        settings.set(['experimental', 'new_api'], true);
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+    });
+
+    it('Should publish bridge info on startup', async () => {
+        const version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/info',
+          JSON.stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"logLevel":"info","permitJoin":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          JSON.stringify([{"ieeeAddress":"0x000b57fffec6a5b2","type":"Router","networkAddress":40369,"supported":true,"friendlyName":"bulb","definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"},"powerSource":"Mains (single phase)","dateCode":null,"interviewing":false,"interviewCompleted":true},{"ieeeAddress":"0x0017880104e45518","type":"EndDevice","networkAddress":6536,"supported":false,"friendlyName":"0x0017880104e45518","definition":null,"powerSource":"Battery","dateCode":null,"interviewing":false,"interviewCompleted":true}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/groups',
+          JSON.stringify([{"ID":1,"friendlyName":"group_1","members":[]},{"ID":15071,"friendlyName":"group_tradfri_remote","members":[]},{"ID":99,"friendlyName":99,"members":[]},{"ID":11,"friendlyName":"group_with_tradfri","members":[]},{"ID":2,"friendlyName":"group_2","members":[]}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device joined', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceJoined","data":{"friendlyName":"bulb","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device interview started', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'started'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"started","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview failed', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'failed'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"failed","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview successful', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'successful'});
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.unsupported, status: 'successful'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"successful","ieeeAddress":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+          'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"0x0017880104e45518","status":"successful","ieeeAddress":"0x0017880104e45518","supported":false,"definition":null}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device leaves', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: zigbeeHerdsman.devices.bulb.ieeeAddr});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceLeave","data":{"ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put transaction in response when request is done with transaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false, "transaction": 22}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when request fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error('Failed to connect to adapter')});
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Coverage satisfaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/random', JSON.stringify({"value": false}));
+        const device = zigbeeHerdsman.devices.bulb;
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+    });
+});

test/configure.test.js

@@ -9,7 +9,7 @@ const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device configure', () => {
+describe('Configure', () => {
     let controller;
 
     expectRemoteConfigured = () => {
@@ -105,6 +105,12 @@ describe('Device configure', () => {
         expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
     });
 
+    it('Should skip reconfigure when device does not require this', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
+        await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
+    });
+
     it('Should not configure when interviewing', async () => {
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
@@ -147,7 +153,7 @@ describe('Device configure', () => {
     });
 
     it('Should configure max 3 times when fails', async () => {
-        controller.extensions.find((e) => e.constructor.name === 'DeviceConfigure').attempts = {};
+        controller.extensions.find((e) => e.constructor.name === 'Configure').attempts = {};
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
         const endpoint = device.getEndpoint(1);

test/controller.test.js

@@ -30,7 +30,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),"serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -57,11 +57,13 @@ describe('Controller', () => {
         const configuration = {
             base_topic: "zigbee2mqtt",
             server: "mqtt://localhost",
+            keepalive: 30,
             ca, cert, key,
             password: 'pass',
             user: 'user1',
             client_id: 'my_client_id',
             reject_unauthorized: false,
+            version: 5,
         }
         settings.set(['mqtt'], configuration)
         await controller.start();
@@ -69,6 +71,7 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         const expected = {
             "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"},
+            keepalive: 30,
             ca: Buffer.from([99, 97]),
             key: Buffer.from([107, 101, 121]),
             cert: Buffer.from([99, 101, 114, 116]),
@@ -76,11 +79,20 @@ describe('Controller', () => {
             username: 'user1',
             clientId: 'my_client_id',
             rejectUnauthorized: false,
+            protocolVersion: 5,
 
         }
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should generate network_key when set to GENERATE', async () => {
+        settings.set(['advanced', 'network_key'], 'GENERATE');
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(data.read().advanced.network_key.length).toStrictEqual(16);
+    });
+
     it('Start controller should publish cached states', async () => {
         data.writeDefaultState();
         await controller.start();
@@ -238,9 +250,16 @@ describe('Controller', () => {
         expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1 with groupID 0`);
     });
 
+    it('Should add entities which are missing from configuration but are in database to configuration', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
+    });
+
     it('On zigbee event message from unkown device should create it', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.notInSettings;
+        settings.removeDevice(device.ieeeAddr);
         expect(settings.getDevice(device.ieeeAddr)).toBeNull();
         const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
         await zigbeeHerdsman.events.message(payload);
@@ -358,6 +377,7 @@ describe('Controller', () => {
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
         expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and settings)', async () => {
@@ -397,18 +417,88 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", 'no', {"qos": 0, "retain": true}, expect.any(Function));
     });
 
+    it('Publish entity state attribute_json output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+
+    it('Publish entity state attribute_json output filtered', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered (device_options)', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
     it('Publish entity state with device information', async () => {
         await controller.start();
         settings.set(['mqtt', 'include_device_information'], true);
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)"}}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}', {"qos": 0, "retain": true}, expect.any(Function));
 
         // Unsupported device should have model "unknown"
         await controller.publishEntityState('unsupported2', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery"}}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}', {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state without retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], false);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state with retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Should publish entity state with expiring retention', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'version'], 5);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retention'], 37);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
     });
 
     it('Publish entity state no empty messages', async () => {
@@ -447,4 +537,15 @@ describe('Controller', () => {
         await flushPromises();
         expect(controller.state.state).toStrictEqual({});
     });
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
 });

test/group.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
@@ -14,11 +15,13 @@ describe('Groups', () => {
     let controller;
 
     beforeEach(() => {
+        data.writeEmptyState();
         controller = new Controller();
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
         settings._reRead();
         MQTT.publish.mockClear();
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     })
 
     it('Apply group updates add', async () => {
@@ -296,6 +299,23 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
+    it('Should publish state change of all members when a group changes its state, filtered', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON', brightness: 100}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","brightness":100}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
     it('Shouldnt publish group state change when a group is not optimistic', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -306,7 +326,6 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        controller.state.state = {};
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
@@ -358,7 +377,7 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -383,8 +402,8 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -409,9 +428,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":50,"color_temp":370,"linkquality":99}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":50,"color_temp":370}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -437,7 +456,7 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
-});
+});

test/homeassistant.test.js

@@ -5,7 +5,6 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const flushPromises = () => new Promise(setImmediate);
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 describe('HomeAssistant extension', () => {
     beforeEach(async () => {
@@ -22,7 +21,7 @@ describe('HomeAssistant extension', () => {
     it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
         const missing = [];
         const HomeAssistant = require('../lib/extension/homeassistant');
-        const ha = new HomeAssistant(null, null, null, null);
+        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
 
         require('zigbee-herdsman-converters').devices.forEach((d) => {
             if (!ha._getMapping()[d.model]) {
@@ -33,6 +32,27 @@ describe('HomeAssistant extension', () => {
         expect(missing).toHaveLength(0);
     });
 
+    it('Should not have duplicate type/object_ids in a mapping', () => {
+        const duplicated = [];
+        const HomeAssistant = require('../lib/extension/homeassistant');
+        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
+
+        require('zigbee-herdsman-converters').devices.forEach((d) => {
+            const mapping = ha._getMapping()[d.model];
+            const cfg_type_object_ids = [];
+
+            mapping.forEach((c) => {
+                if (cfg_type_object_ids.includes(c['type'] + '/' + c['object_id'])) {
+                    duplicated.push(d.model);
+                } else {
+                    cfg_type_object_ids.push(c['type'] + '/' + c['object_id']);
+                }
+            });
+        });
+
+        expect(duplicated).toHaveLength(0);
+    });
+
     it('Should discover devices', async () => {
         controller = new Controller(false);
         await controller.start();
@@ -141,7 +161,8 @@ describe('HomeAssistant extension', () => {
         );
 
         payload = {
-            'unit_of_measurement': '-',
+            'icon': 'mdi:signal',
+            'unit_of_measurement': 'lqi',
             'value_template': '{{ value_json.linkquality }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -183,7 +204,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
-            'value_template': "{{ (value_json.temperature | float) | round(1) }}",
+            'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_temperature',
@@ -208,7 +229,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
-            'value_template': '{{ (value_json.humidity | float) | round(0) }}',
+            'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_humidity',
@@ -233,7 +254,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
-            'value_template': '{{ (value_json.pressure | float) | round(2) }}',
+            'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_pressure',
@@ -496,7 +517,7 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        // 1 publish is the publish from deviceReceive
+        // 1 publish is the publish from receive
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
@@ -537,7 +558,7 @@ describe('HomeAssistant extension', () => {
         );
     });
 
-    it('Shouldnt discover when message has no device yet', async () => {
+    it('Shouldnt discover when device leaves', async () => {
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -564,13 +585,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',
+            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            '{"brightness":255}',
+            '{"brightness":255,"update_available":false}',
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -637,4 +658,306 @@ describe('HomeAssistant extension', () => {
             expect.any(Function),
         );
     });
+
+    it('Should clear discovery when device is removed', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'weather_sensor');
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/humidity/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/pressure/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/battery/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/linkquality/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should not clear discovery when unsupported device is removed', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'unsupported2');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should refresh discovery when device is renamed', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', '{"old": "weather_sensor", "new": "weather_sensor_renamed"}');
+        await flushPromises();
+
+        const payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
+            'name': 'weather_sensor_renamed_temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'weather_sensor_renamed',
+                'sw_version': this.version,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability_topic': 'zigbee2mqtt/bridge/state',
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            JSON.stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should discover update_available sensor when device supports it', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const payload = {
+            "payload_on":true,
+            "payload_off":false,
+            "value_template":"{{ value_json.update_available}}",
+            "state_topic":"zigbee2mqtt/bulb",
+            "json_attributes_topic":"zigbee2mqtt/bulb",
+            "name":"bulb_update_available",
+            "unique_id":"0x000b57fffec6a5b2_update_available_zigbee2mqtt",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x000b57fffec6a5b2"
+                ],
+                "name":"bulb",
+                'sw_version': this.version,
+                "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
+                "manufacturer":"IKEA"
+            },
+            "availability_topic":"zigbee2mqtt/bridge/state"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config',
+            JSON.stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should discover trigger when click is published', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+
+        const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
+        expect(discovered.length).toBe(4);
+        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/click/config');
+        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
+
+        MQTT.publish.mockClear();
+
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        const discoverPayload = {
+            "automation_type":"trigger",
+            "type":"click",
+            "subtype":"single",
+            "payload":"single",
+            "topic":"zigbee2mqtt/button/click",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45520"
+                ],
+                "name":"button",
+                "sw_version": this.version,
+                "model":"Aqara wireless switch (WXKG11LM)",
+                "manufacturer":"Xiaomi"
+            }
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
+            JSON.stringify(discoverPayload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button/click',
+            'single',
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button',
+            JSON.stringify({click: "single", linkquality: 10}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button',
+            JSON.stringify({linkquality: 10, click: ""}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        // Should only discover it once
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
+            JSON.stringify(discoverPayload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button/click',
+            'single',
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should disable Home Assistant legacy triggers', async () => {
+        settings.set(['advanced', 'homeassistant_legacy_triggers'], false);
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+
+        const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
+        expect(discovered.length).toBe(2);
+        expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/click/config');
+        expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/action/config');
+
+        MQTT.publish.mockClear();
+
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        const discoverPayload = {
+            "automation_type":"trigger",
+            "type":"click",
+            "subtype":"single",
+            "payload":"single",
+            "topic":"zigbee2mqtt/button/click",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45520"
+                ],
+                "name":"button",
+                "sw_version": this.version,
+                "model":"Aqara wireless switch (WXKG11LM)",
+                "manufacturer":"Xiaomi"
+            }
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
+            JSON.stringify(discoverPayload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button/click',
+            'single',
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button',
+            JSON.stringify({click: "single", linkquality: 10}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should republish payload to postfix topic with lightWithPostfix config', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/U202DST600ZB/l2/set', JSON.stringify({state: 'ON', brightness: 20}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', JSON.stringify({state_l2:"ON", brightness_l2:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l2', JSON.stringify({state:"ON", brightness:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', JSON.stringify({}), {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Shouldnt crash in onPublishEntityState on group publish', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should counter an action/click payload with an empty payload', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/click_single/config');
+        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/click');
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -1,19 +1,19 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const MQTT = require('./stub/mqtt');
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+const MQTT = require('../stub/mqtt');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
 
-describe('Bridge config', () => {
+describe('Bridge legacy', () => {
     let controller;
 
     beforeAll(async () => {
-        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
         controller = new Controller();
         await controller.start();
     })
@@ -100,6 +100,21 @@ describe('Bridge config', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
+        );
     });
 
     it('Should allow permit join', async () => {
@@ -161,8 +176,8 @@ describe('Bridge config', () => {
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/config/devices');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
         expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
-        expect(payload[0]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
-        expect(payload[1]).toStrictEqual({"friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router"});
+        expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
         Date.now = now;
     });
 
@@ -203,6 +218,20 @@ describe('Bridge config', () => {
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
     });
 
+    it('Should allow rename groups', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'group_1', new: 'group_1_renamed'}));
+        await flushPromises();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
     it('Should allow to rename last joined device', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const payload = {device};
@@ -234,9 +263,15 @@ describe('Bridge config', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', 'new_group');
         await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
         expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3)
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
     });
 
     it('Should allow to add groups with json', async () => {
@@ -245,7 +280,13 @@ describe('Bridge config', () => {
         await flushPromises();
         expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3)
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
     });
 
     it('Should allow to add groups with json specifying id', async () => {
@@ -254,7 +295,13 @@ describe('Bridge config', () => {
         await flushPromises();
         expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
     });
 
     it('Should allow to add groups with json specifying only id', async () => {
@@ -267,10 +314,17 @@ describe('Bridge config', () => {
     });
 
     it('Should allow to remove groups', async () => {
-        settings.addGroup('to_be_removed')
-        MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'to_be_removed');
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
         await flushPromises();
         expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
     });
 
     it('Shouldnt allow add groups without id or friendly_name in json', async () => {
@@ -368,8 +422,9 @@ describe('Bridge config', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
         await flushPromises();
         expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
     it('Should handle when ban fails', async () => {
@@ -381,8 +436,9 @@ describe('Bridge config', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
         await flushPromises();
         expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
     it('Should allow to touchlink factory reset (OK)', async () => {
@@ -392,7 +448,7 @@ describe('Bridge config', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
         expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
-        expect(logger.info).toHaveBeenCalledWith('Succesfully factory reset device through Touchlink');
+        expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
     });
 
     it('Should allow to touchlink factory reset (FAILS)', async () => {

test/logger.test.js

@@ -98,4 +98,14 @@ describe('Logger', () => {
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
     });
+
+    it('Should allow to disable log rotation', () => {
+        settings.set(['advanced', 'log_rotation'], false);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].maxFiles).toBeNull();
+        expect(pipes[1].tailable).toBeFalsy();
+        expect(pipes[1].maxsize).toBeNull();
+    });
 });

test/onEvent.test.js

@@ -13,10 +13,11 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const mockOnEvent = jest.fn();
-const mappedLivolo = zigbeeHerdsmanConverters.findByZigbeeModel(zigbeeHerdsman.devices.LIVOLO.modelID);
+const mappedLivolo = zigbeeHerdsmanConverters.findByDevice(zigbeeHerdsman.devices.LIVOLO);
 mappedLivolo.onEvent = mockOnEvent;
+zigbeeHerdsmanConverters.onEvent = jest.fn();
 
-describe('Device event', () => {
+describe('On event', () => {
     let controller;
     const device = zigbeeHerdsman.devices.LIVOLO;
 
@@ -27,6 +28,7 @@ describe('Device event', () => {
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
+        zigbeeHerdsmanConverters.onEvent.mockClear();
         await flushPromises();
     });
 
@@ -59,4 +61,15 @@ describe('Device event', () => {
         expect(call[1]).toStrictEqual({device})
         expect(call[2]).toBe(device);
     });
+
+    it('Should call index onEvent with zigbee event', async () => {
+        zigbeeHerdsmanConverters.onEvent.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledTimes(1);
+        const call = zigbeeHerdsmanConverters.onEvent.mock.calls[0];
+        expect(call[0]).toBe('deviceAnnounce')
+        expect(call[1]).toStrictEqual({device})
+        expect(call[2]).toBe(device);
+    });
 });

test/otaUpdate.test.js

@@ -0,0 +1,215 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+describe('OTA update', () => {
+    let controller;
+
+    mockClear = (mapped) => {
+        mapped.ota.updateToLatest = jest.fn();
+        mapped.ota.isUpdateAvailable = jest.fn();
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+    });
+
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+    });
+
+    it('Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+    });
+
+    it('Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+    });
+
+    it('Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+    });
+
+    it('Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.error.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+    });
+
+    it('Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+    });
+
+    it('Should refuse to check/update when already in progress', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
+            return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
+        });
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
+        jest.runAllTimers();
+        await flushPromises();
+    });
+
+    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            if (count === 1) throw new Error('Failed!')
+            count++;
+            return {swBuildId: 1, dateCode: '2019010'}
+        });
+
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+    });
+
+    it('Should check for update when device requests it', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+
+        // Should not request again when device asks again after a short time
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`)
+    });
+
+    it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
+        const device = zigbeeHerdsman.devices.QBKG04LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
+    });
+
+    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.error.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
+});

test/publish.test.js

@@ -1,11 +1,11 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
@@ -22,7 +22,7 @@ const expectNothingPublished = () => {
     })
 }
 
-describe('Entity publish', () => {
+describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
@@ -30,7 +30,6 @@ describe('Entity publish', () => {
         controller = new Controller();
         await controller.start();
         await flushPromises();
-        await wait(50);
     });
 
     beforeEach(async () => {
@@ -50,6 +49,8 @@ describe('Entity publish', () => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => {
             g.command.mockClear();
         });
+
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
     it('Should publish messages to zigbee devices', async () => {
@@ -251,14 +252,14 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {x: 100, y: 50}, brightness: 20, state: 'OFF'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
-        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -268,10 +269,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -280,10 +281,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -339,7 +340,7 @@ describe('Entity publish', () => {
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('Should publish messages to groups color', async () => {
@@ -413,6 +414,31 @@ describe('Entity publish', () => {
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
+    it('Should handle get with multiple endpoints', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_left: '', state_right: ''}));
+        await flushPromises();
+        expect(endpoint2.read).toHaveBeenCalledTimes(1);
+        expect(endpoint2.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
+    it('Should log error when device has no such endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        logger.error.mockClear();
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_center: '', state_right: ''}));
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'wall_switch_double' has no endpoint 'center'`);
+        expect(endpoint2.read).toHaveBeenCalledTimes(0);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
     it('Should not respond to bridge/config/devices/get', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', JSON.stringify({state: 'ON'}));
         await flushPromises();
@@ -477,10 +503,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -498,6 +524,21 @@ describe('Entity publish', () => {
         expectNothingPublished();
     });
 
+    it('Should send state update on toggle specific endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(2);
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'ON');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'TOGGLE');
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "toggle", {}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'ON'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'OFF'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
     it('Should parse set with postfix topic and attribute', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(2);
@@ -588,6 +629,22 @@ describe('Entity publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
     });
 
+    it('Transition parameter should not influence brightness on state ON', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3563
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 50}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON"}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 10}, {}]);
+    });
+
     it('Should use transition when color temp', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
@@ -644,11 +701,20 @@ describe('Entity publish', () => {
     it('Should turn device off when brightness 0 is send', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        const payload = {'brightness': 0};
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 50, state: 'ON'}));
         await flushPromises();
-        expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "off", {}, {}]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 0}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should turn device off when brightness 0 is send with light_brightness converter', async () => {
@@ -662,14 +728,16 @@ describe('Entity publish', () => {
     });
 
     it('Should allow to set color via hue and saturation', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color_2;
+        const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const payload = {"color":{"hue":250, "saturation":50}};
-        await MQTT.events.message('zigbee2mqtt/bulb_color_2/set', JSON.stringify(payload));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 45510.416666666664, "saturation": 127, "transtime": 0,}, {}]);
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891.475, "saturation": 199.21474, "transtime": 0,}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}});
     });
 
     it('ZNCLDJ11LM open', async () => {
@@ -712,15 +780,24 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("closuresWindowCovering", "stop", {}, {});
     });
 
-
     it('Should turn device on with on/off when transition is provided', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        const payload = {state: 'ON', transition: 1};
+        const payload = {state: 'ON', transition: 3};
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 30}, {}]);
+    });
+
+    it('Should turn device on with on/off with transition when transition 0 is provided', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const payload = {state: 'ON', transition: 0};
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 0}, {}]);
     });
 
     it('Should turn device off with onOff on off with transition', async () => {
@@ -730,10 +807,68 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 10}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
+    });
+
+    it('When device is turned off and on with transition with report enabled it should restore correct brightness', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        // Set initial brightness in state
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 200}));
+        await flushPromises();
+        endpoint.command.mockClear();
+        MQTT.publish.mockClear();
+
+        // Turn off
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF', transition: 3}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+
+        // Bulb reports brightness while decreasing brightness
+        await zigbeeHerdsman.events.message({data: {currentLevel: 1}, cluster: 'genLevelCtrl', device, endpoint, type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 1, linkquality: 10}), {"qos": 0, "retain": false}, expect.any(Function)]);
+
+        // Turn on again
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON', transition: 3}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 30}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 200, linkquality: 10}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
+    it('When device is turned off with transition and turned on WITHOUT transition it should restore the brightness', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        // Set initial brightness in state
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 200}));
+        await flushPromises();
+        endpoint.command.mockClear();
+        MQTT.publish.mockClear();
+
+        // Turn off
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF', transition: 3}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+
+        // Turn on again
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 0}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Home Assistant: should set state', async () => {
@@ -818,7 +953,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -831,7 +966,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/led_controller_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -846,13 +981,29 @@ describe('Entity publish', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
+    it('Should publish brightness_move up to zigbee devices', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness_move: -40}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "move", {"movemode": 1, "rate": 40}, {});
+    });
+
+    it('Should publish brightness_move down to zigbee devices', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness_move: 30}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "move", {"movemode": 0, "rate": 30}, {});
+    });
+
     it('HS2WD-E burglar warning', async () => {
         const endpoint = zigbeeHerdsman.devices.HS2WD.getEndpoint(1);
         const payload = {warning: {duration: 100, mode: 'burglar', strobe: true, level: 'high'}};
         await MQTT.events.message('zigbee2mqtt/siren/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 22, "warningduration": 100}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 22, "warningduration": 100}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E emergency warning', async () => {
@@ -861,7 +1012,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 51, "warningduration": 10}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 51, "warningduration": 10}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E emergency without level', async () => {
@@ -870,7 +1021,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 49, "warningduration": 10}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 49, "warningduration": 10}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E wrong payload (should use defaults)', async () => {
@@ -879,7 +1030,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 53, "warningduration": 10}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 53, "warningduration": 10}, {disableDefaultResponse: true});
     });
 
     it('Shouldnt do anythign when device is not supported', async () => {
@@ -900,4 +1051,99 @@ describe('Entity publish', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({position: 100});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
+
+    it('Should publish to MKS-CM-W5', async () => {
+        const device = zigbeeHerdsman.devices['MKS-CM-W5'];
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: 16}, {disableDefaultResponse: true});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish separate genOnOff to GL-S-007ZS when setting state and brightness as bulb doesnt turn on with moveToLevelWithOnOff', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/2757
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices['GL-S-007ZS'];
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/GL-S-007ZS/set', JSON.stringify({state: 'ON', brightness: 20}));
+        await flushPromises();
+        jest.runAllTimers();
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[0]).toEqual([ 'genOnOff', 'on', {}, {} ]);
+        expect(endpoint.command.mock.calls[1]).toEqual([ 'genLevelCtrl', 'moveToLevelWithOnOff', { level: 20, transtime: 0 }, {} ]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/GL-S-007ZS', '{"state":"ON","brightness":20}', { qos: 0, retain: false }, expect.any(Function)]);
+        jest.useRealTimers();
+    });
+
+    it('Should log as error when setting property with no defined converter', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const payload = {'brightness_move': 20};
+        logger.error.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/get', JSON.stringify(payload));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith("No converter available for 'get' 'brightness_move' (20)");
+    });
+
+    it('Should restore brightness when its turned off without transition and is turned on with', async () => {
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/1097
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.command.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 123}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 0}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "OFF"}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1.0}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 10}, {}]);
+    });
+
+    it('Shouldnt use moveToLevelWithOnOff on turn on when no transition has been used as some devices do not turn on in that case', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3332
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'brightness': 150}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'OFF'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 150, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genOnOff", "on", {}, {}]);
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON"}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"OFF","brightness":0}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[3]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+    });
 });

test/receive.test.js

@@ -8,7 +8,7 @@ const flushPromises = () => new Promise(setImmediate);
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device receive', () => {
+describe('Receive', () => {
     let controller;
 
     beforeEach(async () => {
@@ -33,23 +33,6 @@ describe('Device receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-
-    it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', async () => {
-        settings.set(['homeassistant'], true);
-        const device = zigbeeHerdsman.devices.WXKG11LM;
-        const data = {onOff: 1}
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
     it('Should handle a zigbee message which uses ep (left)', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM;
         const data = {onOff: 1}
@@ -110,6 +93,32 @@ describe('Device receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
+    it('Should debounce and retain messages when set via device_options', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'debounce'], 0.1);
+        settings.set(['device_options', 'retain'], true);
+        delete settings.get().devices['0x0017880104e45522']['retain'];
+        const data1 = {measuredValue: 8}
+        const payload1 = {data: data1, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
+        const data2 = {measuredValue: 1}
+        const payload2 = {data: data2, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        const data3 = {measuredValue: 2}
+        const payload3 = {data: data3, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload3);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": true});
+    });
+
     it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
         jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
@@ -133,6 +142,20 @@ describe('Device receive', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
     });
 
+    it('Shouldnt republish old state', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3572
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        jest.runAllTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+    });
+
     it('Should handle a zigbee message with 1 precision', async () => {
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
@@ -186,6 +209,25 @@ describe('Device receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
+    it('WSDCGQ11LM pressure precision from non ZCL properties', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
+
+        MQTT.publish.mockClear();
+        let payload = {data: {"65281":{"1":2985,"4":5032,"5":9,"6":[0,1],"10":0,"100":2345,"101":4608,"102":91552}}, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5,"linkquality":10});
+
+        MQTT.publish.mockClear();
+        payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4,"linkquality":10});
+    });
+
     it('Should handle a zigbee message with voltage 3010', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM;
         const data = {'65281': {'1': 3010}}
@@ -242,7 +284,7 @@ describe('Device receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, 'voltage': 3045, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, linkquality: 10});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -385,29 +427,27 @@ describe('Device receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        expect(logger.warn).toHaveBeenCalledWith('No converter available for \'ZNCZ02LM\' with cluster \'genBinaryOutput\' and type \'attributeReport\' and data \'{"inactiveText":"hello"}\'');
+        expect(logger.debug).toHaveBeenCalledWith('No converter available for \'ZNCZ02LM\' with cluster \'genBinaryOutput\' and type \'attributeReport\' and data \'{"inactiveText":"hello"}\'');
     });
 
-    it('Should not log when message is from supported device but has no converters is readResponse', async () => {
-        const device = zigbeeHerdsman.devices.ZNCZ02LM;
-        const data = {inactiveText: 'hello'};
-        const payload = {data, cluster: 'genBinaryOutput', device, endpoint: device.getEndpoint(1), type: 'readResponse', linkquality: 20};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        expect(logger.warn).toBeCalledTimes(0);
-    });
+    it('Should report correct energy and power values for different versions of SP600', async () => {
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/915, OLD and NEW use different date code
+        // divisor of OLD is not correct and therefore underreports by factor 10.
+        const data = {instantaneousDemand:496,currentSummDelivered:[0,6648]}
 
-    it('Should allow to calibrate temperature values', async () => {
-        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
-        settings.set(['devices', device.ieeeAddr, 'temperature_calibration'], 2);
-        const data = {measuredValue: -85}
-        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        const SP600_NEW = zigbeeHerdsman.devices.SP600_NEW;
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 1.15, linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6, linkquality: 10});
+
+        MQTT.publish.mockClear();
+        const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496, linkquality: 10});
     });
 });

test/report.test.js

@@ -8,6 +8,7 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45559');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -18,10 +19,10 @@ const debounce = require('debounce');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
 
-describe('Device report', () => {
+describe('Report', () => {
     let controller;
 
-    function expectOnOffBrightnessColorReport(endpoint) {
+    function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
@@ -30,7 +31,11 @@ describe('Device report', () => {
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
         expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 300, "minimumReportInterval": 0, "reportableChange": 0}]);
         expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
-        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
     }
 
     mockClear = (device) => {
@@ -56,7 +61,7 @@ describe('Device report', () => {
     it('Should configure reporting on startup', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, true);
     });
 
     it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
@@ -69,7 +74,7 @@ describe('Device report', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, false);
         expect(device.save).toHaveBeenCalledTimes(1);
     });
 
@@ -129,7 +134,7 @@ describe('Device report', () => {
         const payload = {device};
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, false);
     });
 
     it('Should not configure reporting on device leave', async () => {
@@ -197,4 +202,44 @@ describe('Device report', () => {
         // Should only call Hue bulb, not e.g. tradfri
         expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
     });
+
+    it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
+        const device = zigbeeHerdsman.devices.ZNLDP12LM;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should not setup colorTemperature reporting when bulb does not support it and should read colorCapabilities when its not there yet ', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        endpoint.getClusterAttributeValue = jest.fn();
+
+        let count = 0;
+        endpoint.getClusterAttributeValue.mockImplementation((d) => {
+            count++;
+            if (count === 1) return undefined;
+            return 17;
+        });
+
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
 });

test/settings.test.js

@@ -125,6 +125,12 @@ describe('Settings', () => {
 
         settings._write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'user'], 'test123');
+        settings.set(['advanced', 'network_key'], [1,2,3, 4]);
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
     });
 
     it('Should read devices form a separate file', () => {
@@ -434,6 +440,31 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Shouldnt crash when removing device from group when group has no devices', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
+                '1': {
+                    friendly_name: 'test123',
+                }
+            }
+        });
+
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
     it('Should throw when adding device to non-existing group', () => {
         write(configurationFile, {
             devices: {
@@ -464,6 +495,44 @@ describe('Settings', () => {
         }).toThrow(new Error("Device '0x123' already exists"));
     });
 
+    it('Should not allow any string values for network_key', () => {
+        write(configurationFile, {
+            advanced: {network_key: 'NOT_GENERATE'},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+    });
+
+
+    it('Should not allow retention configuration without MQTT v5', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError('MQTT retention requires protocol version 5');
+    });
+
+    it('Should not allow non-existing entities in availability blacklist', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
+            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+    });
+
     it('Should ban devices', () => {
         write(configurationFile, {});
         settings.banDevice('0x123');
@@ -512,6 +581,30 @@ describe('Settings', () => {
         }).toThrowError(`Duplicate friendly_name 'myname' found`);
     });
 
+    it('Configuration shouldnt be valid when friendly_name ends with /DIGIT', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname/123', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+    });
+
     it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
@@ -521,7 +614,7 @@ describe('Settings', () => {
 
         expect(() => {
             settings.validate();
-        }).toThrowError(`Following friendly_name are not allowed: '${utils.getPostfixes()}'`);
+        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
@@ -539,6 +632,21 @@ describe('Settings', () => {
         }).toThrowError(`friendly_name 'myname' is already in use`);
     });
 
+    it('Should throw when removing device which doesnt exist', async () => {
+        write(configurationFile, {
+            devices: {
+                '0x0017880104e45519': {friendly_name: 'myname', retain: false},
+                '0x0017880104e45511': {friendly_name: 'myname1', retain: false}
+            },
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.removeDevice('myname33');
+        }).toThrowError(`Device 'myname33' does not exist`);
+    });
+
     it('Shouldnt write to configuration.yaml when there are no changes in it', () => {
         const contentConfiguration = {devices: 'devices.yaml'};
         const contentDevices = {};

test/stub/data.js

@@ -132,6 +132,27 @@ function writeDefaultConfiguration() {
             },
             '0x90fd9ffffe4b64af': {
                 friendly_name: 'roller_shutter',
+            },
+            '0x90fd9ffffe4b64ax': {
+                friendly_name: 'ZNLDP12LM',
+            },
+            '0x90fd9ffffe4b64aa': {
+                friendly_name: 'SP600_OLD',
+            },
+            '0x90fd9ffffe4b64ab': {
+                friendly_name: 'SP600_NEW',
+            },
+            '0x90fd9ffffe4b64ac': {
+                friendly_name: 'MKS-CM-W5',
+            },
+            '0x0017880104e45526': {
+                friendly_name: 'GL-S-007ZS',
+            },
+            '0x0017880104e43559': {
+                friendly_name: 'U202DST600ZB'
+            },
+            '0x0017880104e44559': {
+                friendly_name: '3157100_thermostat',
             }
         },
         groups: {
@@ -193,6 +214,7 @@ writeDefaultState();
 
 module.exports = {
     mockDir,
+    read: () => yaml.read(path.join(mockDir, 'configuration.yaml')),
     writeDefaultConfiguration,
     writeDefaultState,
     removeState,

test/stub/zigbeeHerdsman.js

@@ -7,11 +7,14 @@ class Group {
         this.command = jest.fn();
         this.meta = {};
         this.members = members;
+        this.removeFromDatabase = jest.fn();
         this.hasMember = (endpoint) => this.members.includes(endpoint);
     }
 }
 
 const clusters = {
+    'genBasic': 0,
+    'genOta': 25,
     'genScenes': 5,
     'genOnOff': 6,
     'genLevelCtrl': 8,
@@ -20,12 +23,14 @@ const clusters = {
 }
 
 class Endpoint {
-    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[]) {
+    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}) {
         this.deviceIeeeAddress = deviceIeeeAddress;
+        this.clusterValues = clusterValues;
         this.ID = ID;
         this.inputClusters = inputClusters;
         this.outputClusters = outputClusters;
         this.command = jest.fn();
+        this.commandResponse = jest.fn();
         this.read = jest.fn();
         this.write = jest.fn();
         this.bind = jest.fn();
@@ -33,7 +38,7 @@ class Endpoint {
         this.configureReporting = jest.fn();
         this.binds = binds;
         this.supportsInputCluster = (cluster) => {
-            assert(clusters[cluster], `Undefined '${cluster}'`);
+            assert(clusters[cluster] !== undefined, `Undefined '${cluster}'`);
             return this.inputClusters.includes(clusters[cluster]);
         }
 
@@ -60,13 +65,18 @@ class Endpoint {
         this.removeFromAllGroups = () => {
             Object.values(groups).forEach((g) => this.removeFromGroup(g))
         }
+
+        this.getClusterAttributeValue = (cluster, value) => {
+            return this.clusterValues[cluster][value];
+        }
     }
 }
 
 class Device {
-    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName) {
+    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null) {
         this.type = type;
         this.ieeeAddr = ieeeAddr;
+        this.dateCode = dateCode;
         this.networkAddress = networkAddress;
         this.manufacturerID = manufacturerID;
         this.endpoints = endpoints;
@@ -90,13 +100,13 @@ class Device {
 
 const returnDevices = [];
 
-const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3')], true, "Mains (single phase)", "LLC020");
+const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
 const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 
 const devices = {
     'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
-    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
+    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
@@ -111,9 +121,9 @@ const devices = {
     'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
     'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", "lumi.plug"),
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
-    'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
+    'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
     'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
-    'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
+    'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
     'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
     'ZNCLDJ11LM': new Device('Router', '0x0017880104e45547', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'lumi.curtain'),
@@ -129,6 +139,13 @@ const devices = {
     'LIVOLO': new Device('Router', '0x0017880104e45560', 6541,4152, [new Endpoint(6, [0, 6], [])], true, "Mains (single phase)", 'TI0001          '),
     'tradfri_remote': new Device('EndDevice', '0x90fd9ffffe4b64ae', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64ae')], true, "Battery", "TRADFRI remote control"),
     'roller_shutter': new Device('EndDevice', '0x90fd9ffffe4b64af', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64af')], true, "Battery", "SCM-R_00.00.03.15TC"),
+    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "lumi.light.aqcn02"),
+    'SP600_OLD': new Device('Router', '0x90fd9ffffe4b64aa', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20160120'),
+    'SP600_NEW': new Device('Router', '0x90fd9ffffe4b64ab', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20170220'),
+    'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
+    'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
+    'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
 }
 
 const groups = {

update.sh

@@ -11,8 +11,7 @@ git checkout HEAD -- npm-shrinkwrap.json
 git pull
 
 echo "Installing dependencies..."
-rm -rf node_modules
-npm install
+npm ci
 
 echo "Restore configuration..."
 cp -R data-backup/* data