1.14.2

* Update zigbee-herdsman-converters to 12.0.147

* Update homeassistant.js

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

.github/stale.yml

@@ -1,5 +1,5 @@
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
+daysUntilStale: 30
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
 # Label to use when marking an issue as stale

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - name: Get NodeJS Version
+      id: nodeversion
+      run: |
+        echo "::set-output name=version::$(node --version)"
+    - name: Restore node_modules cache
+      id: cache-node-modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+    - name: Install dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      run: npm ci
+    - name: Test
+      run: npm run test-with-coverage
+    - name: Lint
+      run: npm run eslint

.github/workflows/update_deps.yml

@@ -0,0 +1,25 @@
+name: Update dependencies
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 0 * * 0"
+
+jobs:
+  update_deps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install
+    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee-herdsman" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
+    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update dependencies
+        branch: "update_deps"
+        title: Update dependencies

.github/workflows/update_zh.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman
+on:
+  repository_dispatch:
+    types: update_zh
+
+jobs:
+  update_zh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+        branch: "update_zh"
+        title: Update zigbee-herdsman

.github/workflows/update_zhc.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman-converters
+on:
+  repository_dispatch:
+    types: update_zhc
+
+jobs:
+  update_zhc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+        branch: "update_zhc"
+        title: Update zigbee-herdsman-converters

README.md

@@ -66,6 +66,9 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ## Architecture
 ![Architecture](images/architecture.png)
 
+### Internal Architecture
+Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

azure-pipelines.yaml

@@ -76,15 +76,13 @@ steps:
   displayName: 'Docker build release'
   condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
 
-# Trigger downstream
+# Trigger Home Assistant Zigbee2mqtt addon build
 - bash: |
-    BODY="{\"request\": {\"branch\": \"master\"}}"
-    curl -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "Accept: application/json" \
-        -H "Travis-API-Version: 3" \
-        -H "Authorization: token $(TRAVIS_TOKEN)" \
-        -d "$BODY" \
-        "https://api.travis-ci.org/repo/danielwelch%2Fhassio-zigbee2mqtt/requests"
+    curl -s \
+        --request POST \
+        --header "Content-Type: application/json" \
+        --data '{"definition": {"id": 1}}' \
+        -u "$(HASSIO_TRIGGER_SECRET)" \
+        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
   displayName: 'Trigger Hass.io build'
   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

docker/Dockerfile

@@ -3,12 +3,12 @@ FROM node:12-alpine as base
 WORKDIR /app
 RUN apk add --no-cache tzdata eudev
 
-COPY package.json .
+COPY package.json ./
 
 # Dependencies
 FROM base as dependencies
 
-COPY npm-shrinkwrap.json .
+COPY npm-shrinkwrap.json ./
 
 RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
     npm ci --production && \
@@ -19,12 +19,15 @@ FROM base as release
 
 COPY --from=dependencies /app/node_modules ./node_modules
 COPY lib ./lib
-COPY index.js docker/run.sh data/configuration.yaml ./
+COPY index.js data/configuration.yaml ./
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 
-RUN chmod +x /app/run.sh
 RUN mkdir /app/data
 
 ARG COMMIT
 RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
 
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["npm", "start"]

docker/docker-entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 if [ ! -z "$ZIGBEE2MQTT_DATA" ]; then
     DATA="$ZIGBEE2MQTT_DATA"
@@ -13,4 +14,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-exec npm start
+exec "$@"

lib/controller.js

@@ -19,12 +19,14 @@ const ExtensionHomeAssistant = require('./extension/homeassistant');
 const ExtensionConfigure = require('./extension/configure');
 const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
 const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
 const ExtensionAvailability = require('./extension/availability');
 const ExtensionBind = require('./extension/bind');
 const ExtensionReport = require('./extension/report');
 const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
+const ExtensionExternalConverters = require('./extension/externalConverters');
 
 class Controller {
     constructor() {
@@ -50,7 +52,10 @@ class Controller {
             new ExtensionOTAUpdate(...args),
         ];
 
-        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.extensions.push(new ExtensionBridge(...args));
+        }
+
         if (settings.get().advanced.legacy_api) {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
@@ -71,6 +76,9 @@ class Controller {
         if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
+        }
 
         const extensionPath = data.joinPath('extension');
         if (fs.existsSync(extensionPath)) {
@@ -135,14 +143,18 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        await this.zigbee.permitJoin(settings.get().permit_join);
+        try {
+            await this.zigbee.permitJoin(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
+        }
 
         // MQTT
         this.mqtt.on('message', this.onMQTTMessage.bind(this));
         await this.mqtt.connect();
 
         // Send all cached states.
-        if (settings.get().advanced.cache_state) {
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
             for (const device of this.zigbee.getClients()) {
                 if (this.state.exists(device.ieeeAddr)) {
                     this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
@@ -249,12 +261,6 @@ class Controller {
             return;
         }
 
-        const isDevice = resolvedEntity.type === 'device';
-
-        if (isDevice && settings.get().advanced.last_seen !== 'disable' && resolvedEntity.device.lastSeen) {
-            payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, settings.get().advanced.last_seen);
-        }
-
         let messagePayload = {...payload};
         const currentState = this.state.exists(resolvedEntity.settings.ID) ?
             this.state.get(resolvedEntity.settings.ID) : {};
@@ -279,6 +285,7 @@ class Controller {
             options.properties = {messageExpiryInterval: retention};
         }
 
+        const isDevice = resolvedEntity.type === 'device';
         if (isDevice && settings.get().mqtt.include_device_information) {
             const attributes = [
                 'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
@@ -293,6 +300,11 @@ class Controller {
             attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
         }
 
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+        }
+
         // filter mqtt message attributes
         if (deviceOptions.filtered_attributes) {
             deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
@@ -315,7 +327,7 @@ class Controller {
             }
         }
 
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity});
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {

lib/eventBus.js

@@ -7,6 +7,7 @@ const allowedEvents = [
     'groupRenamed', // Group has been renamed
     'publishEntityState', // Entity state will be published
     'stateChange', // Entity changes its state
+    'groupMembersChanged', // Members of a group has been changed
 ];
 
 class EventBus extends events.EventEmitter {

lib/extension/availability.js

@@ -24,22 +24,27 @@ class Availability extends Extension {
         this.timers = {};
         this.state = {};
 
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
+        this.blocklist = settings.get().advanced.availability_blocklist
+            .concat(settings.get().advanced.availability_blacklist)
+            .map((e) => settings.getEntity(e).ID);
+
+        this.passlist = settings.get().advanced.availability_passlist
+            .concat(settings.get().advanced.availability_whitelist)
+            .map((e) => settings.getEntity(e).ID);
     }
 
-    inWhitelistOrNotInBlacklist(device) {
+    inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
         const deviceSettings = settings.getDevice(ieeeAddr);
         const name = deviceSettings.friendlyName;
 
-        // Whitelist is not empty and device is in it, enable availability
-        if (this.whitelist.length > 0) {
-            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
+        // Passlist is not empty and device is in it, enable availability
+        if (this.passlist.length > 0) {
+            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
         }
 
-        // Device is on blacklist, disable availability
-        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
+        // Device is on blocklist, disable availability
+        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
             return false;
         }
 
@@ -58,13 +63,14 @@ class Availability extends Extension {
     onMQTTConnected() {
         for (const device of this.zigbee.getClients()) {
             // Mark all devices as online on start
-            this.publishAvailability(device, true);
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
 
-            if (this.inWhitelistOrNotInBlacklist(device)) {
+            if (this.inPasslistOrNotInBlocklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
-                    this.timers[device.ieeeAddr] = setInterval(() => {
+                    this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
                     }, utils.secondsToMilliseconds(300));
                 }
@@ -94,12 +100,12 @@ class Availability extends Extension {
     }
 
     async handleIntervalNotPingable(device) {
-        const ago = Date.now() - device.lastSeen;
         const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
         if (!resolvedEntity || !device.lastSeen) {
             return;
         }
 
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
         logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
 
         if (ago > Hours25) {
@@ -143,7 +149,7 @@ class Availability extends Extension {
         }
     }
 
-    publishAvailability(device, available) {
+    publishAvailability(device, available, force=false) {
         const ieeeAddr = device.ieeeAddr;
         if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
             this.onReconnect(device);
@@ -153,7 +159,7 @@ class Availability extends Extension {
         const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
         const topic = `${name}/availability`;
         const payload = available ? 'online' : 'offline';
-        if (this.state[ieeeAddr] !== available) {
+        if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;
             this.mqtt.publish(topic, payload, {retain: true, qos: 0});
         }
@@ -165,7 +171,7 @@ class Availability extends Extension {
             return;
         }
 
-        if (this.inWhitelistOrNotInBlacklist(device)) {
+        if (this.inPasslistOrNotInBlocklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {

lib/extension/bind.js

@@ -1,7 +1,8 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
+const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const Extension = require('./extension');
 
 const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
@@ -21,6 +22,12 @@ class Bind extends Extension {
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
+        }
     }
 
     parseMQTTMessage(topic, message) {
@@ -33,6 +40,11 @@ class Bind extends Extension {
             type = topic.split('/')[0];
             sourceKey = topic.replace(`${type}/`, '');
             targetKey = message;
+        } else if (settings.get().experimental.new_api && topic.match(topicRegex)) {
+            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
+            message = JSON.parse(message);
+            sourceKey = message.from;
+            targetKey = message.to;
         }
 
         return {type, sourceKey, targetKey};
@@ -42,78 +54,104 @@ class Bind extends Extension {
         const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
         if (!type) return null;
 
-        // Find source; can only be a device and target
+        let error = null;
         const source = this.zigbee.resolveEntity(sourceKey);
-        assert(source != null && source.type === 'device', 'Source undefined or not a device');
         const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
-        assert(target != null, 'Target is unknown');
+        const responseData = {from: sourceKey, to: targetKey};
 
-        const sourceName = source.settings.friendlyName;
-        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
-        let attemptedToBindSomething = false;
+        if (!source || source.type !== 'device') {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const sourceName = source.settings.friendlyName;
+            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+            const successfulClusters = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
 
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters.
-        for (const cluster of clusters) {
-            const targetValid = target.type === 'group' || target.type === 'group_number' ||
-                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusters) {
+                const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
 
-            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
-                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
-                attemptedToBindSomething = true;
-                try {
-                    let bindTarget = null;
-                    if (target.type === 'group') bindTarget = target.group;
-                    else if (target.type === 'group_number') bindTarget = target.ID;
-                    else bindTarget = target.endpoint;
+                if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                    attemptedClusters.push(cluster);
 
-                    if (type === 'bind') {
-                        await source.endpoint.bind(cluster, bindTarget);
-                    } else {
-                        await source.endpoint.unbind(cluster, bindTarget);
-                    }
+                    try {
+                        let bindTarget = null;
+                        if (target.type === 'group') bindTarget = target.group;
+                        else if (target.type === 'group_number') bindTarget = target.ID;
+                        else bindTarget = target.endpoint;
 
-                    logger.info(
-                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceName}' to '${targetName}'`,
-                    );
+                        if (type === 'bind') {
+                            await source.endpoint.bind(cluster, bindTarget);
+                        } else {
+                            await source.endpoint.unbind(cluster, bindTarget);
+                        }
 
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        this.mqtt.publish(
-                            'bridge/log',
-                            JSON.stringify({type: `device_${type}`,
-                                message: {from: sourceName, to: targetName, cluster}}),
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${sourceName}' to '${targetName}'`,
                         );
-                    }
-                } catch (error) {
-                    logger.error(
-                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
-                        `'${targetName}' (${error})`,
-                    );
 
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        this.mqtt.publish(
-                            'bridge/log',
-                            JSON.stringify({type: `device_${type}_failed`,
-                                message: {from: sourceName, to: targetName, cluster}}),
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                JSON.stringify({type: `device_${type}`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                            `'${targetName}' (${error})`,
                         );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                JSON.stringify({type: `device_${type}_failed`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
                     }
                 }
             }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
         }
 
-        if (!attemptedToBindSomething) {
-            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, JSON.stringify(response));
+        }
 
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
-                );
-            }
+        if (error) {
+            logger.error(error);
         }
     }
 }

lib/extension/bridge.js

@@ -0,0 +1,390 @@
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.lastJoinedDeviceIeeeAddr = null;
+
+        this.requestLookup = {
+            'device/options': this.deviceOptions.bind(this),
+            'device/remove': this.deviceRemove.bind(this),
+            'device/rename': this.deviceRename.bind(this),
+            'group/add': this.groupAdd.bind(this),
+            'group/options': this.groupOptions.bind(this),
+            'group/remove': this.groupRemove.bind(this),
+            'group/rename': this.groupRename.bind(this),
+            'permit_join': this.permitJoin.bind(this),
+            'config/last_seen': this.configLastSeen.bind(this),
+            'config/elapsed': this.configElapsed.bind(this),
+            'config/log_level': this.configLogLevel.bind(this),
+            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'health_check': this.healthCheck.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
+        }
+
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') {
+                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
+            } else if (type === 'deviceInterview') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
+                };
+                if (data.status === 'successful') {
+                    payload.supported = !!resolvedEntity.definition;
+                    payload.definition = resolvedEntity.definition ? {
+                        model: resolvedEntity.definition.model,
+                        vendor: resolvedEntity.definition.vendor,
+                        description: resolvedEntity.definition.description,
+                        supports: resolvedEntity.definition.supports,
+                    } : null;
+                }
+            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish(
+                'bridge/event',
+                JSON.stringify({type: utils.toSnakeCase(type), data: payload}),
+                {retain: false, qos: 0},
+            );
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    async deviceOptions(message) {
+        return this.changeEntityOptions('device', message);
+    }
+
+    async groupOptions(message) {
+        return this.changeEntityOptions('group', message);
+    }
+
+    async deviceRemove(message) {
+        return this.removeEntity('device', message);
+    }
+
+    async groupRemove(message) {
+        return this.removeEntity('group', message);
+    }
+
+    async healthCheck(message) {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    async groupAdd(message) {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
+    }
+
+    async deviceRename(message) {
+        return this.renameEntity('device', message);
+    }
+
+    async groupRename(message) {
+        return this.renameEntity('group', message);
+    }
+
+    async permitJoin(message) {
+        const value = this.getValue(message);
+        await this.zigbee.permitJoin(value);
+        await this.publishInfo();
+        return utils.getResponse(message, {value: value}, null);
+    }
+
+    configLastSeen(message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configElapsed(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configLogLevel(message) {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    async touchlinkFactoryReset(message) {
+        logger.info('Start touchlink factory reset');
+        const result = await this.zigbee.touchlinkFactoryReset();
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, {}, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
+    /**
+     * Utils
+     */
+
+    getValue(message) {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
+
+            return message.value;
+        } else {
+            return message;
+        }
+    }
+
+    changeEntityOptions(entityType, message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        settings.changeEntityOptions(ID, message.options);
+        const cleanup = (o) => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+        const oldOptions = cleanup(entity.settings);
+        const newOptions = cleanup(settings.getEntity(ID));
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
+
+    renameEntity(entityType, message) {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
+
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const entity = this.getEntity(entityType, from);
+
+        settings.changeFriendlyName(from, to);
+
+        if (entity.type === 'device') {
+            this.publishDevices();
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device});
+        } else {
+            this.publishGroups();
+            this.eventBus.emit(`groupRenamed`, {group: entity.group});
+        }
+
+        return utils.getResponse(message, {from: entity.settings.friendlyName, to}, null);
+    }
+
+    async removeEntity(entityType, message) {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity.type === 'device') {
+                if (block) {
+                    settings.blockDevice(entity.settings.ID);
+                }
+
+                if (force) {
+                    await entity.device.removeFromDatabase();
+                } else {
+                    await entity.device.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    await entity.group.removeFromDatabase();
+                } else {
+                    await entity.group.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity.type === 'device') {
+                this.eventBus.emit('deviceRemoved', {device: entity.device});
+            }
+
+            // Remove from configuration.yaml
+            if (entity.type === 'device') {
+                settings.removeDevice(entity.settings.ID);
+            } else {
+                settings.removeGroup(entity.settings.ID);
+            }
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+
+            if (entity.type === 'device') {
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type, ID) {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.type !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
+
+    async publishInfo() {
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    async publishDevices() {
+        const devices = this.zigbee.getClients().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const definitionPayload = definition ? {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                supports: definition.supports,
+            } : null;
+
+            return {
+                ieee_address: device.ieeeAddr,
+                type: device.type,
+                network_address: device.networkAddress,
+                supported: !!definition,
+                friendly_name: resolved.name,
+                definition: definitionPayload,
+                power_source: device.powerSource,
+                software_build_id: device.softwareBuildID,
+                date_code: device.dateCode,
+                interviewing: device.interviewing,
+                interview_completed: device.interviewCompleted,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+    }
+
+    async publishGroups() {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                id: group.groupID,
+                friendly_name: resolved.name,
+                members: group.members.map((m) => {
+                    return {
+                        ieee_address: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+    }
+}
+
+module.exports = Bridge;

lib/extension/configure.js

@@ -1,4 +1,5 @@
 const settings = require('../util/settings');
+const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 
@@ -12,7 +13,7 @@ class Configure extends Extension {
         this.configuring = new Set();
         this.attempts = {};
 
-        this.legacyApi = settings.get().advanced.legacy_api;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
     }
 
@@ -36,18 +37,18 @@ class Configure extends Extension {
 
     onMQTTConnected() {
         /* istanbul ignore else */
-        if (this.legacyApi) {
+        if (settings.get().advanced.legacy_api) {
             this.mqtt.subscribe(this.legacyTopic);
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(this.topic);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            if (topic !== this.legacyTopic) {
-                return;
-            }
-
+        if (topic === this.legacyTopic) {
             const resolvedEntity = this.zigbee.resolveEntity(message);
             if (!resolvedEntity || resolvedEntity.type !== 'device') {
                 logger.error(`Device '${message}' does not exist`);
@@ -60,6 +61,26 @@ class Configure extends Extension {
             }
 
             this.configure(resolvedEntity, true);
+        } else if (topic === this.topic) {
+            message = utils.parseJSON(message, message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const resolvedEntity = this.zigbee.resolveEntity(ID);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                error = `Device '${ID}' does not exist`;
+            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                error = `Device '${resolvedEntity.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(resolvedEntity, true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, JSON.stringify(response));
         }
     }
 
@@ -87,7 +108,7 @@ class Configure extends Extension {
         }
     }
 
-    async configure(resolvedEntity, force=false) {
+    async configure(resolvedEntity, force=false, thowError=false) {
         const device = resolvedEntity.device;
         if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
             return false;
@@ -106,11 +127,14 @@ class Configure extends Extension {
             device.meta.configured = resolvedEntity.definition.meta.configureKey;
             device.save();
         } catch (error) {
-            logger.error(
-                `Failed to configure '${resolvedEntity.name}', ` +
-                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
-            );
             this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
         }
 
         this.configuring.delete(device.ieeeAddr);

lib/extension/externalConverters.js

@@ -0,0 +1,18 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+
+class ExternalConverters extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/groups.js

@@ -4,6 +4,8 @@ const Extension = require('./extension');
 const utils = require('../util/utils');
 const postfixes = utils.getEndpointNames();
 
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
@@ -26,6 +28,13 @@ class Groups extends Extension {
                 this.mqtt.subscribe(`${topic}remove_all`);
             }
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
+        }
     }
 
     async onZigbeeStarted() {
@@ -143,20 +152,30 @@ class Groups extends Extension {
         let resolvedEntityGroup = null;
         let resolvedEntityDevice = null;
         let hasEndpointName = null;
+        let error = null;
+        let groupKey = null;
+        let deviceKey = null;
+        let triggeredViaLegacyApi = false;
 
         /* istanbul ignore else */
-        if (this.legacyApi) {
-            const topicMatch = topic.match(legacyTopicRegex);
-            if (topicMatch) {
-                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
-                type = topicMatch[2];
+        const topicRegexMatch = topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
+                type = legacyTopicRegexMatch[2];
 
                 if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
-                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        const payload = {
+                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
+                        };
                         this.mqtt.publish(
                             'bridge/log',
                             JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
@@ -165,10 +184,8 @@ class Groups extends Extension {
 
                     return {};
                 }
-            } else if (topic.match(legacyTopicRegexRemoveAll)) {
-                type = 'remove_all';
             } else {
-                return {};
+                type = 'remove_all';
             }
 
             resolvedEntityDevice = this.zigbee.resolveEntity(message);
@@ -177,7 +194,9 @@ class Groups extends Extension {
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                    const payload = {
+                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
                     this.mqtt.publish(
                         'bridge/log',
                         JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
@@ -188,78 +207,125 @@ class Groups extends Extension {
             }
 
             hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1];
+            message = JSON.parse(message);
+            deviceKey = message.device;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
+            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
+                error = `Device '${message.device}' does not exist`;
+            }
+
+            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
         }
 
-        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
+            triggeredViaLegacyApi,
+        };
     }
 
     async onMQTTMessage(topic, message) {
-        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
-            this.parseMQTTMessage(topic, message);
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
+            groupKey, deviceKey,
+        } = this.parseMQTTMessage(topic, message);
         if (!type) return;
 
-        const keys = [
-            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
-            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
-        ];
-
-        const definition = resolvedEntityDevice.definition;
-        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
-        const endpointName = endpoints ?
-            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
-
-        if (endpointName) {
-            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
-            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+        const responseData = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
         }
 
-        if (!hasEndpointName) {
-            keys.push(resolvedEntityDevice.name);
-            keys.push(resolvedEntityDevice.device.ieeeAddr);
-        }
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+                ];
 
-        if (type === 'add') {
-            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
-            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
-            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+                const definition = resolvedEntityDevice.definition;
+                const endpoints = definition && definition.endpoint ?
+                    definition.endpoint(resolvedEntityDevice.device) : null;
+                const endpointName = endpoints ?
+                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
 
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_add`, message: payload}),
-                );
-            }
-        } else if (type === 'remove') {
-            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
-            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
-            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_remove`, message: payload}),
-                );
-            }
-        } else { // remove_all
-            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
-            await resolvedEntityDevice.endpoint.removeFromAllGroups();
-            for (const settingsGroup of settings.getGroups()) {
-                settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: resolvedEntityDevice.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
-                    );
+                if (endpointName) {
+                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
                 }
+
+                if (!hasEndpointName) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.device.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
             }
         }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, JSON.stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit('groupMembersChanged', {group: resolvedEntityGroup});
+        }
     }
 }
 

lib/extension/homeassistant.js

@@ -1,6 +1,7 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../util/settings');
 const logger = require('../util/logger');
+const utils = require('../util/utils');
 const zigbee2mqttVersion = require('../../package.json').version;
 const Extension = require('./extension');
 const objectAssignDeep = require(`object-assign-deep`);
@@ -95,14 +96,14 @@ const cfg = {
             device_class: 'safety',
         },
     },
-    'binary_sensor_router': {
+    'binary_sensor_led': {
         type: 'binary_sensor',
-        object_id: 'router',
+        object_id: 'led',
         discovery_payload: {
             payload_on: true,
             payload_off: false,
-            value_template: '{{ value_json.state }}',
-            device_class: 'connectivity',
+            value_template: '{{ value_json.led }}',
+            device_class: 'light',
         },
     },
     'binary_sensor_battery_low': {
@@ -144,6 +145,16 @@ const cfg = {
             device_class: 'lock',
         },
     },
+    'binary_sensor_power_alarm_active': {
+        type: 'binary_sensor',
+        object_id: 'power_alarm_active',
+        discovery_payload: {
+            payload_on: true,
+            payload_off: false,
+            value_template: '{{ value_json.power_alarm_active}}',
+            device_class: 'power',
+        },
+    },
 
     // Sensor
     'sensor_illuminance': {
@@ -155,6 +166,15 @@ const cfg = {
             value_template: '{{ value_json.illuminance }}',
         },
     },
+    'sensor_illuminance_lux_unit': {
+        type: 'sensor',
+        object_id: 'illuminance',
+        discovery_payload: {
+            unit_of_measurement: 'lx',
+            device_class: 'illuminance',
+            value_template: '{{ value_json.illuminance }}',
+        },
+    },
     'sensor_illuminance_lux': {
         type: 'sensor',
         object_id: 'illuminance_lux',
@@ -406,31 +426,38 @@ const cfg = {
             icon: 'mdi:weight',
         },
     },
+    'sensor_requested_brightness_level': {
+        type: 'sensor',
+        object_id: 'requested_brightness_level',
+        discovery_payload: {
+            value_template: '{{ value_json.requested_brightness_level }}',
+            icon: 'mdi:brightness-5',
+        },
+    },
+    'sensor_requested_brightness_percent': {
+        type: 'sensor',
+        object_id: 'requested_brightness_percent',
+        discovery_payload: {
+            value_template: '{{ value_json.requested_brightness_percent }}',
+            icon: 'mdi:brightness-5',
+        },
+    },
+    'sensor_radioactive_events_per_minute': {
+        type: 'sensor',
+        object_id: 'radioactive_events_per_minute',
+        discovery_payload: {
+            value_template: '{{ value_json.radioactive_events_per_minute }}',
+        },
+    },
+    'sensor_radiation_dose_per_hour': {
+        type: 'sensor',
+        object_id: 'radiation_dose_per_hour',
+        discovery_payload: {
+            value_template: '{{ value_json.radiation_dose_per_hour }}',
+        },
+    },
 
     // Light
-    'light_brightness_colorxy_white': {
-        type: 'light',
-        object_id: 'light',
-        discovery_payload: {
-            brightness: true,
-            xy: true,
-            white_value: true,
-            schema: 'json',
-            command_topic: true,
-        },
-    },
-    'light_brightness_colortemp_colorxy_white': {
-        type: 'light',
-        object_id: 'light',
-        discovery_payload: {
-            brightness: true,
-            xy: true,
-            white_value: true,
-            schema: 'json',
-            command_topic: true,
-            color_temp: true,
-        },
-    },
     'light_brightness_colortemp_colorxy': {
         type: 'light',
         object_id: 'light',
@@ -440,6 +467,7 @@ const cfg = {
             xy: true,
             schema: 'json',
             command_topic: true,
+            brightness_scale: 254,
         },
     },
     'light_brightness_colorxy': {
@@ -450,6 +478,7 @@ const cfg = {
             xy: true,
             schema: 'json',
             command_topic: true,
+            brightness_scale: 254,
         },
     },
     'light_brightness_colortemp_colorhs': {
@@ -461,6 +490,7 @@ const cfg = {
             hs: true,
             schema: 'json',
             command_topic: true,
+            brightness_scale: 254,
         },
     },
     'light_brightness_colortemp': {
@@ -471,6 +501,7 @@ const cfg = {
             color_temp: true,
             schema: 'json',
             command_topic: true,
+            brightness_scale: 254,
         },
     },
     'light_brightness': {
@@ -480,6 +511,7 @@ const cfg = {
             brightness: true,
             schema: 'json',
             command_topic: true,
+            brightness_scale: 254,
         },
     },
 
@@ -654,6 +686,16 @@ const lightEndpoint = (configType, endpointName) => {
     return config;
 };
 
+const sensorEndpoint = (endpointName) => {
+    return {
+        type: 'sensor',
+        object_id: `sensor_${endpointName}`,
+        discovery_payload: {
+            value_template: `{{ value_json.${endpointName} }}`,
+        },
+    };
+};
+
 const thermostat = (minTemp=7, maxTemp=30, temperatureStateProperty='occupied_heating_setpoint', tempStep=1) => {
     return {
         type: 'climate',
@@ -673,21 +715,23 @@ const thermostat = (minTemp=7, maxTemp=30, temperatureStateProperty='occupied_he
             temperature_command_topic: temperatureStateProperty,
             temp_step: tempStep,
             action_topic: true,
-            action_template: '{{ value_json.operation }}',
+            action_template:
+                '{% set values = {\'idle\':\'off\',\'heat\':\'heating\',\'cool\':\'cooling\',\'fan only\':\'fan\'}'+
+                ' %}{{ values[value_json.running_state] }}',
         },
     };
 };
 
-const thermostatHeatCool = /* istanbul ignore next */ (minTemp=7, maxTemp=30, tempStep=0.5, fanModes=[]) => {
+const thermostatHeatCool = /* istanbul ignore next */ (minTemp=7, maxTemp=30, tempStep=0.5, opModes, fanModes=[]) => {
     return {
         type: 'climate',
         object_id: 'climate',
         discovery_payload: {
             state_topic: false,
-            unit_of_measurement: '°C',
+            temperature_unit: 'C',
             min_temp: `${minTemp}`,
             max_temp: `${maxTemp}`,
-            modes: ['off', 'auto', 'heat', 'cool'],
+            modes: opModes,
             mode_state_topic: true,
             mode_state_template: '{{ value_json.system_mode }}',
             mode_command_topic: true,
@@ -719,23 +763,24 @@ const mapping = {
     'WXKG12LM': [cfg.sensor_click, cfg.sensor_battery, cfg.sensor_action],
     // DEPRECATED; BREAKING_IMPROVEMENT: only use sensor_click for WXKG03LM (action hold -> click hold)
     'WXKG03LM': [cfg.sensor_click, cfg.sensor_battery, cfg.sensor_action],
+    'WXKG06LM': [cfg.sensor_battery, cfg.sensor_action],
     'WXKG02LM': [cfg.sensor_click, cfg.sensor_battery],
     'QBKG04LM': [cfg.switch, cfg.sensor_click, cfg.sensor_action],
     'QBKG03LM': [switchEndpoint('left'), switchEndpoint('right'), cfg.sensor_click, cfg.sensor_temperature],
     'WSDCGQ01LM': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
     'WSDCGQ11LM': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_pressure, cfg.sensor_battery],
     'RTCGQ01LM': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
-    'RTCGQ11LM': [cfg.binary_sensor_occupancy, cfg.sensor_illuminance, cfg.sensor_battery],
+    'RTCGQ11LM': [cfg.binary_sensor_occupancy, cfg.sensor_illuminance_lux_unit, cfg.sensor_battery],
     'MCCGQ01LM': [cfg.binary_sensor_contact, cfg.sensor_battery],
     'MCCGQ11LM': [cfg.binary_sensor_contact, cfg.sensor_battery],
     'SJCGQ11LM': [cfg.binary_sensor_water_leak, cfg.sensor_battery],
     'MFKZQ01LM': [cfg.sensor_action, cfg.sensor_battery],
-    'ZNCZ02LM': [cfg.switch, cfg.sensor_power],
+    'ZNCZ02LM': [cfg.switch, cfg.sensor_power, cfg.sensor_temperature, cfg.sensor_consumption],
     'QBCZ11LM': [cfg.switch, cfg.sensor_power],
     'LED1545G12': [cfg.light_brightness_colortemp],
     'LED1623G12': [cfg.light_brightness],
     'LED1622G12': [cfg.light_brightness],
-    'LED1537R6': [cfg.light_brightness_colortemp],
+    'LED1537R6/LED1739R5': [cfg.light_brightness_colortemp],
     'LED1650R5': [cfg.light_brightness],
     'LED1536G5': [cfg.light_brightness_colortemp],
     '7299760PH': [cfg.light_brightness_colorxy],
@@ -746,9 +791,10 @@ const mapping = {
     'JTYJ-GD-01LM/BW': [cfg.binary_sensor_smoke, cfg.sensor_battery, cfg.sensor_sensitivity, cfg.sensor_smoke_density],
     'PLUG EDP RE:DY': [cfg.switch, cfg.sensor_power],
     'SWITCH EDP RE:DY': [cfg.switch],
-    'CC2530.ROUTER': [cfg.binary_sensor_router],
+    'CC2530.ROUTER': [cfg.binary_sensor_led],
     'AA70155': [cfg.light_brightness_colortemp],
     'A9A19A60WESDZ02': [cfg.light_brightness_colortemp],
+    'A9BR3065WESDZ02': [cfg.light_brightness_colortemp],
     '4058075816718': [cfg.light_brightness_colortemp_colorxy],
     'AA69697': [cfg.light_brightness_colortemp_colorxy],
     'HALIGHTDIMWWE27': [cfg.light_brightness],
@@ -793,8 +839,11 @@ const mapping = {
     'SL 110 N': [cfg.light_brightness],
     'SL 110 M': [cfg.light_brightness],
     'SL 110 W': [cfg.light_brightness],
+    'AE 260': [cfg.light_brightness],
     'AA68199': [cfg.light_brightness_colortemp],
-    'QBKG11LM': [cfg.switch, cfg.sensor_power, cfg.sensor_click],
+    'QBKG11LM': [cfg.switch, cfg.sensor_power, cfg.sensor_click, cfg.sensor_temperature],
+    'QBKG21LM': [cfg.switch, cfg.sensor_click, cfg.sensor_action],
+    'QBKG22LM': [cfg.switch, cfg.sensor_click, cfg.sensor_temperature],
     'QBKG12LM': [
         switchEndpoint('left'), switchEndpoint('right'), cfg.sensor_power, cfg.sensor_click,
         cfg.sensor_temperature,
@@ -816,7 +865,7 @@ const mapping = {
         cfg.binary_sensor_occupancy, cfg.sensor_temperature, cfg.sensor_illuminance, cfg.sensor_illuminance_lux,
         cfg.sensor_battery,
     ],
-    'GL-C-008': [cfg.light_brightness_colortemp_colorxy],
+    'GL-C-008-1ID': [cfg.light_brightness_colortemp_colorxy],
     'GL-C-009': [cfg.light_brightness],
     'STSS-MULT-001': [cfg.binary_sensor_contact, cfg.sensor_battery],
     'E11-G23/E11-G33': [cfg.light_brightness],
@@ -858,7 +907,7 @@ const mapping = {
     '7299355PH': [cfg.light_brightness_colorxy],
     '45857GE': [cfg.light_brightness],
     'A6121': [cfg.sensor_lock],
-    '433714': [cfg.light_brightness],
+    '433714': [cfg.light_brightness_colortemp],
     '3261030P7': [cfg.light_brightness_colortemp],
     '3216431P5': [cfg.light_brightness_colortemp],
     'DJT11LM': [cfg.sensor_action, cfg.sensor_battery, cfg.sensor_sensitivity, cfg.sensor_strength],
@@ -886,7 +935,8 @@ const mapping = {
     '3216331P5': [cfg.light_brightness_colortemp],
     'AC08562': [cfg.light_brightness],
     '900008-WW': [cfg.light_brightness],
-    'Mega23M12': [cfg.light_brightness_colortemp_colorxy],
+    'Mega23M12': [lightEndpoint('light_brightness_colortemp_colorxy', 'rgb'),
+        lightEndpoint('light_brightness', 'white')],
     'PSS-23ZBS': [cfg.switch],
     'HS1SA-M': [cfg.binary_sensor_smoke, cfg.binary_sensor_battery_low],
     'Z01-A19NAE26': [cfg.light_brightness_colortemp],
@@ -904,13 +954,14 @@ const mapping = {
     'HS1-WL-E': [cfg.binary_sensor_water_leak],
     '421786': [cfg.light_brightness],
     'ICZB-IW11D': [cfg.light_brightness],
-    '3321-S': [cfg.binary_sensor_contact, cfg.sensor_temperature],
+    '3321-S': [cfg.binary_sensor_contact, cfg.sensor_temperature, cfg.sensor_battery],
     'ZPIR-8000': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
     'ZCTS-808': [cfg.binary_sensor_contact, cfg.sensor_battery],
     'ZNLDP12LM': [cfg.light_brightness_colortemp],
+    'XDD12LM': [cfg.light_brightness_colortemp],
     'D1821': [cfg.light_brightness_colortemp_colorxy],
     'ZNCLDJ11LM': [cfg.cover_position, cfg.sensor_cover],
-    'owvfni3': [cfg.cover_position],
+    'TS0601': [cfg.cover_position],
     'LTFY004': [cfg.light_brightness_colorxy],
     'GL-S-007Z': [cfg.light_brightness_colortemp_colorxy],
     '3325-S': [cfg.sensor_temperature, cfg.binary_sensor_occupancy],
@@ -937,9 +988,12 @@ const mapping = {
     'HGZB-07A': [cfg.light_brightness_colortemp_colorxy],
     'E1524/E1810': [cfg.sensor_action, cfg.sensor_battery],
     'GL-C-006': [cfg.light_brightness_colortemp],
-    'GL-C-007': [cfg.light_brightness_colorxy_white],
-    'GL-C-007S': [cfg.light_brightness_colorxy_white],
-    'GL-C-007/GL-C-008': [cfg.light_brightness_colortemp_colorxy_white],
+    'GL-C-007-1ID': [cfg.light_brightness_colortemp_colorxy],
+    'GL-C-007-2ID': [lightEndpoint('light_brightness_colortemp_colorxy', 'rgb'),
+        lightEndpoint('light_brightness', 'white')],
+    'GL-C-008-2ID': [lightEndpoint('light_brightness_colorxy', 'rgb'),
+        lightEndpoint('light_brightness_colortemp', 'cct')],
+    'GL-C-007S': [cfg.light_brightness_colorxy],
     '100.424.11': [cfg.light_brightness_colortemp],
     'AC0251100NJ/AC0251700NJ': [cfg.sensor_action, cfg.sensor_battery],
     '71831': [cfg.light_brightness_colortemp],
@@ -947,14 +1001,14 @@ const mapping = {
     '44435': [cfg.light_brightness_colortemp_colorxy],
     '404006/404008/404004': [cfg.light_brightness_colortemp],
     'MLI-404011': [cfg.sensor_action],
-    'GL-S-003Z': [cfg.light_brightness_colorxy_white],
+    'GL-S-003Z': [cfg.light_brightness_colorxy],
     'GL-S-005Z': [cfg.light_brightness_colortemp_colorxy],
     'HS1DS-E': [cfg.binary_sensor_contact],
-    'SP600': [cfg.switch, cfg.sensor_power],
+    'SP600': [cfg.switch, cfg.sensor_power, cfg.sensor_energy],
     '1613V': [cfg.switch, cfg.sensor_power],
     'XVV-Mega23M12': [cfg.light_brightness_colortemp],
     'GL-B-007Z': [cfg.light_brightness_colortemp_colorxy],
-    '81809': [cfg.light_brightness_colortemp_colorxy],
+    '81809/81813': [cfg.light_brightness_colortemp_colorxy],
     '4090130P7': [cfg.light_brightness_colortemp_colorxy],
     '100.110.39': [cfg.light_brightness_colortemp_colorxy],
     'TI0001': [switchEndpoint('left'), switchEndpoint('right')],
@@ -967,9 +1021,11 @@ const mapping = {
     ],
     '07005B': [cfg.light_brightness],
     '07004D': [cfg.light_brightness_colortemp_colorxy],
+    '07008L': [cfg.light_brightness_colortemp_colorxy],
     'E1746': [],
     'LED1836G9': [cfg.light_brightness],
     'YRD426NRSC': [cfg.lock, cfg.sensor_battery],
+    'BE468': [cfg.lock, cfg.sensor_battery],
     'YRD246HA20BP': [cfg.lock, cfg.sensor_battery],
     'E1743': [cfg.sensor_click, cfg.sensor_battery],
     'LED1732G11': [cfg.light_brightness_colortemp],
@@ -982,6 +1038,7 @@ const mapping = {
     'HGZB-042': [switchEndpoint('top'), switchEndpoint('bottom')],
     'HGZB-42': [switchEndpoint('top'), switchEndpoint('bottom')],
     'GL-FL-004TZ': [cfg.light_brightness_colortemp_colorxy],
+    'GL-FL-004TZS': [cfg.light_brightness_colortemp_colorxy],
     'IM6001-OTP05': [cfg.switch],
     'SV01': [
         cfg.cover_position, cfg.sensor_temperature, cfg.sensor_pressure,
@@ -998,7 +1055,10 @@ const mapping = {
         cfg.sensor_local_temperature,
         cfg.lock_keypad_lockout,
     ],
-    'E1525/E1745': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
+    'E1525/E1745': [
+        cfg.binary_sensor_occupancy, cfg.sensor_battery, cfg.sensor_requested_brightness_level,
+        cfg.sensor_requested_brightness_percent,
+    ],
     'ZYCT-202': [cfg.sensor_action],
     'GR-ZB01-W': [cfg.cover_position],
     '4090531P7': [cfg.light_brightness_colortemp_colorxy],
@@ -1010,6 +1070,10 @@ const mapping = {
     '9290018195': [cfg.light_brightness],
     'HGZB-04D / HGZB-4D-UK': [cfg.light_brightness],
     'HGZB-043': [switchEndpoint('top'), switchEndpoint('bottom'), switchEndpoint('center')],
+    'HGZB-44': [
+        switchEndpoint('top_left'), switchEndpoint('top_right'), switchEndpoint('bottom_left'),
+        switchEndpoint('bottom_right'),
+    ],
     'NCZ-3043-HA': [
         cfg.binary_sensor_occupancy, cfg.sensor_temperature,
         cfg.sensor_battery, cfg.binary_sensor_battery_low,
@@ -1096,14 +1160,14 @@ const mapping = {
     '3315-G': [
         cfg.sensor_temperature, cfg.sensor_battery, cfg.binary_sensor_water_leak,
     ],
-    'N2G-SP': [cfg.sensor_power, cfg.switch],
+    'N2G-SP': [cfg.sensor_power, cfg.switch, cfg.sensor_energy],
     'AC0363900NJ': [cfg.light_brightness_colortemp_colorxy],
     'LXZB-02A': [cfg.light_brightness],
     'GL-S-004Z': [cfg.light_brightness_colortemp],
     'SCM-5ZBS': [cfg.cover_position],
     'YRD226HA2619': [cfg.sensor_battery, cfg.lock],
     'YMF40/YDM4109+': [cfg.lock, cfg.sensor_battery],
-    'V3-BTZB': [cfg.lock],
+    'V3-BTZB': [cfg.lock, cfg.sensor_battery],
     '3RSS008Z': [cfg.switch, cfg.sensor_battery],
     '3RSS007Z': [cfg.switch],
     '99432': [cfg.fan, cfg.light_brightness],
@@ -1113,6 +1177,7 @@ const mapping = {
     ],
     'HLC821-Z-SC': [cfg.light_brightness],
     'RS 228 T': [cfg.light_brightness_colortemp],
+    'RS 229 T': [cfg.light_brightness_colortemp],
     '67200BL': [cfg.switch],
     'InstaRemote': [cfg.sensor_action],
     '100.425.90': [cfg.switch],
@@ -1128,7 +1193,7 @@ const mapping = {
     'ZigUP': [cfg.switch],
     'YRD256HA20BP': [cfg.sensor_battery, cfg.lock],
     'SZ-ESW01-AU': [cfg.sensor_power, cfg.switch],
-    'PSM-29ZBSR': [cfg.switch],
+    'PSM-29ZBSR': [cfg.switch, cfg.sensor_power],
     'ZM350STW1TCF': [cfg.light_brightness_colortemp],
     'M350STW1': [cfg.light_brightness],
     'A806S-Q1R': [cfg.light_brightness],
@@ -1208,14 +1273,21 @@ const mapping = {
     'TH1124ZB': [thermostat()],
     'TH1400ZB': [thermostat()],
     'TH1500ZB': [thermostat()],
-    'Zen-01-W': [thermostat()],
+    'Zen-01-W': [thermostat(10, 30, 'occupied_heating_setpoint', 0.5)],
     '9290022166': [cfg.light_brightness_colortemp_colorxy],
     'PM-C140-ZB': [cfg.sensor_power, cfg.switch],
     'PM-B530-ZB': [cfg.sensor_power, cfg.switch],
+    'PM-B540-ZB': [cfg.sensor_power, cfg.switch],
     'PM-B430-ZB': [cfg.sensor_power, cfg.switch],
     'ptvo.switch': [
-        switchEndpoint('bottom_left'), switchEndpoint('bottom_right'), switchEndpoint('top_left'),
-        switchEndpoint('top_right'), switchEndpoint('center'), cfg.sensor_click,
+        switchEndpoint('l1'), switchEndpoint('l2'), switchEndpoint('l3'),
+        switchEndpoint('l4'), switchEndpoint('l5'), switchEndpoint('l6'),
+        switchEndpoint('l7'), switchEndpoint('l8'),
+        sensorEndpoint('l1'), sensorEndpoint('l2'), sensorEndpoint('l3'),
+        sensorEndpoint('l4'), sensorEndpoint('l5'), sensorEndpoint('l6'),
+        sensorEndpoint('l7'), sensorEndpoint('l8'),
+        cfg.sensor_click, cfg.sensor_temperature, cfg.sensor_voltage,
+        cfg.sensor_pressure, cfg.sensor_humidity,
     ],
     'DIYRuZ_R4_5': [
         switchEndpoint('bottom_left'), switchEndpoint('bottom_right'), switchEndpoint('top_left'),
@@ -1224,6 +1296,9 @@ const mapping = {
     'DIYRuZ_KEYPAD20': [],
     'DTB190502A1': [],
     'FL 130 C': [cfg.light_brightness_colortemp_colorxy],
+    'OFL 120 C': [cfg.light_brightness_colortemp_colorxy],
+    'OFL 140 C': [cfg.light_brightness_colortemp_colorxy],
+    'OSL 130 C': [cfg.light_brightness_colortemp_colorxy],
     'BF 263': [cfg.light_brightness],
     'RF 263': [cfg.light_brightness],
     'HS1CG-M': [cfg.binary_sensor_gas],
@@ -1286,7 +1361,7 @@ const mapping = {
     'ZM-CSW032-D': [cfg.cover_position],
     'LH-32ZB': [cfg.sensor_humidity, cfg.sensor_temperature, cfg.sensor_battery],
     '511.201': [cfg.light_brightness],
-    'ZNCLDJ12LM': [cfg.cover_position],
+    'ZNCLDJ12LM': [cfg.cover_position, cfg.sensor_battery],
     '046677552343': [cfg.switch],
     '3115331PH': [cfg.light_brightness_colortemp_colorxy],
     'ZWLD-100': [cfg.binary_sensor_water_leak, cfg.binary_sensor_battery_low, cfg.sensor_battery],
@@ -1330,7 +1405,10 @@ const mapping = {
     'Eco-Dim.07': [cfg.light_brightness],
     'DIYRuZ_rspm': [cfg.switch, cfg.sensor_action, cfg.sensor_power, cfg.sensor_current],
     'ZG9101SAC-HP-Switch': [cfg.switch],
-    'ZNCZ04LM': [cfg.switch, cfg.sensor_power],
+    'ZNCZ04LM': [
+        cfg.switch, cfg.sensor_power, cfg.sensor_consumption,
+        cfg.sensor_current, cfg.sensor_voltage, cfg.sensor_temperature,
+    ],
     'ZNCZ12LM': [cfg.switch, cfg.sensor_power],
     'GL-S-007ZS': [cfg.light_brightness_colortemp_colorxy],
     '4058075816732': [cfg.light_brightness_colortemp_colorxy],
@@ -1383,9 +1461,9 @@ const mapping = {
     'WV704R0A0902': [thermostat()],
     '067776': [cfg.cover_position],
     '067773': [cfg.sensor_action, cfg.sensor_battery],
-    '067771': [cfg.switch],
+    '067771': [cfg.light_brightness],
     '064873': [cfg.sensor_action],
-    'K4003C': [cfg.switch],
+    'K4003C': [cfg.switch, cfg.sensor_action],
     'STZB402': [
         thermostat(5, 30, 'occupied_heating_setpoint', 0.5),
         cfg.sensor_local_temperature,
@@ -1406,6 +1484,7 @@ const mapping = {
     '12126': [cfg.switch],
     '067775': [cfg.switch, cfg.sensor_power],
     '064888': [cfg.switch],
+    '412015': [cfg.sensor_power, cfg.binary_sensor_power_alarm_active],
     'gq8b1uv': [cfg.light_brightness],
     'GZCGQ01LM': [cfg.sensor_battery, cfg.sensor_illuminance, cfg.sensor_illuminance_lux],
     '9290018215': [cfg.light_brightness],
@@ -1413,13 +1492,14 @@ const mapping = {
     '1744130P7': [cfg.light_brightness_colortemp_colorxy],
     '1743130P7': [cfg.light_brightness_colortemp_colorxy],
     '100.110.51': [cfg.light_brightness_colortemp],
-    'ZL1000100-CCT-US-V1A02': [cfg.light_brightness],
+    'ZL1000100-CCT-US-V1A02': [cfg.light_brightness_colortemp],
+    'ZL1000701-27-EU-V1A02': [cfg.light_brightness],
     'HGZB-DLC4-N12B': [cfg.light_brightness_colortemp_colorxy],
     'U86KCJ-ZP': [cfg.sensor_action],
     'HS1HT': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
     'HS2ESK-E': [cfg.switch, cfg.sensor_power],
     'B01M7Y8BP9': [cfg.sensor_action],
-    'GP-WOU019BBDWG': [cfg.switch, cfg.sensor_power],
+    'GP-WOU019BBDWG': [cfg.switch, cfg.sensor_power, cfg.sensor_energy],
     'AV2010/21A': [cfg.binary_sensor_battery_low, cfg.binary_sensor_contact, cfg.binary_sensor_tamper],
     'AL-PIR02': [cfg.binary_sensor_occupancy, cfg.binary_sensor_tamper, cfg.sensor_battery],
     'MKS-CM-W5': [
@@ -1427,7 +1507,7 @@ const mapping = {
         switchEndpoint('l3'), switchEndpoint('l4'),
     ],
     'STS-WTR-250': [cfg.binary_sensor_water_leak, cfg.sensor_battery, cfg.sensor_temperature],
-    'ZG2835RAC': [cfg.light_brightness],
+    'ZG2835RAC': [cfg.light_brightness, cfg.sensor_power, cfg.sensor_energy],
     'BW-IS2': [cfg.binary_sensor_contact, cfg.sensor_battery],
     'BW-IS3': [cfg.binary_sensor_occupancy],
     'SLR1b': [thermostat()],
@@ -1449,15 +1529,15 @@ const mapping = {
     '10011725': [cfg.light_brightness_colortemp_colorxy],
     '929002277501': [cfg.light_brightness],
     'RS 230 C': [cfg.light_brightness_colortemp_colorxy],
-    'LED1903C5': [cfg.light_brightness_colortemp],
+    'LED1903C5/LED1835C6': [cfg.light_brightness_colortemp],
     '1402755': [cfg.light_brightness],
     '4503848C5': [cfg.light_brightness_colortemp],
     '500.48': [cfg.light_brightness],
     'TS0042': [cfg.sensor_action],
-    'RHK06': [cfg.binary_sensor_contact, cfg.sensor_battery],
-    'RHK07': [cfg.sensor_action, cfg.sensor_battery],
-    'RHK08': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
-    'RHK09': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
+    'SNZB-04': [cfg.binary_sensor_contact, cfg.sensor_battery],
+    'SNZB-01': [cfg.sensor_action, cfg.sensor_battery],
+    'SNZB-02': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
+    'SNZB-03': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
     '07046L': [cfg.sensor_action],
     '07045L': [cfg.binary_sensor_contact, cfg.binary_sensor_tamper, cfg.binary_sensor_battery_low],
     '3402831P7': [cfg.light_brightness_colortemp],
@@ -1489,6 +1569,9 @@ const mapping = {
     'PM-S140-ZB': [cfg.switch],
     'PM-S240-ZB': [switchEndpoint('top'), switchEndpoint('bottom')],
     'PM-S340-ZB': [switchEndpoint('top'), switchEndpoint('center'), switchEndpoint('bottom')],
+    'PM-S140R-ZB': [cfg.switch],
+    'PM-S240R-ZB': [switchEndpoint('top'), switchEndpoint('bottom')],
+    'PM-S340R-ZB': [switchEndpoint('top'), switchEndpoint('center'), switchEndpoint('bottom')],
     'U201DST600ZB': [cfg.light_brightness],
     'U201SRY2KWZB': [cfg.switch],
     'U202SRY2KWZB': [switchEndpoint('l1'), switchEndpoint('l2')],
@@ -1517,6 +1600,7 @@ const mapping = {
     '4080248P9': [cfg.light_brightness_colortemp_colorxy],
     '4080148P9': [cfg.light_brightness_colortemp_colorxy],
     '4058075148338': [cfg.light_brightness_colortemp],
+    '4058075181472': [cfg.light_brightness_colortemp],
     '484719': [cfg.light_brightness],
     'SEB01ZB': [cfg.binary_sensor_sos, cfg.sensor_battery],
     'SBM01ZB': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
@@ -1533,7 +1617,7 @@ const mapping = {
     'E12-N1E': [cfg.light_brightness_colortemp_colorxy],
     '4040B': [cfg.sensor_power, switchEndpoint('l1'), switchEndpoint('l2')],
     '3460-L': [cfg.sensor_action, cfg.sensor_temperature, cfg.sensor_battery],
-    '3157100': [thermostatHeatCool(10, 30, 1, ['auto', 'on']), cfg.sensor_battery],
+    '3157100': [thermostatHeatCool(10, 30, 1, ['off', 'heat', 'cool'], ['auto', 'on']), cfg.sensor_battery],
     '4257050-RZHAC': [cfg.switch, cfg.sensor_power],
     '27087-03': [cfg.switch, cfg.sensor_battery],
     '99140-002': [cfg.lock, cfg.sensor_battery],
@@ -1541,12 +1625,14 @@ const mapping = {
     'GL-S-004ZS': [cfg.light_brightness_colortemp_colorxy],
     '7121131PU': [cfg.light_brightness_colortemp_colorxy],
     'RL804QZB': [switchEndpoint('l1'), switchEndpoint('l2'), switchEndpoint('l3')],
-    'RC-2000WH': [thermostatHeatCool(10, 30, 1, ['auto', 'on', 'smart'])],
+    'RC-2000WH': [thermostatHeatCool(10, 30, 1, ['off', 'auto', 'heat', 'cool'], ['auto', 'on', 'smart'])],
     'TH1300ZB': [thermostat()],
     'SP 220': [cfg.switch],
     '511.040': [cfg.light_brightness_colortemp_colorxy],
     '511.344': [cfg.sensor_battery, cfg.sensor_action, cfg.sensor_action_color, cfg.sensor_action_color_temperature],
     'SMSZB-120': [cfg.binary_sensor_smoke, cfg.sensor_temperature, cfg.sensor_battery],
+    'DWS003': [cfg.binary_sensor_contact, cfg.sensor_battery, cfg.binary_sensor_battery_low, cfg.sensor_temperature],
+    'MOT003': [cfg.binary_sensor_occupancy, cfg.sensor_temperature, cfg.sensor_battery, cfg.binary_sensor_battery_low],
     'HALIGHTDIMWWE14': [cfg.light_brightness],
     'GreenPower_On_Off_Switch': [cfg.sensor_action],
     'GreenPower_7': [cfg.sensor_action],
@@ -1558,14 +1644,12 @@ const mapping = {
     'AU-A1GUZB5/30': [cfg.light_brightness],
     'AU-A1GUZBRGBW': [cfg.light_brightness_colortemp_colorxy],
     'AU-A1GSZ9RGBW': [cfg.light_brightness_colortemp_colorxy],
+    'AU-A1ZB2WDM': [cfg.light_brightness],
     'RF 261': [cfg.light_brightness],
     'RF 264': [cfg.light_brightness],
-    'HY369RT': [
-        cfg.lock_child_lock,
-        cfg.switch_window_detection,
-        cfg.switch_valve_detection,
+    'TS0601_thermostat': [
+        cfg.lock_child_lock, cfg.switch_window_detection, cfg.switch_valve_detection, cfg.sensor_battery,
         thermostat(5, 30, 'current_heating_setpoint', 0.5),
-        cfg.sensor_battery,
     ],
     'WXKG07LM': [cfg.sensor_action, cfg.sensor_battery],
     'MCLH-03': [cfg.switch, cfg.sensor_voltage, cfg.sensor_power, cfg.sensor_current],
@@ -1573,7 +1657,7 @@ const mapping = {
     '676-00301024955Z': [cfg.light_brightness],
     '1743030P7': [cfg.light_brightness_colortemp_colorxy],
     'XBee': [],
-    'ZL1000701-27-EU-V1A02': [cfg.light_brightness],
+    'ZBHT-1': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
     'rgbw2.zbee27': [cfg.light_brightness_colortemp_colorxy],
     'MCLH-05': [cfg.binary_sensor_occupancy, cfg.sensor_battery],
     '5045148P7': [cfg.light_brightness_colortemp_colorxy],
@@ -1581,16 +1665,129 @@ const mapping = {
     '5AA-SS-ZA-H0': [cfg.binary_sensor_occupancy, cfg.sensor_illuminance, cfg.sensor_illuminance_lux],
     'MOSZB-130': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
     'AU-A1ZBRC': [cfg.sensor_action, cfg.sensor_battery],
-    'AU-A1ZBPIRS': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
-    'TS0121': [cfg.switch],
+    'AU-A1ZBPIRS': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low, cfg.sensor_illuminance_lux],
+    'TS0121': [cfg.switch, cfg.sensor_voltage, cfg.sensor_power, cfg.sensor_current],
     'ZK03840': [thermostat()],
     'ZS1100400-IN-V1A02': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
     'MCLH-08': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_eco2, cfg.sensor_voc],
     'GL-FL-005TZ': [cfg.light_brightness_colortemp_colorxy],
-    'ED2004-012': [],
+    'ED2004-012': [cfg.switch],
     'ZG192910-4': [cfg.light_brightness_colortemp],
+    'SLT2': [],
+    'ZL1000700-22-EU-V1A02': [cfg.light_brightness],
+    'SLB2': [],
+    '8840100H': [cfg.binary_sensor_water_leak, cfg.sensor_temperature, cfg.sensor_battery],
+    '404037': [cfg.light_brightness_colortemp],
+    '9290022408': [cfg.switch],
+    'HGZB-14A': [cfg.binary_sensor_water_leak, cfg.sensor_battery],
+    'TERNCY-DC01': [cfg.sensor_temperature, cfg.binary_sensor_contact],
+    'ZS110050078': [cfg.binary_sensor_contact, cfg.binary_sensor_battery_low],
+    'HGZB-DLC4-N15B': [cfg.light_brightness_colortemp_colorxy],
+    'ECW-100-A03': [switchEndpoint('top'), switchEndpoint('center'), switchEndpoint('bottom')],
+    '4098430P7': [cfg.light_brightness_colortemp],
+    'PQC19-DY01': [cfg.light_brightness],
+    'DIYRuZ_FreePad': [cfg.sensor_action, cfg.sensor_battery],
+    'ST20': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
+    'ST21': [cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_battery],
+    'T30W3Z': [switchEndpoint('top'), switchEndpoint('center'), switchEndpoint('bottom')],
+    'T21W2Z': [switchEndpoint('top'), switchEndpoint('bottom')],
+    'T21W1Z': [cfg.switch],
+    'W40CZ': [cfg.cover_position],
+    'R11W2Z': [switchEndpoint('l1'), switchEndpoint('l2')],
+    'R20W2Z': [switchEndpoint('l1'), switchEndpoint('l2')],
+    'SW21': [cfg.binary_sensor_water_leak, cfg.binary_sensor_battery_low],
+    'SE21': [cfg.sensor_action],
+    '4052899926127': [cfg.light_brightness],
+    'GL-B-001ZS': [cfg.light_brightness_colortemp_colorxy],
+    'LH-990ZB': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
+    'HO-09ZB': [cfg.binary_sensor_contact, cfg.binary_sensor_battery_low],
+    '500.67': [cfg.sensor_action],
+    'E1E-G7F': [cfg.sensor_action],
+    'LH-990F': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
+    'QBKG25LM': [
+        switchEndpoint('left'), switchEndpoint('center'), switchEndpoint('right'), cfg.sensor_action,
+        cfg.sensor_power,
+    ],
+    'QBKG23LM': [cfg.switch, cfg.sensor_power],
+    'QBKG24LM': [switchEndpoint('left'), switchEndpoint('right'), cfg.sensor_power],
+    'WS-USC01': [cfg.switch],
+    'WS-USC02': [switchEndpoint('top'), switchEndpoint('bottom')],
+    'WS-USC04': [switchEndpoint('top'), switchEndpoint('bottom')],
+    '100.462.31': [cfg.sensor_action],
+    'SN10ZW': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
+    'AU-A1ZBPIAB': [cfg.switch, cfg.sensor_voltage, cfg.sensor_current, cfg.sensor_power],
+    'AU-A1ZBDWS': [cfg.binary_sensor_contact, cfg.sensor_battery],
+    '4058075816459': [cfg.sensor_action, cfg.sensor_battery],
+    '14592.0': [cfg.switch],
+    '73699': [cfg.light_brightness_colorxy],
+    'SAGE206612': [cfg.sensor_action, cfg.sensor_battery],
+    'TI0001-switch': [cfg.switch],
+    'TI0001-socket': [cfg.switch],
+    '9290022891': [cfg.light_brightness_colortemp_colorxy],
+    '160-01': [cfg.switch, cfg.sensor_power],
+    'ZS232000178': [cfg.sensor_action],
+    'mcdj3aq': [cfg.cover_position],
+    'DIYRuZ_Geiger': [cfg.sensor_radioactive_events_per_minute, cfg.sensor_radiation_dose_per_hour, cfg.sensor_action],
+    '8718696170557': [cfg.light_brightness_colortemp_colorxy],
+    '12127': [switchEndpoint('l1'), switchEndpoint('l2')],
+    'SWO-MOS1PA': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
+    'STS-IRM-251': [cfg.sensor_temperature, cfg.binary_sensor_occupancy, cfg.sensor_battery],
+    'WSDCGQ12LM': [cfg.sensor_temperature, cfg.sensor_pressure, cfg.sensor_humidity, cfg.sensor_battery],
+    'SJCGQ12LM': [cfg.sensor_battery, cfg.binary_sensor_water_leak],
+    'DJT12LM': [cfg.sensor_action],
+    'AV2010/22A': [cfg.binary_sensor_occupancy, cfg.binary_sensor_battery_low],
+    'D1523': [cfg.light_brightness],
+    'RS-23ZBS': [cfg.sensor_temperature, cfg.sensor_humidity],
+    'E11-U3E': [cfg.light_brightness_colortemp_colorxy],
+    '5062231P7': [cfg.light_brightness_colortemp_colorxy],
+    '5062431P7': [cfg.light_brightness_colortemp_colorxy],
+    '5062131P7': [cfg.light_brightness_colortemp_colorxy],
+    '5062331P7': [cfg.light_brightness_colortemp_colorxy],
+    'GL-D-004Z': [cfg.light_brightness_colortemp_colorxy],
+    '9290022268': [cfg.light_brightness],
+    '73773': [cfg.light_brightness_colortemp_colorxy],
+    'KMPCIL_RES005': [
+        cfg.sensor_battery, cfg.sensor_temperature, cfg.sensor_humidity, cfg.sensor_pressure, cfg.sensor_illuminance,
+        cfg.binary_sensor_occupancy, cfg.switch,
+    ],
+    'DIYRuZ_R8_8': [
+        switchEndpoint('l1'), switchEndpoint('l2'), switchEndpoint('l3'), switchEndpoint('l4'),
+        switchEndpoint('l5'), switchEndpoint('l6'), switchEndpoint('l7'), switchEndpoint('l8'),
+    ],
+    '511.010': [cfg.light_brightness],
+    '43080': [cfg.light_brightness],
+    'DIYRuZ_RT': [cfg.switch, cfg.sensor_temperature],
+    '929002039801': [cfg.light_brightness],
+    'DM2500ZB': [cfg.light_brightness],
+    '99100-006': [cfg.lock, cfg.sensor_battery],
+    'SPE600': [cfg.switch, cfg.sensor_power],
+    '067774': [cfg.sensor_action, cfg.sensor_battery],
+    '067694': [cfg.sensor_action, cfg.sensor_battery],
+    '170-33505': [cfg.switch, cfg.sensor_voltage, cfg.sensor_current, cfg.sensor_power],
+    '07115L': [cfg.light_brightness_colortemp_colorxy],
+    'A806S-Q1G': [cfg.light_brightness_colortemp_colorxy],
+    'YRL-220L': [cfg.lock, cfg.sensor_battery],
+    'YSR-MINI-01': [cfg.light_brightness_colortemp_colorxy],
+    '9290023349': [cfg.light_brightness],
+    'TS0115': [
+        switchEndpoint('l1'), switchEndpoint('l2'), switchEndpoint('l3'), switchEndpoint('l4'),
+        switchEndpoint('l5'),
+    ],
+    'WL4200': [cfg.binary_sensor_water_leak, cfg.binary_sensor_battery_low],
+    '404031': [cfg.light_brightness_colortemp],
+    'AE 280 C': [cfg.light_brightness_colortemp_colorxy],
+    'HLC833-Z-SC': [cfg.light_brightness],
+    '5996411U5': [cfg.light_brightness_colortemp],
+    'LXN59-2S7LX1.0': [switchEndpoint('left'), switchEndpoint('right')],
+    'X711A': [cfg.switch],
+    'X712A': [switchEndpoint('l1'), switchEndpoint('l2')],
+    'TYZS1L': [cfg.light_brightness_colortemp_colorxy],
+    '43102': [cfg.switch],
+    '1746430P7': [cfg.light_brightness_colortemp_colorxy],
 };
 
+const defaultStatusTopic = 'homeassistant/status';
+
 /**
  * This extensions handles integration with HomeAssistant
  */
@@ -1616,9 +1813,16 @@ class HomeAssistant extends Extension {
         this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.device));
         this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
         this.eventBus.on('deviceRenamed', (data) => this.onDeviceRenamed(data.device));
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            if (definition.hasOwnProperty('homeassistant')) {
+                mapping[definition.model] = definition.homeassistant;
+            }
+        }
     }
 
     onDeviceRemoved(device) {
+        delete this.discovered[device.ieeeAddr];
         const definition = zigbeeHerdsmanConverters.findByDevice(device);
         if (definition) {
             logger.info(`Clearing Home Assistant discovery topic for '${device.ieeeAddr}'`);
@@ -1684,7 +1888,7 @@ class HomeAssistant extends Extension {
                 this.discoveredTriggers[device.ieeeAddr] = new Set();
             }
 
-            const value = data.payload[key];
+            const value = data.payload[key].toString();
             const discoveredKey = `${key}_${value}`;
 
             if (!this.discoveredTriggers[device.ieeeAddr].has(discoveredKey)) {
@@ -1740,6 +1944,7 @@ class HomeAssistant extends Extension {
 
     async onMQTTConnected() {
         this.mqtt.subscribe(this.statusTopic);
+        this.mqtt.subscribe(defaultStatusTopic);
 
         // MQTT discovery of all paired devices on startup.
         for (const device of this.zigbee.getClients()) {
@@ -1766,7 +1971,7 @@ class HomeAssistant extends Extension {
     discover(resolvedEntity, force=false) {
         // Check if already discoverd and check if there are configs.
         const {device, definition} = resolvedEntity;
-        const deviceSettings = resolvedEntity.settings;
+        const deviceSettings = {...settings.get().device_options, ...resolvedEntity.settings};
         const discover = force || !this.discovered[device.ieeeAddr];
         if (!discover || !device || !definition || !mapping[definition.model] || !deviceSettings ||
             (deviceSettings.hasOwnProperty('homeassistant') && !deviceSettings.homeassistant)) {
@@ -1820,15 +2025,6 @@ class HomeAssistant extends Extension {
                 payload.availability_topic = `${settings.get().mqtt.base_topic}/bridge/state`;
             }
 
-            // Add precision to value_template
-            if (deviceSettings.hasOwnProperty(`${config.object_id}_precision`)) {
-                const precision = deviceSettings[`${config.object_id}_precision`];
-                let template = payload.value_template;
-                template = template.replace('{{ ', '').replace(' }}', '');
-                template = `{{ (${template} | float) | round(${precision}) }}`;
-                payload.value_template = template;
-            }
-
             if (payload.command_topic) {
                 payload.command_topic = `${settings.get().mqtt.base_topic}/${friendlyName}/`;
 
@@ -1941,7 +2137,7 @@ class HomeAssistant extends Extension {
     }
 
     onMQTTMessage(topic, message) {
-        if (topic !== this.statusTopic) {
+        if (topic !== this.statusTopic && topic !== defaultStatusTopic) {
             return false;
         }
 
@@ -1960,7 +2156,7 @@ class HomeAssistant extends Extension {
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
-        if (resolvedEntity && type !== 'deviceLeave') {
+        if (resolvedEntity && type !== 'deviceLeave' && this.mqtt.isConnected()) {
             this.discover(resolvedEntity);
         }
     }

lib/extension/legacy/bridgeLegacy.js

@@ -52,6 +52,7 @@ class BridgeLegacy extends Extension {
             'device_options': this.deviceOptions,
             'add_group': this.addGroup,
             'remove_group': this.removeGroup,
+            'force_remove_group': this.removeGroup,
             'whitelist': this.whitelist,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
         };
@@ -88,7 +89,7 @@ class BridgeLegacy extends Extension {
 
         const entity = settings.getEntity(json.friendly_name);
         assert(entity, `Entity '${json.friendly_name}' does not exist`);
-        settings.changeDeviceOptions(entity.ID, json.options);
+        settings.changeEntityOptions(entity.ID, json.options);
         logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
     }
 
@@ -272,8 +273,14 @@ class BridgeLegacy extends Extension {
         const name = message;
         const entity = this.zigbee.resolveEntity(message);
         assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+
+        if (topic.includes('force')) {
+            entity.group.removeFromDatabase();
+        } else {
+            entity.group.removeFromNetwork();
+        }
         settings.removeGroup(message);
-        entity.group.removeFromDatabase();
+
         this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }

lib/extension/networkMap.js

@@ -1,6 +1,5 @@
 const settings = require('../util/settings');
 const utils = require('../util/utils');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 
@@ -14,15 +13,18 @@ class NetworkMap extends Extension {
         this.legacyApi = settings.get().advanced.legacy_api;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
         this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
 
         // Bind
         this.raw = this.raw.bind(this);
         this.graphviz = this.graphviz.bind(this);
+        this.plantuml = this.plantuml.bind(this);
 
         // Set supported formats
         this.supportedFormats = {
             'raw': this.raw,
             'graphviz': this.graphviz,
+            'plantuml': this.plantuml,
         };
     }
 
@@ -32,6 +34,10 @@ class NetworkMap extends Extension {
             this.mqtt.subscribe(this.legacyTopic);
             this.mqtt.subscribe(this.legacyTopicRoutes);
         }
+
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(this.topic);
+        }
     }
 
     async onMQTTMessage(topic, message) {
@@ -45,6 +51,29 @@ class NetworkMap extends Extension {
                 this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
             }
         }
+
+        if (settings.get().experimental.new_api && topic === this.topic) {
+            try {
+                message = utils.parseJSON(message, message);
+                const type = typeof message === 'object' ? message.type : message;
+                if (!this.supportedFormats.hasOwnProperty(type)) {
+                    throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
+                }
+
+                const routes = typeof message === 'object' && message.routes;
+                const topology = await this.networkScan(routes);
+                const value = this.supportedFormats[type](topology);
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    JSON.stringify(utils.getResponse(message, {routes, type, value}, null)),
+                );
+            } catch (error) {
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    JSON.stringify(utils.getResponse(message, {}, error.message)),
+                );
+            }
+        }
     }
 
     raw(topology) {
@@ -57,43 +86,43 @@ class NetworkMap extends Extension {
         let text = 'digraph G {\nnode[shape=record];\n';
         let style = '';
 
-        topology.nodes.forEach((device) => {
+        topology.nodes.forEach((node) => {
             const labels = [];
 
             // Add friendly name
-            labels.push(`${device.friendlyName}`);
+            labels.push(`${node.friendlyName}`);
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${device.ieeeAddr} (${device.networkAddress})` +
-                ((device.failed && device.failed.length) ? `failed: ${device.failed.join(',')}` : ''),
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
-            if (device.type !== 'Coordinator') {
-                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            if (node.type !== 'Coordinator') {
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
                 if (definition) {
                     labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
-                    labels.push(`${device.manufacturerName} ${device.modelID}`);
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
                 }
             }
 
             // Add the device last_seen timestamp
             let lastSeen = 'unknown';
-            const date = device.type === 'Coordinator' ? Date.now() : device.lastSeen;
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'ISO_8601_local');
+                lastSeen = utils.formatDate(date, 'relative');
             }
 
             labels.push(lastSeen);
 
             // Shape the record according to device type
-            if (device.type == 'Coordinator') {
+            if (node.type == 'Coordinator') {
                 style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
                     `fontcolor="${colors.font.coordinator}"`;
-            } else if (device.type == 'Router') {
+            } else if (node.type == 'Router') {
                 style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
                     `fontcolor="${colors.font.router}"`;
             } else {
@@ -102,22 +131,22 @@ class NetworkMap extends Extension {
             }
 
             // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
 
             /**
              * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.links.filter((e) => (e.source.ieeeAddr === device.ieeeAddr)).forEach((e) => {
-                const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, ' :
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
                 const textRoutes = e.routes.map((r) => r.destinationAddress);
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
-                text += `  "${device.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
                 text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
             });
         });
@@ -127,6 +156,64 @@ class NetworkMap extends Extension {
         return text.replace(/\0/g, '');
     }
 
+    plantuml(topology) {
+        const text = [];
+
+        text.push(`' paste into: https://www.planttext.com/`);
+        text.push(``);
+        text.push('@startuml');
+
+        topology.nodes.sort((a, b) => a.friendlyName.localeCompare(b.friendlyName)).forEach((node) => {
+            // Add friendly name
+            text.push(`card ${node.ieeeAddr} [`);
+            text.push(`${node.friendlyName}`);
+            text.push(`---`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            text.push(
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                text.push(`---`);
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    text.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative');
+            }
+            text.push(`---`);
+            text.push(lastSeen);
+            text.push(`]`);
+            text.push(``);
+        });
+
+        /**
+         * Add edges between the devices
+         * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+         * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+         */
+        topology.links.forEach((link) => {
+            text.push(`${link.sourceIeeeAddr} --> ${link.targetIeeeAddr}: ${link.lqi}`);
+        });
+        text.push('');
+
+        text.push(`@enduml`);
+
+        return text.join(`\n`);
+    }
+
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
         const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');

lib/extension/otaUpdate.js

@@ -1,7 +1,10 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
+const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
+
 const Extension = require('./extension');
 const MINUTES_10 = 1000 * 60 * 10;
 
@@ -19,6 +22,12 @@ class OTAUpdate extends Extension {
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
+        }
     }
 
     async onZigbeeEvent(type, data, resolvedEntity) {
@@ -80,140 +89,147 @@ class OTAUpdate extends Extension {
     }
 
     async onMQTTMessage(topic, message) {
-        let resolvedEntity = null;
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            if (!topic.match(legacyTopicRegex)) {
-                return null;
-            }
-
-            resolvedEntity = this.zigbee.resolveEntity(message);
-        } else {
+        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
             return null;
         }
 
-        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
-        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
-            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
-            logger.error(message);
+        message = utils.parseJSON(message, message);
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+        const resolvedEntity = this.zigbee.resolveEntity(ID);
+        const type = topic.substring(topic.lastIndexOf('/') + 1);
+        const responseData = {id: ID};
+        let error = null;
+
+        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+            error = `Device '${ID}' does not exist`;
+        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
                 const meta = {status: `not_supported`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
+                    JSON.stringify({type: `ota_update`, message: error, meta}),
                 );
             }
+        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else {
+            this.inProgress.add(resolvedEntity.device.ieeeAddr);
 
-            return;
-        }
-
-        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
-            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
-            return;
-        }
-        this.inProgress.add(resolvedEntity.device.ieeeAddr);
-
-        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
-        if (type === 'check') {
-            const message = `Checking if update available for '${resolvedEntity.name}'`;
-            logger.info(message);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
-                );
-            }
-
-            try {
-                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
-                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
-                logger.info(message);
+            if (type === 'check') {
+                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                        JSON.stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
-                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
-                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
-            } catch (error) {
-                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
-                logger.error(message);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `check_failed`, device: resolvedEntity.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                try {
+                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
+                        resolvedEntity.device, logger,
                     );
-                }
-            }
-        } else { // type === 'update'
-            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
-            logger.info(message);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
-                );
-            }
-
-            try {
-                const onProgress = (progress, remaining) => {
-                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
-                    if (remaining) {
-                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
-                    }
-
-                    logger.info(message);
+                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    logger.info(msg);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `ota_update`, message: msg, meta}),
+                        );
+                    }
+
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
+                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    responseData.updateAvailable = available;
+                } catch (e) {
+                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `ota_update`, message: error, meta}),
+                        );
+                    }
+                }
+            } else { // type === 'update'
+                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, msg, meta}),
+                    );
+                }
+
+                try {
+                    const onProgress = (progress, remaining) => {
+                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                        if (remaining) {
+                            msg += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                        }
+
+                        logger.info(msg);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message: msg, meta}));
+                        }
+                    };
+
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
+                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                        (to ? `, from '${fromS}' to '${toS}'` : ``);
+                    logger.info(msg);
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
+                    responseData.to = to ? utils.toSnakeCase(to) : null;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
                         this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
                     }
-                };
+                } catch (e) {
+                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
 
-                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
-                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
-                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
-                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
-                const message = `Finished update of '${resolvedEntity.name}'` +
-                    (to ? `, from '${fromS}' to '${toS}'` : ``);
-                logger.info(message);
-                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
-                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
-                }
-            } catch (error) {
-                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
-                logger.error(message);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_failed`, device: resolvedEntity.name};
-                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message: error, meta}));
+                    }
                 }
             }
+
+            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
         }
 
-        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, JSON.stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        }
     }
 }
 

lib/extension/publish.js

@@ -18,6 +18,7 @@ const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
     zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
 ];
 
 class EntityPublish extends Extension {
@@ -90,11 +91,11 @@ class EntityPublish extends Extension {
             definition = resolvedEntity.definition;
             target = resolvedEntity.endpoint;
             converters = resolvedEntity.definition.toZigbee;
-            options = resolvedEntity.settings;
+            options = {...settings.get().device_options, ...resolvedEntity.settings};
         } else {
             converters = groupConverters;
             target = resolvedEntity.group;
-            options = resolvedEntity.settings;
+            options = {...settings.get().device_options, ...resolvedEntity.settings};
             definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
         }
 
@@ -143,7 +144,7 @@ class EntityPublish extends Extension {
         entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
 
         // For each attribute call the corresponding converter
-        const usedConverters = [];
+        const usedConverters = {};
         for (let [key, value] of entries) {
             let endpointName = topic.endpointName;
             let actualTarget = target;
@@ -157,12 +158,19 @@ class EntityPublish extends Extension {
                     key = key.substring(0, underscoreIndex);
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+
+                    if (!actualTarget) {
+                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
+                        continue;
+                    }
                 }
             }
 
+            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters.includes(converter)) {
+            if (usedConverters[endpointOrGroupID].includes(converter)) {
                 // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
@@ -191,7 +199,7 @@ class EntityPublish extends Extension {
                         const msg = result.state;
 
                         if (endpointName) {
-                            for (const key of ['state', 'brightness']) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                                 if (msg.hasOwnProperty(key)) {
                                     msg[`${key}_${endpointName}`] = msg[key];
                                     delete msg[key];
@@ -238,7 +246,7 @@ class EntityPublish extends Extension {
                 }
             }
 
-            usedConverters.push(converter);
+            usedConverters[endpointOrGroupID].push(converter);
         }
 
         return true;

lib/extension/receive.js

@@ -9,18 +9,33 @@ class Receive extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.elapsed = {};
         this.debouncers = {};
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
     }
 
     async onZigbeeStarted() {
         this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
     }
 
+    async onPublishEntityState(data) {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+            }
+        }
+    }
+
     publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
         if (!this.debouncers[ieeeAddr]) {
             this.debouncers[ieeeAddr] = {
                 payload: {},
                 publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload);
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
                     this.debouncers[ieeeAddr].payload = {};
                 }, time * 1000),
             };
@@ -76,20 +91,20 @@ class Receive extends Extension {
          * As the same message is also received directly from the end device, it makes no sense
          * to handle these messages.
          */
-        const hasGroupID = data.hasOwnProperty('groupID') && data.groupID != 0;
+        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
         if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
             logger.debug('Skipping re-transmitted Xiaomi message');
             return false;
         }
 
-        if (!data.device.modelID && data.device.interviewing) {
-            logger.debug(`Skipping message, modelID is undefined and still interviewing`);
-            return false;
-        }
-
         if (!resolvedEntity.definition) {
-            logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            if (data.device.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
+
             return false;
         }
 
@@ -107,7 +122,7 @@ class Receive extends Extension {
         });
 
         // Check if there is an available converter, genOta messages are not interesting.
-        if (!converters.length && data.cluster !== 'genOta') {
+        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
             logger.debug(
                 `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
                 `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
@@ -146,7 +161,7 @@ class Receive extends Extension {
             }
         };
 
-        const meta = {device: data.device};
+        const meta = {device: data.device, logger};
         let payload = {};
         converters.forEach((converter) => {
             const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};

lib/extension/report.js

@@ -173,6 +173,7 @@ class Report extends Extension {
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
         if (messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
 
+        if (resolvedEntity.device.interviewing === true) return false;
         if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
         if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
         // Gledopto devices don't support reporting.
@@ -186,14 +187,14 @@ class Report extends Extension {
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
             if (this.shouldSetupReporting(resolvedEntity, null)) {
-                this.setupReporting(resolvedEntity);
+                await this.setupReporting(resolvedEntity);
             }
         }
     }
 
-    onZigbeeEvent(type, data, resolvedEntity) {
+    async onZigbeeEvent(type, data, resolvedEntity) {
         if (this.shouldSetupReporting(resolvedEntity, type)) {
-            this.setupReporting(resolvedEntity);
+            await this.setupReporting(resolvedEntity);
         }
 
         if (type === 'message') {

lib/mqtt.js

@@ -95,11 +95,15 @@ class MQTT extends events.EventEmitter {
         this.emit('message', {topic, message: message.toString()});
     }
 
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
+
     async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
-        if (!this.client || this.client.reconnecting) {
+        if (!this.isConnected()) {
             logger.error(`Not connected to MQTT server!`);
             logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
             return;

lib/state.js

@@ -1,5 +1,6 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
+const settings = require('./util/settings');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
 
@@ -53,9 +54,13 @@ class State {
     }
 
     save() {
-        logger.debug(`Saving state to file ${this.file}`);
-        const json = JSON.stringify(this.state, null, 4);
-        fs.writeFileSync(this.file, json, 'utf8');
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = JSON.stringify(this.state, null, 4);
+            fs.writeFileSync(this.file, json, 'utf8');
+        } else {
+            logger.debug(`Not saving state`);
+        }
     }
 
     exists(ID) {

lib/util/logger.js

@@ -44,7 +44,8 @@ const transportsToUse = [
         format: winston.format.combine(
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
+                let {timestamp, level, message} = info;
+                level = level === 'warning' ? 'warn' : level;
                 const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
@@ -61,7 +62,8 @@ const transportFileOptions = {
     format: winston.format.combine(
         winston.format.timestamp({format: timestampFormat}),
         winston.format.printf(/* istanbul ignore next */(info) => {
-            const {timestamp, level, message} = info;
+            let {timestamp, level, message} = info;
+            level = level === 'warning' ? 'warn' : level;
             return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
         }),
     ),
@@ -77,8 +79,16 @@ if (output.includes('file')) {
     transportsToUse.push(new winston.transports.File(transportFileOptions));
 }
 
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
+    transportsToUse.push(new winston.transports.Syslog(options));
+}
+
 // Create logger
-const logger = winston.createLogger({transports: transportsToUse});
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
 
 // Cleanup any old log directory.
 function cleanup() {
@@ -117,4 +127,8 @@ if (output.includes('file')) {
     logger.info(`Logging to console only'`);
 }
 
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
+
 module.exports = logger;

lib/util/settings.js

@@ -8,6 +8,9 @@ const Ajv = require('ajv');
 const ajv = new Ajv({allErrors: true});
 
 const defaults = {
+    passlist: [],
+    blocklist: [],
+    // Deprecated: use block/passlist
     whitelist: [],
     ban: [],
     permit_join: false,
@@ -41,6 +44,7 @@ const defaults = {
     experimental: {
         // json or attribute or attribute_and_json
         output: 'json',
+        new_api: false,
     },
     advanced: {
         legacy_api: true,
@@ -49,16 +53,18 @@ const defaults = {
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
         log_file: 'log.txt',
         log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
         soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
-        baudrate: 115200,
-        rtscts: true,
         adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
+        availability_blocklist: [],
+        availability_passlist: [],
+        // Deprecated, use block/passlist
         availability_blacklist: [],
         availability_whitelist: [],
 
@@ -71,6 +77,8 @@ const defaults = {
          * https://www.zigbee2mqtt.io/configuration/configuration.html
          */
         cache_state: true,
+        cache_state_persistent: true,
+        cache_state_send_on_startup: true,
 
         /**
          * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
@@ -120,6 +128,7 @@ const defaults = {
          */
         timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
+    external_converters: [],
 };
 
 const schema = {
@@ -154,12 +163,16 @@ const schema = {
                 adapter: {type: 'string', enum: ['deconz', 'zstack']},
             },
         },
-        ban: {type: 'array', items: {type: 'string'}},
+        blocklist: {type: 'array', items: {type: 'string'}},
+        passlist: {type: 'array', items: {type: 'string'}},
+        // Deprecated: use block/passlist
         whitelist: {type: 'array', items: {type: 'string'}},
+        ban: {type: 'array', items: {type: 'string'}},
         experimental: {
             type: 'object',
             properties: {
                 transmit_power: {type: 'number'},
+                output: {type: 'string', enum: ['attribute_and_json', 'attribute', 'json']},
             },
         },
         advanced: {
@@ -170,6 +183,8 @@ const schema = {
                 ext_pan_id: {type: 'array', items: {type: 'number'}},
                 channel: {type: 'number', minimum: 11, maximum: 26},
                 cache_state: {type: 'boolean'},
+                cache_state_persistent: {type: 'boolean'},
+                cache_state_send_on_startup: {type: 'boolean'},
                 log_rotation: {type: 'boolean'},
                 log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
                 log_output: {type: 'array', items: {type: 'string'}},
@@ -182,6 +197,9 @@ const schema = {
                 last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
                 elapsed: {type: 'boolean'},
                 availability_timeout: {type: 'number', minimum: 0},
+                availability_blocklist: {type: 'array', items: {type: 'string'}},
+                availability_passlist: {type: 'array', items: {type: 'string'}},
+                // Deprecated, use block/passlist
                 availability_blacklist: {type: 'array', items: {type: 'string'}},
                 availability_whitelist: {type: 'array', items: {type: 'string'}},
                 report: {type: 'boolean'},
@@ -269,6 +287,10 @@ const schema = {
             },
         },
     },
+    external_converters: {
+        type: 'array',
+        items: {type: 'string'},
+    },
     required: ['homeassistant', 'permit_join', 'mqtt'],
 };
 
@@ -354,6 +376,8 @@ function validate() {
 
     checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
     checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blocklist, 'availability_blocklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_passlist, 'availability_passlist');
 
     return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
 }
@@ -527,6 +551,7 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
+// Legacy: can be removed after bridgeLegacy has been removed
 function whitelistDevice(ID) {
     const settings = get();
     if (!settings.whitelist) {
@@ -541,6 +566,16 @@ function whitelistDevice(ID) {
     write();
 }
 
+function blockDevice(ID) {
+    const settings = get();
+    if (!settings.blocklist) {
+        settings.blocklist = [];
+    }
+
+    settings.blocklist.push(ID);
+    write();
+}
+
 function banDevice(ID) {
     const settings = get();
     if (!settings.ban) {
@@ -636,10 +671,18 @@ function removeGroup(groupIDorName) {
     write();
 }
 
-function changeDeviceOptions(IDorName, newOptions) {
-    const device = getDeviceThrowIfNotExists(IDorName);
+function changeEntityOptions(IDorName, newOptions) {
     const settings = get();
-    objectAssignDeep(settings.devices[device.ID], newOptions);
+    delete newOptions.friendly_name;
+    delete newOptions.devices;
+    if (getDevice(IDorName)) {
+        objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+    } else if (getGroup(IDorName)) {
+        objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
     write();
 }
 
@@ -670,13 +713,14 @@ module.exports = {
     getEntity,
     whitelistDevice,
     banDevice,
+    blockDevice,
     addDevice,
     removeDevice,
     addGroup,
     removeGroup,
     addDeviceToGroup,
     removeDeviceFromGroup,
-    changeDeviceOptions,
+    changeEntityOptions,
     changeFriendlyName,
 
     // For tests only

lib/util/utils.js

@@ -1,5 +1,7 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const equals = require('fast-deep-equal');
+const equals = require('fast-deep-equal/es6');
+const humanizeDuration = require('humanize-duration');
+const data = require('./data');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -29,9 +31,13 @@ function toLocalISOString(dDate) {
 
 const endpointNames = [
     'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
     'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
     'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
 function flatten(arr) {
@@ -40,6 +46,10 @@ function flatten(arr) {
     }, []);
 }
 
+function capitalize(s) {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
 const forceEndDevice = flatten(
     ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
         .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
@@ -83,6 +93,10 @@ function formatDate(date, type) {
     case 'epoch':
         result = date;
         break;
+    case 'relative':
+        // https://github.com/EvanHahn/HumanizeDuration.js#options
+        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
+        break;
     default:
         throw new Error(`Unsupported type '${type}'`);
     }
@@ -120,11 +134,62 @@ function getObjectsProperty(objects, key, defaultValue) {
     return defaultValue;
 }
 
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+function parseJSON(value, failedReturnValue) {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+function* getExternalConvertersDefinitions(settings) {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converterModule = moduleName;
+
+        if (moduleName.endsWith('.js')) {
+            converterModule = data.joinPath(moduleName.split('.')[0]);
+        }
+
+        const converter = require(converterModule);
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+function toSnakeCase(value) {
+    if (typeof value === 'object') {
+        for (const key of Object.keys(value)) {
+            value[toSnakeCase(key)] = value[key];
+            delete value[key];
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
     objectHasProperties,
+    toSnakeCase,
     getObjectsProperty,
     getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
@@ -136,4 +201,8 @@ module.exports = {
     isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     formatDate: (date, type) => formatDate(date, type),
     equalsPartial,
+    getResponse,
+    capitalize,
+    parseJSON,
+    getExternalConvertersDefinitions,
 };

lib/util/yaml.js

@@ -1,6 +1,6 @@
 const yaml = require('js-yaml');
 const fs = require('fs');
-const equals = require('fast-deep-equal');
+const equals = require('fast-deep-equal/es6');
 
 function read(file) {
     try {

lib/zigbee.js

@@ -71,14 +71,16 @@ class Zigbee extends events.EventEmitter {
         logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
 
         for (const device of this.getClients()) {
-            // If a whitelist is used, all other device will be removed from the network.
-            if (settings.get().whitelist.length > 0) {
-                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
-                    logger.warn(`Blacklisted device is connected (${device.ieeeAddr}), removing...`);
+            // If a passlist is used, all other device will be removed from the network.
+            const passlist = settings.get().passlist.concat(settings.get().whitelist);
+            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            if (passlist.length > 0) {
+                if (!passlist.includes(device.ieeeAddr)) {
+                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
                     device.removeFromNetwork();
                 }
-            } else if (settings.get().ban.includes(device.ieeeAddr)) {
-                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+            } else if (blocklist.includes(device.ieeeAddr)) {
+                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
                 device.removeFromNetwork();
             }
         }
@@ -151,7 +153,8 @@ class Zigbee extends events.EventEmitter {
     resolveEntity(key) {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device', `Wrong type '${typeof key}'`,
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
         );
 
         if (typeof key === 'string' || typeof key === 'number') {
@@ -215,7 +218,7 @@ class Zigbee extends events.EventEmitter {
                 if (!group) group = this.createGroup(entity.ID);
                 return {type: 'group', group, settings: entity, name: entity.friendlyName};
             }
-        } else {
+        } else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
             return {
                 type: 'device',
@@ -225,6 +228,14 @@ class Zigbee extends events.EventEmitter {
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
                 definition: zigbeeHerdsmanConverters.findByDevice(key),
             };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: setting,
+                name: setting ? setting.friendlyName : key.groupID,
+            };
         }
     }
 
@@ -241,21 +252,23 @@ class Zigbee extends events.EventEmitter {
     }
 
     acceptJoiningDeviceHandler(ieeeAddr) {
-        // If set whitelist devices, all other device will be rejected to join the network
-        if (settings.get().whitelist.length > 0) {
-            if (settings.get().whitelist.includes(ieeeAddr)) {
-                logger.info(`Accepting joining whitelisted device '${ieeeAddr}'`);
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
                 return true;
             } else {
-                logger.info(`Rejecting joining non-whitelisted device '${ieeeAddr}'`);
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
                 return false;
             }
-        } else if (settings.get().ban.length > 0) {
-            if (settings.get().ban.includes(ieeeAddr)) {
-                logger.info(`Rejecting joining banned device '${ieeeAddr}'`);
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
                 return false;
             } else {
-                logger.info(`Accepting joining non-banned device '${ieeeAddr}'`);
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
                 return true;
             }
         } else {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.13.0",
+  "version": "1.14.2",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13"
+    "node": "^10 || ^12 || ^13 || ^14"
   },
   "keywords": [
     "xiaomi",
@@ -33,20 +33,22 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "*",
-    "debounce": "*",
-    "fast-deep-equal": "*",
-    "git-last-commit": "*",
-    "js-yaml": "*",
-    "mkdir-recursive": "*",
-    "moment": "*",
-    "mqtt": "*",
-    "object-assign-deep": "*",
-    "rimraf": "*",
-    "semver": "*",
-    "winston": "*",
-    "zigbee-herdsman": "0.12.83",
-    "zigbee-herdsman-converters": "12.0.81"
+    "ajv": "^6.12.3",
+    "debounce": "^1.2.0",
+    "fast-deep-equal": "^3.1.3",
+    "git-last-commit": "^1.0.0",
+    "humanize-duration": "^3.23.1",
+    "js-yaml": "^3.14.0",
+    "mkdir-recursive": "^0.4.0",
+    "moment": "^2.27.0",
+    "mqtt": "^4.1.0",
+    "object-assign-deep": "^0.4.0",
+    "rimraf": "^3.0.2",
+    "semver": "^7.3.2",
+    "winston": "^3.3.3",
+    "winston-syslog": "^2.4.4",
+    "zigbee-herdsman": "0.12.108",
+    "zigbee-herdsman-converters": "12.0.147"
   },
   "devDependencies": {
     "eslint": "*",

test/assets/mock-external-converter-multiple.js

@@ -0,0 +1,20 @@
+const homeassistantSwitch = {
+    type: 'switch',
+    object_id: 'switch',
+    discovery_payload: {
+        payload_off: 'OFF',
+        payload_on: 'ON',
+        value_template: '{{ value_json.state }}',
+        command_topic: true,
+    },
+};
+
+const mockDevices = [{
+    mock: 1,
+    model: 'external_converters_device',
+    homeassistant: [homeassistantSwitch],
+}, {
+    mock: 2
+}];
+
+module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -0,0 +1,5 @@
+const mockDevice = {
+    mock: true
+};
+
+module.exports = mockDevice;

test/availability.test.js

@@ -197,7 +197,21 @@ describe('Availability', () => {
         expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
     });
 
-    it('Should not ping devices blacklisted by friendly name', async () => {
+    it('Should not ping devices on blocklist by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
         await controller.stop();
@@ -211,9 +225,9 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not ping devices blacklisted by IEEE address', async () => {
+    it('Should not ping devices on blocklist by IEEE address', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
+        settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -238,7 +252,21 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
         await controller.stop();
@@ -252,9 +280,9 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
+        settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -266,10 +294,10 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should not ping non-whitelisted devices if availability_whitelist is set', async () => {
+    it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         getExtension().state[device.ieeeAddr] = false;
-        settings.set(['advanced', 'availability_whitelist'], ['0x000b57fffec6a5b3'])
+        settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -352,4 +380,21 @@ describe('Availability', () => {
 
         device.lastSeen = defaultLastSeen;
     });
+
+    it('Should republish existing state on MQTT connected', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        await controller.stop();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        getExtension().state[device.ieeeAddr] = false;
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
 });

test/bind.test.js

@@ -14,6 +14,7 @@ describe('Bind', () => {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
             endpoint.configureReporting.mockClear();
+            endpoint.bind = jest.fn();
             endpoint.bind.mockClear();
             endpoint.unbind.mockClear();
         }
@@ -23,6 +24,7 @@ describe('Bind', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -31,10 +33,212 @@ describe('Bind', () => {
     });
 
     it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
     });
 
     it('Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'button'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: 'Coordinator'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            JSON.stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: '1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementation(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote/ep2', to: 'wall_switch'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: target}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            JSON.stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when source not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote_not_existing', to: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when target not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should bind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -54,7 +258,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should log error when there is nothing to bind', async () => {
+    it('Legacy api: Should log error when there is nothing to bind', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
@@ -65,7 +269,7 @@ describe('Bind', () => {
         expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
     });
 
-    it('Should unbind', async () => {
+    it('Legacy api: Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -85,7 +289,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should unbind coordinator', async () => {
+    it('Legacy api: Should unbind coordinator', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -106,7 +310,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should bind to groups', async () => {
+    it('Legacy api: Should bind to groups', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
@@ -126,7 +330,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should bind to group by number', async () => {
+    it('Legacy api: Should bind to group by number', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
@@ -146,7 +350,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should log when bind fails', async () => {
+    it('Legacy api: Should log when bind fails', async () => {
         logger.error.mockClear();
         const device = zigbeeHerdsman.devices.remote;
         const endpoint = device.getEndpoint(1);
@@ -154,12 +358,11 @@ describe('Bind', () => {
         endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
         MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
     });
 
-    it('Should bind from non default endpoints', async () => {
+    it('Legacy api: Should bind from non default endpoints', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
         const endpoint = device.getEndpoint(2);
@@ -170,7 +373,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
 
-    it('Should bind to default endpoint returned by endpoints()', async () => {
+    it('Legacy api: Should bind to default endpoint returned by endpoints()', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
         const endpoint = device.getEndpoint(2);
@@ -181,7 +384,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
 
-    it('Should unbind from default_bind_group', async () => {
+    it('Legacy api: Should unbind from default_bind_group', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = 'default_bind_group';
         const endpoint = device.getEndpoint(1);

test/bridge.test.js

@@ -0,0 +1,605 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const {coordinator, bulb, unsupported} = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported.ieeeAddr);
+
+describe('Bridge', () => {
+    let controller;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        settings.set(['advanced', 'legacy_api'], false);
+        settings.set(['experimental', 'new_api'], true);
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb;
+        device.removeFromDatabase.mockClear();
+        device.removeFromNetwork.mockClear();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+    });
+
+    it('Should publish bridge info on startup', async () => {
+        const version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/info',
+          JSON.stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"log_level":"info","permit_join":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          JSON.stringify([{"ieee_address":"0x000b57fffec6a5b2","type":"Router","network_address":40369,"supported":true,"friendly_name":"bulb","definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"},"power_source":"Mains (single phase)","date_code":null,"interviewing":false,"interview_completed":true},{"ieee_address":"0x0017880104e45518","type":"EndDevice","network_address":6536,"supported":false,"friendly_name":"0x0017880104e45518","definition":null,"power_source":"Battery","date_code":null,"interviewing":false,"interview_completed":true}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/groups',
+          JSON.stringify([{"id":1,"friendly_name":"group_1","members":[]},{"id":15071,"friendly_name":"group_tradfri_remote","members":[]},{"id":99,"friendly_name":99,"members":[]},{"id":11,"friendly_name":"group_with_tradfri","members":[]},{"id":2,"friendly_name":"group_2","members":[]}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device joined', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_joined","data":{"friendly_name":"bulb","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device interview started', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'started'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"started","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview failed', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'failed'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"failed","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview successful', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'successful'});
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.unsupported, status: 'successful'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"successful","ieee_address":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+          'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"device_interview","data":{"friendly_name":"0x0017880104e45518","status":"successful","ieee_address":"0x0017880104e45518","supported":false,"definition":null}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device leaves', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: zigbeeHerdsman.devices.bulb.ieeeAddr});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_leave","data":{"ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put transaction in response when request is done with transaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value": false, "transaction": 22}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when request fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error('Failed to connect to adapter')});
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when format is incorrect', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value_not_good": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{},"status":"error","error": "No value given"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Coverage satisfaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/random', JSON.stringify({"value": false}));
+        const device = zigbeeHerdsman.devices.bulb;
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+    });
+
+    it('Should allow a healthcheck', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/health_check', '');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/health_check',
+            JSON.stringify({"data":{"healthy": true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to remove device by string', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb');
+        await flushPromises();
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(device.removeFromDatabase).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":false,"force":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().blocklist).toStrictEqual([]);
+    });
+
+    it('Should allow to remove device by object ID', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb"}));
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(device.removeFromDatabase).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":false,"force":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb", force: true}));
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(device.removeFromNetwork).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":false,"force":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to block device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb", block: true, force: true}));
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":true,"force":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().blocklist).toStrictEqual(["0x000b57fffec6a5b2"]);
+    });
+
+    it('Should allow to remove group', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/remove', 'group_1');
+        await flushPromises();
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/remove',
+            JSON.stringify({"data":{"id": "group_1", "force": false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove group', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/remove', JSON.stringify({id: "group_1", force: true}));
+        await flushPromises();
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/remove',
+            JSON.stringify({"data":{"id": "group_1", "force": true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on removing non-existing device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "non-existing-device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{},"status":"error","error":"Device 'non-existing-device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when remove device fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('device timeout')})
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{},"status":"error","error":"Failed to remove device 'bulb' (block: false, force: false) (Error: device timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow rename device', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({from: 'bulb', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(settings.getDevice('bulb_new_name')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb_new_name", "friendlyName": "bulb_new_name", "retain": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{"from":"bulb","to":"bulb_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow rename group', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/rename', JSON.stringify({from: 'group_1', to: 'group_new_name'}));
+        await flushPromises();
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(settings.getGroup('group_new_name')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_new_name", "friendlyName": "group_new_name", "optimistic": true, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/rename',
+            JSON.stringify({"data":{"from":"group_1","to":"group_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on invalid device rename payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({from_bla: 'bulb', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on non-existing device rename', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({from: 'bulb_not_existing', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{},"status":"error","error":"Device 'bulb_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to rename last joined device', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({last: true, to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(settings.getDevice('bulb_new_name')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb_new_name", "friendlyName": "bulb_new_name", "retain": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{"from":"bulb","to":"bulb_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when renaming last joined device but none has joined', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({last: true, to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{},"status":"error","error":"No device has joined since start"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow change device options', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getDevice('bulb')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb", "friendlyName": "bulb", "retain": true});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/options', JSON.stringify({options: {retain: false, transition: 1}, id: 'bulb'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb", "friendlyName": "bulb", "retain": false, "transition": 1});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            JSON.stringify({"data":{"from":{"retain": true},"to":{"retain": false,"transition":1}, "id":"bulb"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow change group options', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup('group_1')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_1", "retain": false, "friendlyName": "group_1", "optimistic": true});
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/options', JSON.stringify({options: {retain: true, transition: 1}, id: 'group_1'}));
+        await flushPromises();
+        expect(settings.getGroup('group_1')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_1", "retain": true, "friendlyName": "group_1", "optimistic": true, "transition": 1});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/options',
+            JSON.stringify({"data":{"from":{"optimistic": true,"retain": false},"to":{"optimistic": true,"retain": true,"transition":1}, "id":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on invalid device change options payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/options', JSON.stringify({options_: {retain: true, transition: 1}, id: 'bulb'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            JSON.stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to add group by string', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', 'group_193');
+        await flushPromises();
+        expect(settings.getGroup('group_193')).toStrictEqual({"ID": 3, "devices": [], "friendly_name": "group_193", "friendlyName": "group_193", "optimistic": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            JSON.stringify({"data":{"friendly_name":"group_193","id": 3},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to add group with ID', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', JSON.stringify({friendly_name: "group_193", id: 9}));
+        await flushPromises();
+        expect(settings.getGroup('group_193')).toStrictEqual({"ID": 9, "devices": [], "friendly_name": "group_193", "friendlyName": "group_193", "optimistic": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            JSON.stringify({"data":{"friendly_name":"group_193","id": 9},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when add with invalid payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', JSON.stringify({friendly_name9: "group_193"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            JSON.stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set last_seen', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/last_seen', 'ISO_8601');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('ISO_8601');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/last_seen',
+            JSON.stringify({"data":{"value":"ISO_8601"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set last_seen when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/last_seen', 'invalid_one');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/last_seen',
+            JSON.stringify({"data":{},"status":"error","error":"'invalid_one' is not an allowed value, allowed: disable,ISO_8601,epoch,ISO_8601_local"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set elapsed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/elapsed', 'true');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(true);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/elapsed',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set last_seen when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/elapsed', 'not_valid');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/elapsed',
+            JSON.stringify({"data":{},"status":"error","error":"'not_valid' is not an allowed value, allowed: true,false"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set log level', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/log_level', 'debug');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('debug');
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/log_level',
+            JSON.stringify({"data":{"value":'debug'},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set log level when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/log_level', 'not_valid');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/log_level',
+            JSON.stringify({"data":{},"status":"error","error":"'not_valid' is not an allowed value, allowed: error,warn,info,debug"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to touchlink factory reset (succeeds)', async () => {
+        MQTT.publish.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/request/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/touchlink/factory_reset',
+            JSON.stringify({"data":{},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to touchlink factory reset (fails)', async () => {
+        MQTT.publish.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/request/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/touchlink/factory_reset',
+            JSON.stringify({"data":{},"status":"error","error":"Failed to factory reset device through Touchlink"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+});

test/configure.test.js

@@ -90,8 +90,52 @@ describe('Configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Should allow to reconfigure manually', async () => {
+    it('Should allow to configure via MQTT', async () => {
         expectRemoteConfigured();
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "remote"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "not_existing_device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when configure fails', async () => {
+        zigbeeHerdsman.devices.remote.getEndpoint(1).bind.mockImplementationOnce(async () => {throw new Error('Bind timeout after 10s')});
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "remote"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device has no configure', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "bulb", transaction: 20}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should allow to reconfigure manually', async () => {
         mockClear(zigbeeHerdsman.devices.remote);
         expectRemoteNotConfigured();
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
@@ -99,13 +143,13 @@ describe('Configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Shouldnt manually reconfigure when device does not exist', async () => {
+    it('Legacy api: Shouldnt manually reconfigure when device does not exist', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
         await flushPromises();
         expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
     });
 
-    it('Should skip reconfigure when device does not require this', async () => {
+    it('Legacy api: Should skip reconfigure when device does not require this', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
         await flushPromises();
         expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)

test/controller.test.js

@@ -30,7 +30,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -46,6 +46,13 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
     });
 
+    it('Start controller when permit join fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error("failed!")});
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+    });
+
     it('Start controller with specific MQTT settings', async () => {
         const ca = tmp.fileSync().name;
         fs.writeFileSync(ca, "ca");
@@ -101,6 +108,15 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
     });
 
+    it('Start controller should not publish cached states when disabled', async () => {
+        settings.set(['advanced', 'cache_state_send_on_startup'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        const publishedTopics = MQTT.publish.mock.calls.map(m => m[0]);
+        expect(publishedTopics).toEqual(expect.not.arrayContaining(["zigbee2mqtt/bulb", "zigbee2mqtt/remote"]));
+    });
+
     it('Start controller should not publish cached states when cache_state is false', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeDefaultState();
@@ -143,6 +159,14 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
+    it('Should remove device not on passlist on startup', async () => {
+        settings.set(['passlist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
     it('Should remove non whitelisted devices on startup', async () => {
         settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -151,6 +175,14 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
     });
 
+    it('Should remove device on blocklist on startup', async () => {
+        settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
     it('Should remove banned devices on startup', async () => {
         settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -276,48 +308,48 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('acceptJoiningDeviceHandler reject banned device', async () => {
+    it('acceptJoiningDeviceHandler reject device on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler accept not banned device', async () => {
+    it('acceptJoiningDeviceHandler accept device not on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], ['123']);
+        settings.set(['blocklist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler accept device on passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler reject non-whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler reject device not in passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], ['123']);
+        settings.set(['passlist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler should prefer whitelist above ban', async () => {
+    it('acceptJoiningDeviceHandler should prefer passlist above blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept when no ban and whitelist', async () => {
+    it('acceptJoiningDeviceHandler accept when not on blocklist and passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
@@ -510,6 +542,21 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should allow to disable state persistency', async () => {
+        settings.set(['advanced', 'cache_state_persistent'], false);
+        data.removeState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", JSON.stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", JSON.stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
+        await controller.stop();
+        expect(data.stateExists()).toBeFalsy();
+    });
+
     it('Publish should not cache when set', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeEmptyState();

test/externalConverters.test.js

@@ -0,0 +1,115 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const path = require('path');
+const fs = require('fs');
+
+zigbeeHerdsmanConverters.addDeviceDefinition = jest.fn();
+
+const mocksClear = [zigbeeHerdsmanConverters.addDeviceDefinition, zigbeeHerdsman.permitJoin,
+    mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+jest.mock(
+    'mock-external-converter-module', () => {
+        return {
+            mock: true
+        };
+    }, {
+        virtual: true
+    });
+
+jest.mock(
+    'mock-multiple-external-converter-module', () => {
+        return [{
+            mock: 1
+        }, {
+            mock: 2
+        }];
+    }, {
+        virtual: true
+    });
+
+describe('Loads external converters', () => {
+    let controller;
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        await flushPromises();
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        data.writeEmptyState();
+        settings._reRead();
+    });
+
+    it('Does not load external converters', async () => {
+        settings.set(['external_converters'], []);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
+    });
+
+    it('Loads external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter.js']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1,
+            model: 'external_converters_device',
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+
+    it('Loads external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-external-converter-module']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+});

test/group.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
@@ -15,11 +16,13 @@ describe('Groups', () => {
 
     beforeEach(() => {
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
         settings._reRead();
         MQTT.publish.mockClear();
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     })
 
     it('Apply group updates add', async () => {
@@ -79,7 +82,7 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
 
-    it('Add to group via MQTT', async () => {
+    it('Legacy api: Add to group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -94,7 +97,7 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Add to group with slashes via MQTT', async () => {
+    it('Legacy api: Add to group with slashes via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups["group/with/slashes"];
@@ -115,7 +118,7 @@ describe('Groups', () => {
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
     });
 
-    it('Add to group via MQTT with postfix', async () => {
+    it('Legacy api: Add to group via MQTT with postfix', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
@@ -128,7 +131,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+    it('Legacy api: Add to group via MQTT with postfix shouldnt add it twice', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
@@ -143,7 +146,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Remove from group via MQTT', async () => {
+    it('Legacy api: Remove from group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -158,7 +161,7 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+    it('Legacy api: Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -172,7 +175,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
     });
 
-    it('Remove from group via MQTT with postfix variant 1', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 1', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -186,7 +189,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 2', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 2', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -200,7 +203,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 3', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 3', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -214,7 +217,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group all', async () => {
+    it('Legacy api: Remove from group all', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -243,7 +246,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Log when adding to non-existing group', async () => {
+    it('Legacy api: Log when adding to non-existing group', async () => {
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -252,7 +255,7 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
     });
 
-    it('Log when adding to non-existing device', async () => {
+    it('Legacy api: Log when adding to non-existing device', async () => {
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -375,7 +378,7 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -400,8 +403,8 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -426,9 +429,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF"}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -454,7 +457,269 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group/with/slashes', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'wall_switch_double/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', JSON.stringify({device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove_all',
+            JSON.stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/homeassistant.test.js

@@ -5,6 +5,9 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const flushPromises = () => new Promise(setImmediate);
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
+const fs = require('fs');
+const path = require('path');
+const HomeAssistant = require('../lib/extension/homeassistant');
 
 describe('HomeAssistant extension', () => {
     beforeEach(async () => {
@@ -20,7 +23,6 @@ describe('HomeAssistant extension', () => {
 
     it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
         const missing = [];
-        const HomeAssistant = require('../lib/extension/homeassistant');
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
 
         require('zigbee-herdsman-converters').devices.forEach((d) => {
@@ -34,7 +36,6 @@ describe('HomeAssistant extension', () => {
 
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
-        const HomeAssistant = require('../lib/extension/homeassistant');
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
 
         require('zigbee-herdsman-converters').devices.forEach((d) => {
@@ -204,7 +205,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
-            'value_template': "{{ (value_json.temperature | float) | round(1) }}",
+            'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_temperature',
@@ -229,7 +230,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
-            'value_template': '{{ (value_json.humidity | float) | round(0) }}',
+            'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_humidity',
@@ -254,7 +255,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
-            'value_template': '{{ (value_json.pressure | float) | round(2) }}',
+            'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_pressure',
@@ -572,12 +573,39 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
+    it('Should send all status when home assistant comes online (default topic)', async () => {
+        jest.useFakeTimers();
+        data.writeDefaultState();
+        controller = new Controller();
+        await controller.start();
+        expect(MQTT.subscribe).toHaveBeenCalledWith('homeassistant/status');
+        await flushPromises();
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/status', 'online');
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote',
+            '{"brightness":255,"update_available":false}',
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
     it('Should send all status when home assistant comes online', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
         controller = new Controller();
         await controller.start();
         await flushPromises();
+        expect(MQTT.subscribe).toHaveBeenCalledWith('hass/status');
         MQTT.publish.mockClear();
         await MQTT.events.message('hass/status', 'online');
         await flushPromises();
@@ -960,4 +988,28 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/click_single/config');
         expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/click');
     });
+
+    it('Load Home Assistant mapping from external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const beforeCount = Object.entries((new HomeAssistant(null, null, null, null, {on: () => {}}))._getMapping()).length;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller();
+        const ha = controller.extensions.find((e) => e.constructor.name === 'HomeAssistant');
+        await controller.start();
+        await flushPromises();
+        const afterCount = Object.entries(ha._getMapping()).length;
+        expect(beforeCount + 1).toStrictEqual(afterCount);
+
+        const homeassistantSwitch = {
+            type: 'switch',
+            object_id: 'switch',
+            discovery_payload: {
+                payload_off: 'OFF',
+                payload_on: 'ON',
+                value_template: '{{ value_json.state }}',
+                command_topic: true,
+            },
+        };
+        expect(ha._getMapping()['external_converters_device']).toStrictEqual([homeassistantSwitch]);
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -318,6 +318,20 @@ describe('Bridge legacy', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
         await flushPromises();
         expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
         expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',

test/networkMap.test.js

@@ -26,6 +26,7 @@ describe('Networkmap', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
@@ -75,7 +76,7 @@ describe('Networkmap', () => {
         unsupported_router.routingTable = () => {throw new Error('failed')};
     }
 
-    it('Output raw networkmap', async () => {
+    it('Output raw networkmap legacy api', async () => {
         mock();
         MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'raw');
         await flushPromises();
@@ -126,11 +127,7 @@ describe('Networkmap', () => {
         expect(JSON.parse(call[1])).toStrictEqual(expected);
     });
 
-    it('Output graphviz networkmap', async () => {
-        const getTimezoneOffset = Date.prototype.getTimezoneOffset;
-        const getHours = Date.prototype.getHours;
-        Date.prototype.getTimezoneOffset = () => -60;
-        Date.prototype.getHours = () => 1;
+    it('Output graphviz networkmap legacy api', async () => {
         mock();
         const device = zigbeeHerdsman.devices.bulb_color;
         device.lastSeen = null;
@@ -146,16 +143,16 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|1970-01-01T01:00:10+01:00}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|1970-01-01T01:00:01+01:00}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
               "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
               "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|1970-01-01T01:00:01+01:00}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
             }`;
 
@@ -165,7 +162,149 @@ describe('Networkmap', () => {
         for (let i = 0; i < expectedLines.length; i++) {
             expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
         }
-        Date.prototype.getTimezoneOffset = getTimezoneOffset;
-        Date.prototype.getHours = getHours;
+    });
+
+    it('Output plantuml networkmap legacy api', async () => {
+        mock();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = null;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'plantuml');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/plantuml');
+
+        const expected = `' paste into: https://www.planttext.com/
+
+        @startuml
+        card 0x0017880104e45525 [
+        0x0017880104e45525
+        ---
+        0x0017880104e45525 (6536) failed: lqi,routingTable
+        ---
+        Boef notSupportedModelID
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b2 [
+        bulb
+        ---
+        0x000b57fffec6a5b2 (40369)
+        ---
+        IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b3 [
+        bulb_color
+        ---
+        0x000b57fffec6a5b3 (40399)
+        ---
+        Philips Hue Go (7146060PH)
+        ---
+        unknown
+        ]
+
+        card 0x0017880104e45521 [
+        button_double_key
+        ---
+        0x0017880104e45521 (6538)
+        ---
+        Xiaomi Aqara double key wireless wall switch (WXKG02LM)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x0017880104e45559 [
+        cc2530_router
+        ---
+        0x0017880104e45559 (6540)
+        ---
+        Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x00124b00120144ae [
+        Coordinator
+        ---
+        0x00124b00120144ae (0)
+        ---
+        0 seconds ago
+        ]
+
+        0x000b57fffec6a5b3 --> 0x00124b00120144ae: 120
+        0x000b57fffec6a5b2 --> 0x00124b00120144ae: 92
+        0x000b57fffec6a5b3 --> 0x000b57fffec6a5b2: 110
+        0x0017880104e45559 --> 0x000b57fffec6a5b2: 100
+        0x0017880104e45521 --> 0x0017880104e45559: 130
+
+        @enduml`;
+
+        const expectedLines = expected.split('\n');
+        const actualLines = call[1].split('\n');
+
+        for (let i = 0; i < expectedLines.length; i++) {
+            expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
+        }
+    });
+
+    it('Should output raw networkmap', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', JSON.stringify({type: 'raw', routes: true}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
+
+        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
+
+        const expected = {
+            status: 'ok',
+            data: {
+                type: 'raw',
+                routes: true,
+                value: {
+                    "nodes":[
+                        {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
+                        {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
+                        {"lastSeen": null,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
+                        {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
+                        {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
+                        {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
+                    ],
+                    "links":[
+                        {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
+                        {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
+                        {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
+                        {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
+                        {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
+                    ]
+                }
+            }
+        };
+
+        const actual = JSON.parse(call[1]);
+        actual.data.value = JSON.parse(actual.data.value);
+        expect(actual).toStrictEqual(expected);
+    });
+
+    it('Should throw error when rquesting invalid type', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', 'not_existing');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/networkmap',
+            JSON.stringify({"data":{},"status":"error","error":"Type 'not_existing' not supported, allowed are: raw,graphviz,plantuml"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/otaUpdate.test.js

@@ -18,6 +18,7 @@ describe('OTA update', () => {
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings._reRead();
+        settings.set(['experimental', 'new_api'], true);
         data.writeEmptyState();
         controller = new Controller();
         await controller.start();
@@ -28,6 +29,8 @@ describe('OTA update', () => {
     it('Should subscribe to topics', async () => {
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/check');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/update');
     });
 
     it('Should OTA update a device', async () => {
@@ -41,14 +44,13 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
-        logger.error.mockClear();
         device.save.mockClear();
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             onUpdate(0, null);
             onUpdate(10, 3600);
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', 'bulb');
         await flushPromises();
         expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
@@ -57,10 +59,14 @@ describe('OTA update', () => {
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
-        expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');
         expect(device.softwareBuildID).toBe(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            JSON.stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should handle when OTA update fails', async () => {
@@ -69,17 +75,18 @@ describe('OTA update', () => {
         endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.info.mockClear();
-        logger.error.mockClear();
         device.save.mockClear();
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             throw new Error('Update failed');
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', JSON.stringify({id: "bulb"}));
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error":"Update of 'bulb' failed (Update failed)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should be able to check if OTA update is available', async () => {
@@ -87,41 +94,65 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb","updateAvailable":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
 
-        logger.info.mockClear();
+        MQTT.publish.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb","updateAvailable":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should handle if OTA update check fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.error.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": `Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail when device does not exist', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "not_existing_deviceooo");
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "not_existing_deviceooo"},"status":"error","error": `Device 'not_existing_deviceooo' does not exist`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should not check for OTA when device does not support it', async () => {
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "ZNLDP12LM");
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "ZNLDP12LM"},"status":"error","error": `Device 'ZNLDP12LM' does not support OTA updates`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should refuse to check/update when already in progress', async () => {
@@ -130,36 +161,38 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
             return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
         });
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
         jest.runAllTimers();
         await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": `Update or check for update already in progress for 'bulb'`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
-    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+    it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         let count = 0;
-        endpoint.read.mockImplementation(() => {
-            if (count === 1) throw new Error('Failed!')
-            count++;
-            return {swBuildId: 1, dateCode: '2019010'}
-        });
+        endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
 
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.info.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', "bulb");
         await flushPromises();
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            JSON.stringify({"data":{"id":"bulb","from":null,"to":null},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should check for update when device requests it', async () => {
@@ -212,4 +245,116 @@ describe('OTA update', () => {
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
+
+    it('Legacy api: Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+    });
+
+    it('Legacy api: Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+    });
+
+    it('Legacy api: Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+    });
+
+    it('Legacy api: Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.error.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+    });
+
+    it('Legacy api: Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+    });
+
+    it('Legacy api: Shouldnt crash when read modelID after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            if (count === 1) throw new Error('Failed!')
+            count++;
+            return {swBuildId: 1, dateCode: '2019010'}
+        });
+
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+    });
 });

test/publish.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -48,6 +49,8 @@ describe('Publish', () => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => {
             g.command.mockClear();
         });
+
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
     it('Should publish messages to zigbee devices', async () => {
@@ -256,7 +259,7 @@ describe('Publish', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -266,10 +269,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -278,10 +281,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -337,7 +340,7 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('Should publish messages to groups color', async () => {
@@ -411,6 +414,31 @@ describe('Publish', () => {
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
+    it('Should handle get with multiple endpoints', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_left: '', state_right: ''}));
+        await flushPromises();
+        expect(endpoint2.read).toHaveBeenCalledTimes(1);
+        expect(endpoint2.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
+    it('Should log error when device has no such endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        logger.error.mockClear();
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_center: '', state_right: ''}));
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'wall_switch_double' has no endpoint 'center'`);
+        expect(endpoint2.read).toHaveBeenCalledTimes(0);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
     it('Should not respond to bridge/config/devices/get', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', JSON.stringify({state: 'ON'}));
         await flushPromises();
@@ -475,10 +503,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -496,6 +524,21 @@ describe('Publish', () => {
         expectNothingPublished();
     });
 
+    it('Should send state update on toggle specific endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(2);
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'ON');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'TOGGLE');
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "toggle", {}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'ON'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'OFF'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
     it('Should parse set with postfix topic and attribute', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(2);
@@ -575,7 +618,7 @@ describe('Publish', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 100});
     });
 
-    it('Should use transition when brightness', async () => {
+    it('Should use transition on brightness command', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
         const endpoint = device.getEndpoint(1);
@@ -586,6 +629,56 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
     });
 
+    it('Should use transition from device_options on brightness command', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['device_options'], {transition: 20});
+        const endpoint = device.getEndpoint(1);
+        const payload = {brightness: 20};
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
+    });
+
+    it('Should turn bulb on with full brightness when transition is used and no brightness is known', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3799
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "OFF", "transition": 0.5}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 0.5}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color',
+            JSON.stringify({state: 'OFF', brightness: 0}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2,
+            'zigbee2mqtt/bulb_color',
+            JSON.stringify({state: 'ON', brightness: 254}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 5}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 5}, {}]);
+    });
+
+    it('Transition parameter should not influence brightness on state ON', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3563
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 50}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON"}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 10}, {}]);
+    });
+
     it('Should use transition when color temp', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
@@ -642,11 +735,20 @@ describe('Publish', () => {
     it('Should turn device off when brightness 0 is send', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        const payload = {'brightness': 0};
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 50, state: 'ON'}));
         await flushPromises();
-        expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "off", {}, {}]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 0}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should turn device off when brightness 0 is send with light_brightness converter', async () => {
@@ -666,7 +768,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 45510.416666666664, "saturation": 127, "transtime": 0,}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891.475, "saturation": 199.21474, "transtime": 0,}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}});
@@ -719,7 +821,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 30}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 30}, {}]);
     });
 
     it('Should turn device on with on/off with transition when transition 0 is provided', async () => {
@@ -729,7 +831,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 0}, {}]);
     });
 
     it('Should turn device off with onOff on off with transition', async () => {
@@ -742,7 +844,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 10}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('When device is turned off and on with transition with report enabled it should restore correct brightness', async () => {
@@ -760,7 +862,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Bulb reports brightness while decreasing brightness
         await zigbeeHerdsman.events.message({data: {currentLevel: 1}, cluster: 'genLevelCtrl', device, endpoint, type: 'attributeReport', linkquality: 10});
@@ -792,7 +894,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Turn on again
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
@@ -885,7 +987,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -898,7 +1000,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/led_controller_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -977,7 +1079,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/roller_shutter/set', JSON.stringify({state: 'OPEN'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevel", {"level": "255", "transtime": 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {"level": "255", "transtime": 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/roller_shutter');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({position: 100});
@@ -990,7 +1092,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', JSON.stringify({state: 'ON'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: 16}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});
@@ -1075,7 +1177,7 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON"}', { qos: 0, retain: false }, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[1]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"OFF","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"OFF","brightness":0}', { qos: 0, retain: false }, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[3]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
     });
 });

test/receive.test.js

@@ -142,6 +142,20 @@ describe('Receive', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
     });
 
+    it('Shouldnt republish old state', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3572
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        jest.runAllTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+    });
+
     it('Should handle a zigbee message with 1 precision', async () => {
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
@@ -352,23 +366,27 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from unsupported devices', async () => {
+    it('Should not handle messages from unsupported devices and link to docs', async () => {
         const device = zigbeeHerdsman.devices.unsupported;
         const data = {onOff: 1};
+        logger.warn.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from still interviewing devices with unknown modelID', async () => {
+    it('Should not handle messages from still interviewing devices with unknown definition', async () => {
         const device = zigbeeHerdsman.devices.interviewing;
         const data = {onOff: 1};
+        logger.debug.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, modelID is undefined and still interviewing`);
+        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, definition is undefined and still interviewing`);
     });
 
     it('Should handle a command', async () => {

test/report.test.js

@@ -108,6 +108,20 @@ describe('Report', () => {
         expect(device.meta.reporting).toBeUndefined();
     });
 
+    it('Should not configure reporting when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.bulb_2;
+        const endpoint = device.getEndpoint(1);
+        device.interviewing = true;
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
     it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);

test/settings.test.js

@@ -359,6 +359,14 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Should throw error when changing entity options of non-existing device', () => {
+        write(configurationFile, {});
+
+        expect(() => {
+            settings.changeEntityOptions('not_existing_123', {});
+        }).toThrow(new Error("Device or group 'not_existing_123' does not exist"));
+    });
+
     it('Should not add duplicate groups', () => {
         write(configurationFile, {});
 
@@ -520,17 +528,17 @@ describe('Settings', () => {
         }).toThrowError('MQTT retention requires protocol version 5');
     });
 
-    it('Should not allow non-existing entities in availability blacklist', () => {
+    it('Should not allow non-existing entities in availability_blocklist', () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
-            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+            advanced: {availability_blocklist: ['0x0017880104e45519', 'non_existing']},
         });
 
         settings._reRead();
 
         expect(() => {
             settings.validate();
-        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blocklist'`);
     });
 
     it('Should ban devices', () => {
@@ -541,6 +549,14 @@ describe('Settings', () => {
         expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
     });
 
+    it('Should add devices to blocklist', () => {
+        write(configurationFile, {});
+        settings.blockDevice('0x123');
+        expect(settings.get().blocklist).toStrictEqual(['0x123']);
+        settings.blockDevice('0x1234');
+        expect(settings.get().blocklist).toStrictEqual(['0x123', '0x1234']);
+    });
+
     it('Should throw error when yaml file is invalid', () => {
         fs.writeFileSync(configurationFile, `
              good: 9

test/stub/data.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 
 const mockDir = tmp.dirSync().name;
 const mockDirStorage = tmp.dirSync().name;
+const stateFile = path.join(mockDir, 'state.json');
 
 function writeDefaultConfiguration() {
     const config = {
@@ -174,14 +175,25 @@ function writeDefaultConfiguration() {
                 retain: false,
                 devices: ['bulb_2']
             },
-        }
+        },
+        external_converters: [],
     };
 
     yaml.writeIfChanged(path.join(mockDir, 'configuration.yaml'), config);
 }
 
 function writeEmptyState() {
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify({}));
+    fs.writeFileSync(stateFile, JSON.stringify({}));
+}
+
+function removeState() {
+    if (stateExists()) {
+        fs.unlinkSync(stateFile);
+    }
+}
+
+function stateExists() {
+    return fs.existsSync(stateFile);
 }
 
 function writeDefaultState() {
@@ -200,10 +212,6 @@ function writeDefaultState() {
     fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
 }
 
-function removeState() {
-    fs.unlinkSync(path.join(mockDir, 'state.json'))
-}
-
 jest.mock('../../lib/util/data', () => ({
     joinPath: (file) => require('path').join(mockDir, file),
     getPath: () => mockDir,
@@ -219,4 +227,5 @@ module.exports = {
     writeDefaultState,
     removeState,
     writeEmptyState,
+    stateExists,
 };

test/stub/zigbeeHerdsman.js

@@ -8,6 +8,7 @@ class Group {
         this.meta = {};
         this.members = members;
         this.removeFromDatabase = jest.fn();
+        this.removeFromNetwork = jest.fn();
         this.hasMember = (endpoint) => this.members.includes(endpoint);
     }
 }
@@ -47,9 +48,10 @@ class Endpoint {
             return this.outputClusters.includes(clusters[cluster]);
         }
 
-        this.addToGroup = (group) => {
+        this.addToGroup = jest.fn();
+        this.addToGroup.mockImplementation((group) => {
             if (!group.members.includes(this)) group.members.push(this);
-        }
+        })
 
         this.getDevice = () => {
             return Object.values(devices).find(d => d.ieeeAddr === deviceIeeeAddress);