1.16.2

Co-authored-by: nurikk <nurikk@users.noreply.github.com>

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,14 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Frontend issues
+    url: https://github.com/nurikk/z2m-frontend/issues
+    about: Issues related to the frontend.
+  - name: Home Assistant addon issues
+    url: https://github.com/danielwelch/hassio-zigbee2mqtt/issues
+    about: Issues related to the Home Assistant addon.
+  - name: FAQ
+    url: https://www.zigbee2mqtt.io/information/FAQ.html
+    about: Frequently asked questions.
+  - name: Support Chat
+    url: https://discord.gg/NyseBeK
+    about: Chat for feedback, questions and troubleshooting.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature request
+assignees: ''
+
+---
+
+## Is your feature request related to a problem? Please describe
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -0,0 +1,18 @@
+---
+name: New device support request
+about: Request support for a new device
+title: ''
+labels: new device support
+assignees: ''
+
+---
+
+<!--
+IMPORTANT: Before submitting a request, first follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+-->
+
+## Information about the device + link
+..
+
+## data/database.db entry of the device
+..

.github/ISSUE_TEMPLATE/problem_report.md

@@ -0,0 +1,54 @@
+---
+name: Problem report
+about: Create a report to help us improve
+title: ''
+labels: problem
+assignees: ''
+
+---
+
+<!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
+Did you read the FAQ?
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
+
+Unsupported device?
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
+
+Bug report?
+- If applicable, provide steps how to reproduce the problem.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
+-->
+
+<!--Start your bug report below this line-->
+
+## What happened
+
+## What did you expect to happen
+
+## How to reproduce it (minimal and precise)
+
+## Debug info
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1, CC1352P-2, Conbee II
+Adapter firmware version:

.github/workflows/ci.yml

@@ -31,20 +31,23 @@ jobs:
     - name: Docker login
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
       run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
-    - name: Docker setup
+    - name: Docker setup - QEMU
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
-      run: |
-        sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
-        sudo chmod a+x /usr/local/bin/buildx
-        docker run --rm --privileged hypriot/qemu-register:v2.7.0
-        buildx create --use
-        buildx ls
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+    - name: Docker setup - Buildx
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
     - name: Docker build dev
       if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
       run: |
-        buildx build \
+        docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
           -t koenkk/zigbee2mqtt:latest-dev \
@@ -53,9 +56,9 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
       run: |
         TAG="$(git describe --tags)"
-        buildx build \
+        docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
           -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
@@ -73,4 +76,13 @@ jobs:
           --header "Content-Type: application/json" \
           --data '{"definition": {"id": 1}}' \
           -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 9}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/stale.yml

@@ -2,6 +2,7 @@ name: "Close stale issues/pull requests"
 on:
   schedule:
   - cron: "0 0 * * *"
+  workflow_dispatch:
 
 jobs:
   stale:
@@ -14,3 +15,5 @@ jobs:
         stale-pr-message: 'This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
         days-before-stale: 30
         days-before-close: 7
+        operations-per-run: 500
+        exempt-issue-labels: 'breaking change'

.github/workflows/update_deps.yml

@@ -16,9 +16,9 @@ jobs:
       with:
         node-version: 12
     - run: npm install
-    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee-herdsman" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
+    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
     - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies
         branch: "update_deps"

.github/workflows/update_frontend.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-frontend
+on:
+  repository_dispatch:
+    types: update_frontend
+
+jobs:
+  update_frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
+        branch: "update_frontend"
+        title: Update zigbee-frontend to ${{ github.event.client_payload.version }}

.github/workflows/update_zh.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
         branch: "update_zh"

.github/workflows/update_zhc.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
         branch: "update_zhc"

.gitignore

@@ -70,4 +70,7 @@ data/database.db.backup
 data/extension
 
 # MacOS indexing file
-.DS_Store
+.DS_Store
+
+# Ignore config
+data/*.yaml

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "files.associations": {
-        "*.yaml": "home-assistant"
-    }
-}

README.md

@@ -53,7 +53,7 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Hassio](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
 - Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12-alpine as base
+FROM node:12-alpine3.12 as base
 
 WORKDIR /app
 RUN apk add --no-cache tzdata eudev
@@ -10,7 +10,7 @@ FROM base as dependencies
 
 COPY npm-shrinkwrap.json ./
 
-RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
     npm ci --production && \
     apk del .buildtools
 

lib/controller.js

@@ -6,12 +6,14 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
 const fs = require('fs');
 const data = require('./util/data');
 const path = require('path');
 const assert = require('assert');
 
 // Extensions
+const ExtensionFrontend = require('./extension/frontend');
 const ExtensionPublish = require('./extension/publish');
 const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
@@ -66,6 +68,10 @@ class Controller {
             this.extensions.push(new ExtensionBridge(...args, this.enableDisableExtension));
         }
 
+        if (settings.get().frontend) {
+            this.extensions.push(new ExtensionFrontend(...args));
+        }
+
         if (settings.get().advanced.legacy_api) {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
@@ -98,6 +104,7 @@ class Controller {
 
     async start() {
         this.state.start();
+        logger.logOutput();
 
         const info = await utils.getZigbee2mqttVersion();
         logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
@@ -120,14 +127,10 @@ class Controller {
         logger.info(`Currently ${devices.length} devices are joined:`);
         for (const device of devices) {
             const entity = this.zigbee.resolveEntity(device);
-            logger.info(
-                (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
-                ` (${entity.device.ieeeAddr}): ` +
-                (entity.definition ?
-                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
-                    'Not supported ') +
-                `(${entity.device.type})`,
-            );
+            const model = entity.definition ?
+                `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
+                'Not supported';
+            logger.info(`${entity.name} (${entity.device.ieeeAddr}): ${model} (${entity.device.type})`);
         }
 
         // Enable zigbee join.
@@ -208,9 +211,14 @@ class Controller {
 
     async onZigbeeEvent(type, data) {
         const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
+        if (data.device && data.device.type === 'Coordinator') {
+            logger.debug('Ignoring message from coordinator');
+            return;
+        }
+
+        if (data.device && !settings.getDevice(data.device.ieeeAddr)) {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
+            resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
         }
 
         const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
@@ -218,7 +226,7 @@ class Controller {
         if (type === 'message') {
             logger.debug(
                 `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
-                `, data '${JSON.stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                `, data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
                 (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
             );
         } else if (type === 'deviceJoined') {
@@ -232,7 +240,8 @@ class Controller {
                     logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
-                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' and manufacturer name ` +
+                        `'${data.device.manufacturerName}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
                 }
@@ -250,6 +259,7 @@ class Controller {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
                 logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                this.state.remove(data.ieeeAddr);
             }
         }
 
@@ -285,13 +295,12 @@ class Controller {
             messagePayload = newState;
         }
 
-        const deviceOptions = settings.get().device_options;
         const options = {
-            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
-            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
+            retain: utils.getObjectProperty(resolvedEntity.settings, 'retain', false),
+            qos: utils.getObjectProperty(resolvedEntity.settings, 'qos', 0),
         };
 
-        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
+        const retention = utils.getObjectProperty(resolvedEntity.settings, 'retention', false);
         if (retention !== false) {
             options.properties = {messageExpiryInterval: retention};
         }
@@ -311,30 +320,30 @@ class Controller {
             attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
         }
 
+        // Add lastseen
         const lastSeen = settings.get().advanced.last_seen;
         if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
             messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
         }
 
-        // filter mqtt message attributes
-        if (deviceOptions.filtered_attributes) {
-            deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        // Add device linkquality.
+        if (resolvedEntity.type === 'device' && resolvedEntity.device.linkquality !== undefined) {
+            messagePayload.linkquality = resolvedEntity.device.linkquality;
         }
+
+        // filter mqtt message attributes
         if (resolvedEntity.settings.filtered_attributes) {
             resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
         if (Object.entries(messagePayload).length) {
-            if (settings.get().experimental.output === 'attribute_and_json') {
-                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+            const output = settings.get().experimental.output;
+            if (output === 'attribute_and_json' || output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, stringify(messagePayload), options);
+            }
+
+            if (output === 'attribute_and_json' || output === 'attribute') {
                 await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-            } else if (settings.get().experimental.output === 'json') {
-                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
-            } else {
-                /* istanbul ignore else */
-                if (settings.get().experimental.output === 'attribute') {
-                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-                }
             }
         }
 
@@ -359,7 +368,7 @@ class Controller {
             } else if (typeof subPayload === 'object') {
                 this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
             } else {
-                message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
+                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
             }
 
             if (message !== null) {

lib/eventBus.js

@@ -9,6 +9,7 @@ const allowedEvents = [
     'stateChange', // Entity changes its state
     'groupMembersChanged', // Members of a group has been changed
     'reportingDisabled', // Reporting is disabled for a device
+    'devicesChanged', // Devices changed
 ];
 
 class EventBus extends events.EventEmitter {

lib/extension/availability.js

@@ -24,6 +24,8 @@ class Availability extends Extension {
         this.timers = {};
         this.state = {};
 
+        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+
         this.blocklist = settings.get().advanced.availability_blocklist
             .concat(settings.get().advanced.availability_blacklist)
             .map((e) => settings.getEntity(e).ID);
@@ -33,6 +35,12 @@ class Availability extends Extension {
             .map((e) => settings.getEntity(e).ID);
     }
 
+    onDeviceRemoved(resolvedEntity) {
+        this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
+        delete this.state[resolvedEntity.device.ieeeAddr];
+        clearTimeout(this.timers[resolvedEntity.device.ieeeAddr]);
+    }
+
     inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
         const deviceSettings = settings.getDevice(ieeeAddr);
@@ -114,10 +122,7 @@ class Availability extends Extension {
     }
 
     setTimerPingable(device) {
-        if (this.timers[device.ieeeAddr]) {
-            clearTimeout(this.timers[device.ieeeAddr]);
-        }
-
+        clearTimeout(this.timers[device.ieeeAddr]);
         this.timers[device.ieeeAddr] = setTimeout(async () => {
             await this.handleIntervalPingable(device);
         }, utils.secondsToMilliseconds(this.availability_timeout));
@@ -135,13 +140,11 @@ class Availability extends Extension {
     async onReconnect(device) {
         const resolvedEntity = this.zigbee.resolveEntity(device);
         if (resolvedEntity && resolvedEntity.definition) {
-            const used = [];
             try {
                 for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                     const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
-                    if (converter && !used.includes(converter)) {
+                    if (converter) {
                         await converter.convertGet(device.endpoints[0], key, {});
-                        used.push(converter);
                     }
                 }
             } catch (error) {

lib/extension/bind.js

@@ -4,6 +4,7 @@ const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
 
@@ -16,20 +17,6 @@ class Bind extends Extension {
         this.legacyApi = settings.get().advanced.legacy_api;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
-        }
-    }
-
     parseMQTTMessage(topic, message) {
         let type = null;
         let sourceKey = null;
@@ -106,7 +93,7 @@ class Bind extends Extension {
                         if (settings.get().advanced.legacy_api) {
                             this.mqtt.publish(
                                 'bridge/log',
-                                JSON.stringify({type: `device_${type}`,
+                                stringify({type: `device_${type}`,
                                     message: {from: sourceName, to: targetName, cluster}}),
                             );
                         }
@@ -121,7 +108,7 @@ class Bind extends Extension {
                         if (settings.get().advanced.legacy_api) {
                             this.mqtt.publish(
                                 'bridge/log',
-                                JSON.stringify({type: `device_${type}_failed`,
+                                stringify({type: `device_${type}_failed`,
                                     message: {from: sourceName, to: targetName, cluster}}),
                             );
                         }
@@ -137,7 +124,7 @@ class Bind extends Extension {
                 if (settings.get().advanced.legacy_api) {
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                        stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
                     );
                 }
             } else if (failedClusters.length === attemptedClusters.length) {
@@ -151,11 +138,13 @@ class Bind extends Extension {
         const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
         if (!triggeredViaLegacyApi) {
             const response = utils.getResponse(message, responseData, error);
-            await this.mqtt.publish(`bridge/response/device/${type}`, JSON.stringify(response));
+            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
         }
 
         if (error) {
             logger.error(error);
+        } else {
+            this.eventBus.emit(`devicesChanged`);
         }
     }
 }

lib/extension/bridge.js

@@ -4,6 +4,8 @@ const Extension = require('./extension');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../util/settings');
 const Transport = require('winston-transport');
+const stringify = require('json-stable-stringify-without-jsonify');
+const objectAssignDeep = require(`object-assign-deep`);
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
@@ -28,6 +30,8 @@ class Bridge extends Extension {
             'config/elapsed': this.configElapsed.bind(this),
             'config/log_level': this.configLogLevel.bind(this),
             'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'touchlink/identify': this.touchlinkIdentify.bind(this),
+            'touchlink/scan': this.touchlinkScan.bind(this),
             'health_check': this.healthCheck.bind(this),
         };
     }
@@ -37,7 +41,7 @@ class Bridge extends Extension {
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
         this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        this.eventBus.on(`devicesChanged`, () => this.publishDevices());
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
@@ -47,7 +51,7 @@ class Bridge extends Extension {
         const mqtt = this.mqtt;
         class EventTransport extends Transport {
             log(info, callback) {
-                const payload = JSON.stringify({message: info.message, level: info.level});
+                const payload = stringify({message: info.message, level: info.level});
                 mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
                 callback();
             }
@@ -63,11 +67,11 @@ class Bridge extends Extension {
 
             try {
                 const response = await this.requestLookup[match[1].toLowerCase()](message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             } catch (error) {
                 logger.error(`Request '${topic}' failed with error: '${error.message}'`);
                 const response = utils.getResponse(message, {}, error.message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             }
         }
     }
@@ -77,7 +81,7 @@ class Bridge extends Extension {
             this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
         }
 
-        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview', 'deviceAnnounce'].includes(type)) {
             let payload;
             const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
             if (type === 'deviceJoined') {
@@ -87,19 +91,19 @@ class Bridge extends Extension {
                     friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
                 };
                 if (data.status === 'successful') {
-                    payload.supported = !!resolvedEntity.definition;
-                    payload.definition = resolvedEntity.definition ? {
-                        model: resolvedEntity.definition.model,
-                        vendor: resolvedEntity.definition.vendor,
-                        description: resolvedEntity.definition.description,
-                        supports: resolvedEntity.definition.supports,
-                    } : null;
+                    const definition = resolvedEntity.definition;
+                    payload.supported = !!definition;
+                    payload.definition = this.getDefinitionPayload(definition);
                 }
+            } else if (type === 'deviceAnnounce') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
+                };
             } else payload = {ieee_address: ieeeAddress}; // deviceLeave
 
             await this.mqtt.publish(
                 'bridge/event',
-                JSON.stringify({type: utils.toSnakeCase(type), data: payload}),
+                stringify({type: utils.toSnakeCase(type), data: payload}),
                 {retain: false, qos: 0},
             );
         }
@@ -155,10 +159,29 @@ class Bridge extends Extension {
     }
 
     async permitJoin(message) {
-        const value = this.getValue(message);
-        await this.zigbee.permitJoin(value);
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        let value;
+        let resolvedEntity;
+        if (typeof message === 'object') {
+            value = message.value;
+            if (message.device) {
+                resolvedEntity = this.zigbee.resolveEntity(message.device);
+                if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                    throw new Error(`Device '${message.device}' does not exist`);
+                }
+            }
+        } else {
+            value = message;
+        }
+
+        await this.zigbee.permitJoin(value, resolvedEntity);
         await this.publishInfo();
-        return utils.getResponse(message, {value: value}, null);
+        return utils.getResponse(
+            message, resolvedEntity ? {value: value, device: message.device} : {value: value}, null,
+        );
     }
 
     configLastSeen(message) {
@@ -169,6 +192,7 @@ class Bridge extends Extension {
         }
 
         settings.set(['advanced', 'last_seen'], message);
+        this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
@@ -181,6 +205,7 @@ class Bridge extends Extension {
 
         this.enableDisableExtension(value, 'HomeAssistant');
         settings.set(['homeassistant'], value);
+        this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
@@ -192,6 +217,7 @@ class Bridge extends Extension {
         }
 
         settings.set(['advanced', 'elapsed'], value);
+        this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
@@ -207,18 +233,50 @@ class Bridge extends Extension {
         return utils.getResponse(message, {value}, null);
     }
 
+    async touchlinkIdentify(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
+            !message.hasOwnProperty('channel')) {
+            throw new Error('Invalid payload');
+        }
+
+        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
+        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
+        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
+    }
+
     async touchlinkFactoryReset(message) {
-        logger.info('Start touchlink factory reset');
-        const result = await this.zigbee.touchlinkFactoryReset();
+        let result = false;
+        const payload = {};
+        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
+            message.hasOwnProperty('channel')) {
+            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
+            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
+            payload.ieee_address = message.ieee_address;
+            payload.channel = message.channel;
+        } else {
+            logger.info('Start Touchlink factory reset of first found device');
+            result = await this.zigbee.touchlinkFactoryResetFirst();
+        }
+
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
-            return utils.getResponse(message, {}, null);
+            return utils.getResponse(message, payload, null);
         } else {
             logger.error('Failed to factory reset device through Touchlink');
             throw new Error('Failed to factory reset device through Touchlink');
         }
     }
 
+    async touchlinkScan(message) {
+        logger.info('Start Touchlink scan');
+        const result = await this.zigbee.touchlinkScan();
+        const found = result.map((r) => {
+            return {ieee_address: r.ieeeAddr, channel: r.channel};
+        });
+        logger.info('Finished Touchlink scan');
+        return utils.getResponse(message, {found}, null);
+    }
+
     /**
      * Utils
      */
@@ -263,21 +321,38 @@ class Bridge extends Extension {
             throw new Error('No device has joined since start');
         }
 
+        const validationErrors = utils.validateFriendlyName(message.to);
+        if (validationErrors.length !== 0) {
+            throw new Error(validationErrors[0]);
+        }
+
         const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
         const to = message.to;
+        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
+            message.homeassistant_rename : false;
         const entity = this.getEntity(entityType, from);
 
         settings.changeFriendlyName(from, to);
 
+        // Clear retained messages
+        this.mqtt.publish(entity.name, '', {retain: true});
+
         if (entity.type === 'device') {
             this.publishDevices();
-            this.eventBus.emit(`deviceRenamed`, {device: entity.device});
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename});
         } else {
             this.publishGroups();
-            this.eventBus.emit(`groupRenamed`, {group: entity.group});
+            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename});
         }
 
-        return utils.getResponse(message, {from: entity.settings.friendlyName, to}, null);
+        // Repulish entity state
+        this.publishEntityState(to, {});
+
+        return utils.getResponse(
+            message,
+            {from: entity.settings.friendlyName, to, homeassistant_rename: homeAssisantRename},
+            null,
+        );
     }
 
     async removeEntity(entityType, message) {
@@ -319,7 +394,7 @@ class Bridge extends Extension {
 
             // Fire event
             if (entity.type === 'device') {
-                this.eventBus.emit('deviceRemoved', {device: entity.device});
+                this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
             }
 
             // Remove from configuration.yaml
@@ -332,9 +407,13 @@ class Bridge extends Extension {
             // Remove from state
             this.state.remove(entity.settings.ID);
 
+            // Clear any retained messages
+            this.mqtt.publish(entity.name, '', {retain: true});
+
             logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
 
             if (entity.type === 'device') {
+                this.publishGroups();
                 this.publishDevices();
                 return utils.getResponse(message, {id: ID, block, force}, null);
             } else {
@@ -357,6 +436,10 @@ class Bridge extends Extension {
     }
 
     async publishInfo() {
+        const config = objectAssignDeep.noMutate({}, settings.get());
+        delete config.advanced.network_key;
+        delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
         const payload = {
             version: this.zigbee2mqttVersion.version,
             commit: this.zigbee2mqttVersion.commitHash,
@@ -364,21 +447,42 @@ class Bridge extends Extension {
             network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
             log_level: logger.getLevel(),
             permit_join: await this.zigbee.getPermitJoin(),
+            config,
         };
 
-        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish('bridge/info', stringify(payload), {retain: true, qos: 0});
     }
 
     async publishDevices() {
-        const devices = this.zigbee.getClients().map((device) => {
+        const devices = this.zigbee.getDevices().map((device) => {
             const definition = zigbeeHerdsmanConverters.findByDevice(device);
             const resolved = this.zigbee.resolveEntity(device);
-            const definitionPayload = definition ? {
-                model: definition.model,
-                vendor: definition.vendor,
-                description: definition.description,
-                supports: definition.supports,
-            } : null;
+            const endpoints = {};
+            for (const endpoint of device.endpoints) {
+                const data = {
+                    bindings: [],
+                    clusters: {
+                        input: endpoint.getInputClusters().map((c) => c.name),
+                        output: endpoint.getOutputClusters().map((c) => c.name),
+                    },
+                };
+
+                for (const bind of endpoint.binds) {
+                    let target;
+
+                    if (bind.target.constructor.name === 'Endpoint') {
+                        target = {
+                            type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID,
+                        };
+                    } else {
+                        target = {type: 'group', id: bind.target.groupID};
+                    }
+
+                    data.bindings.push({cluster: bind.cluster.name, target});
+                }
+
+                endpoints[endpoint.ID] = data;
+            }
 
             return {
                 ieee_address: device.ieeeAddr,
@@ -386,16 +490,17 @@ class Bridge extends Extension {
                 network_address: device.networkAddress,
                 supported: !!definition,
                 friendly_name: resolved.name,
-                definition: definitionPayload,
+                definition: this.getDefinitionPayload(definition),
                 power_source: device.powerSource,
                 software_build_id: device.softwareBuildID,
                 date_code: device.dateCode,
                 interviewing: device.interviewing,
                 interview_completed: device.interviewCompleted,
+                endpoints,
             };
         });
 
-        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+        await this.mqtt.publish('bridge/devices', stringify(devices), {retain: true, qos: 0});
     }
 
     async publishGroups() {
@@ -403,7 +508,7 @@ class Bridge extends Extension {
             const resolved = this.zigbee.resolveEntity(group);
             return {
                 id: group.groupID,
-                friendly_name: resolved.name,
+                friendly_name: group.groupID === 901 ? 'default_bind_group' : resolved.name,
                 members: group.members.map((m) => {
                     return {
                         ieee_address: m.deviceIeeeAddress,
@@ -413,7 +518,20 @@ class Bridge extends Extension {
             };
         });
 
-        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+        await this.mqtt.publish('bridge/groups', stringify(groups), {retain: true, qos: 0});
+    }
+
+    getDefinitionPayload(definition) {
+        if (definition) {
+            return {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                exposes: definition.exposes,
+            };
+        } else {
+            return null;
+        }
     }
 }
 

lib/extension/configure.js

@@ -2,6 +2,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 /**
  * This extension calls the zigbee-herdsman-converters definition configure() method
@@ -52,18 +53,6 @@ class Configure extends Extension {
         return true;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (settings.get().advanced.legacy_api) {
-            this.mqtt.subscribe(this.legacyTopic);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         if (topic === this.legacyTopic) {
             const resolvedEntity = this.zigbee.resolveEntity(message);
@@ -97,7 +86,7 @@ class Configure extends Extension {
             }
 
             const response = utils.getResponse(message, {id: ID}, error);
-            await this.mqtt.publish(`bridge/response/device/configure`, JSON.stringify(response));
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
         }
     }
 

lib/extension/frontend.js

@@ -0,0 +1,127 @@
+const http = require('http');
+const serveStatic = require('serve-static');
+const finalhandler = require('finalhandler');
+const Extension = require('./extension');
+const logger = require('../util/logger');
+const frontend = require('zigbee2mqtt-frontend');
+const WebSocket = require('ws');
+const url = require('url');
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+/**
+ * This extension servers the frontend
+ */
+class Frontend extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.onRequest = this.onRequest.bind(this);
+        this.onUpgrade = this.onUpgrade.bind(this);
+        this.mqttBaseTopic = settings.get().mqtt.base_topic;
+        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
+        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
+        this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+        this.host = settings.get().frontend.host || '0.0.0.0';
+        this.port = settings.get().frontend.port || 8080;
+        this.authToken = settings.get().frontend.auth_token || false;
+        this.retainedMessages = new Map();
+        /* istanbul ignore next */
+        const options = {setHeaders: (res, path) => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+    }
+
+    onZigbeeStarted() {
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    async stop() {
+        for (const client of this.wss.clients) {
+            client.close();
+        }
+
+        return new Promise((resolve) => {
+            this.server.close(resolve);
+        });
+    }
+    onRequest(request, response) {
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+    authenticate(request, cb) {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    onUpgrade(request, socket, head) {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    onWebSocketConnection(ws) {
+        ws.on('message', (message) => {
+            if (message) {
+                const {topic, payload} = utils.parseJSON(message, message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            let payload = {};
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.state.exists(device.ieeeAddr)) {
+                payload = {...payload, ...this.state.get(device.ieeeAddr)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+            }
+
+            if (resolvedEntity.device.linkquality !== undefined) {
+                payload.linkquality = resolvedEntity.device.linkquality;
+            }
+
+            ws.send(stringify({topic: resolvedEntity.name, payload}));
+        }
+    }
+
+    onMQTTPublishedMessage(data) {
+        let {topic, payload, options} = data;
+        if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            topic = topic.substring(this.mqttBaseTopic.length + 1);
+            payload = utils.parseJSON(payload, payload);
+            if (options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            for (const client of this.wss.clients) {
+                if (client.readyState === WebSocket.OPEN) {
+                    client.send(stringify({topic, payload}));
+                }
+            }
+        }
+    }
+}
+
+module.exports = Frontend;

lib/extension/groups.js

@@ -3,6 +3,8 @@ const logger = require('../util/logger');
 const Extension = require('./extension');
 const utils = require('../util/utils');
 const postfixes = utils.getEndpointNames();
+const stringify = require('json-stable-stringify-without-jsonify');
+const equals = require('fast-deep-equal/es6');
 
 const topicRegex =
     new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
@@ -14,27 +16,7 @@ class Groups extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
         this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-
-            for (let step = 1; step < 20; step++) {
-                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-                this.mqtt.subscribe(`${topic}remove`);
-                this.mqtt.subscribe(`${topic}add`);
-                this.mqtt.subscribe(`${topic}remove_all`);
-            }
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
-        }
+        this.lastOptimisticState = {};
     }
 
     async onZigbeeStarted() {
@@ -118,8 +100,10 @@ class Groups extends Extension {
 
             if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint) &&
+                        !equals(this.lastOptimisticState[zigbeeGroup.groupID], payload)) {
                         if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                            this.lastOptimisticState[zigbeeGroup.groupID] = payload;
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }
@@ -188,7 +172,7 @@ class Groups extends Extension {
                         };
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                            stringify({type: `device_group_${type}_failed`, message: payload}),
                         );
                     }
 
@@ -209,7 +193,7 @@ class Groups extends Extension {
                     };
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        stringify({type: `device_group_${type}_failed`, message: payload}),
                     );
                 }
 
@@ -289,7 +273,7 @@ class Groups extends Extension {
                         const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_group_add`, message: payload}),
+                            stringify({type: `device_group_add`, message: payload}),
                         );
                     }
                 } else if (type === 'remove') {
@@ -302,7 +286,7 @@ class Groups extends Extension {
                         const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_group_remove`, message: payload}),
+                            stringify({type: `device_group_remove`, message: payload}),
                         );
                     }
                 } else { // remove_all
@@ -316,7 +300,7 @@ class Groups extends Extension {
                             const payload = {friendly_name: resolvedEntityDevice.name};
                             this.mqtt.publish(
                                 'bridge/log',
-                                JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                                stringify({type: `device_group_remove_all`, message: payload}),
                             );
                         }
                     }
@@ -328,7 +312,7 @@ class Groups extends Extension {
 
         if (!triggeredViaLegacyApi) {
             const response = utils.getResponse(message, responseData, error);
-            await this.mqtt.publish(`bridge/response/group/members/${type}`, JSON.stringify(response));
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
         }
 
         if (error) {

lib/extension/legacy/bridgeLegacy.js

@@ -4,6 +4,7 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const utils = require('../../util/utils');
 const assert = require('assert');
 const Extension = require('../extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
@@ -66,7 +67,7 @@ class BridgeLegacy extends Extension {
             logger.info(`Whitelisted '${entity.friendlyName}'`);
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
             );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
@@ -90,7 +91,7 @@ class BridgeLegacy extends Extension {
         const entity = settings.getEntity(json.friendly_name);
         assert(entity, `Entity '${json.friendly_name}' does not exist`);
         settings.changeEntityOptions(entity.ID, json.options);
-        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${stringify(json.options)})`);
     }
 
     async permitJoin(topic, message) {
@@ -176,9 +177,9 @@ class BridgeLegacy extends Extension {
         });
 
         if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
+            this.mqtt.publish(`bridge/config/devices`, stringify(devices), {});
         } else {
-            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
+            this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
         }
     }
 
@@ -189,7 +190,7 @@ class BridgeLegacy extends Extension {
             return group;
         });
 
-        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
+        this.mqtt.publish('bridge/log', stringify({type: 'groups', message: payload}));
     }
 
     rename(topic, message) {
@@ -229,11 +230,12 @@ class BridgeLegacy extends Extension {
             logger.info(`Successfully renamed - ${from} to ${to} `);
             const entity = this.zigbee.resolveEntity(to);
             const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            eventData.homeAssisantRename = false;
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
 
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+                stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
             );
         } catch (error) {
             logger.error(`Failed to rename - ${from} to ${to}`);
@@ -265,7 +267,7 @@ class BridgeLegacy extends Extension {
 
         const group = settings.addGroup(name, id);
         this.zigbee.createGroup(group.ID);
-        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
+        this.mqtt.publish('bridge/log', stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
@@ -281,7 +283,7 @@ class BridgeLegacy extends Extension {
         }
         settings.removeGroup(message);
 
-        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
+        this.mqtt.publish('bridge/log', stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }
 
@@ -308,13 +310,13 @@ class BridgeLegacy extends Extension {
         if (!entity) {
             logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
 
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
             return;
         }
 
         const cleanup = () => {
             // Fire event
-            this.eventBus.emit('deviceRemoved', {device: entity.device});
+            this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
 
             // Remove from configuration.yaml
             settings.removeDevice(entity.settings.ID);
@@ -323,7 +325,7 @@ class BridgeLegacy extends Extension {
             this.state.remove(entity.settings.ID);
 
             logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
@@ -340,7 +342,7 @@ class BridgeLegacy extends Extension {
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
 
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
         }
 
         if (action === 'ban') {
@@ -349,8 +351,6 @@ class BridgeLegacy extends Extension {
     }
 
     async onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
         await this.publish();
     }
 
@@ -383,7 +383,7 @@ class BridgeLegacy extends Extension {
             permit_join: await this.zigbee.getPermitJoin(),
         };
 
-        await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
@@ -394,7 +394,7 @@ class BridgeLegacy extends Extension {
         if (type === 'deviceJoined') {
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+                stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
             );
         } else if (type === 'deviceInterview') {
             if (data.status === 'successful') {
@@ -403,20 +403,20 @@ class BridgeLegacy extends Extension {
                     const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                        stringify({type: `pairing`, message: 'interview_successful', meta: log}),
                     );
                 } else {
                     const meta = {friendly_name: resolvedEntity.name, supported: false};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                        stringify({type: `pairing`, message: 'interview_successful', meta}),
                     );
                 }
             } else if (data.status === 'failed') {
                 const meta = {friendly_name: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                    stringify({type: `pairing`, message: 'interview_failed', meta}),
                 );
             } else {
                 /* istanbul ignore else */
@@ -424,13 +424,13 @@ class BridgeLegacy extends Extension {
                     const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                        stringify({type: `pairing`, message: 'interview_started', meta}),
                     );
                 }
             }
         } else if (type === 'deviceAnnounce') {
             const meta = {friendly_name: resolvedEntity.name};
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_announced`, message: 'announce', meta}));
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
@@ -438,7 +438,7 @@ class BridgeLegacy extends Extension {
                 const meta = {friendly_name: name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                    stringify({type: `device_removed`, message: 'left_network', meta}),
                 );
             }
         }
@@ -448,21 +448,21 @@ class BridgeLegacy extends Extension {
         logger.info('Starting touchlink factory reset...');
         this.mqtt.publish(
             'bridge/log',
-            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+            stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
         );
-        const result = await this.zigbee.touchlinkFactoryReset();
+        const result = await this.zigbee.touchlinkFactoryResetFirst();
 
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+                stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
             );
         } else {
             logger.warn('Failed to factory reset device through Touchlink');
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+                stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
             );
         }
     }

lib/extension/legacy/deviceGroupMembership.js

@@ -6,13 +6,6 @@ const Extension = require('../extension');
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
 class DeviceGroupMembership extends Extension {
-    onMQTTConnected() {
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
-            this.mqtt.subscribe(topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         const match = topic.match(topicRegex);
         if (!match) {
@@ -29,7 +22,18 @@ class DeviceGroupMembership extends Extension {
             `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
         );
 
-        const {grouplist, capacity} = response;
+        if (!response) {
+            logger.warn(`Couldn't get group membership of ${entity.device.ieeeAddr}`);
+            return;
+        }
+
+        let {grouplist, capacity} = response;
+
+        grouplist = grouplist.map((gid) => {
+            const g = settings.getGroup(gid);
+            return g ? g.friendlyName : gid;
+        });
+
         const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
         let msgCapacity;
         if (capacity === 254) {

lib/extension/networkMap.js

@@ -2,6 +2,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 /**
  * This extension creates a network map
@@ -28,18 +29,6 @@ class NetworkMap extends Extension {
         };
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(this.legacyTopic);
-            this.mqtt.subscribe(this.legacyTopicRoutes);
-        }
-
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         /* istanbul ignore else */
         if (this.legacyApi) {
@@ -47,7 +36,8 @@ class NetworkMap extends Extension {
                 this.supportedFormats.hasOwnProperty(message)) {
                 const includeRoutes = topic === this.legacyTopicRoutes;
                 const topology = await this.networkScan(includeRoutes);
-                const converted = this.supportedFormats[message](topology);
+                let converted = this.supportedFormats[message](topology);
+                converted = message === 'raw' ? stringify(converted) : converted;
                 this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
             }
         }
@@ -65,19 +55,19 @@ class NetworkMap extends Extension {
                 const value = this.supportedFormats[type](topology);
                 await this.mqtt.publish(
                     'bridge/response/networkmap',
-                    JSON.stringify(utils.getResponse(message, {routes, type, value}, null)),
+                    stringify(utils.getResponse(message, {routes, type, value}, null)),
                 );
             } catch (error) {
                 await this.mqtt.publish(
                     'bridge/response/networkmap',
-                    JSON.stringify(utils.getResponse(message, {}, error.message)),
+                    stringify(utils.getResponse(message, {}, error.message)),
                 );
             }
         }
     }
 
     raw(topology) {
-        return JSON.stringify(topology);
+        return topology;
     }
 
     graphviz(topology) {
@@ -100,9 +90,8 @@ class NetworkMap extends Extension {
 
             // Add the device model
             if (node.type !== 'Coordinator') {
-                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
-                if (definition) {
-                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                if (node.definition) {
+                    labels.push(`${node.definition.vendor} ${node.definition.description} (${node.definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
                     labels.push(`${node.manufacturerName} ${node.modelID}`);
@@ -251,10 +240,22 @@ class NetworkMap extends Extension {
         // Add nodes
         for (const device of devices) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
+            const definition = resolvedEntity.definition ? {
+                model: resolvedEntity.definition.model,
+                vendor: resolvedEntity.definition.vendor,
+                description: resolvedEntity.definition.description,
+                supports: Array.from(new Set(resolvedEntity.definition.exposes.map((e) => {
+                    return e.hasOwnProperty('name') ?
+                        e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
+            } : null;
+
             networkMap.nodes.push({
                 ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+                definition,
             });
         }
 
@@ -266,6 +267,13 @@ class NetworkMap extends Extension {
                     continue;
                 }
 
+                // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
+                // Determine the correct ieeeAddr based on the networkAddress.
+                const neighborDevice = this.zigbee.getDeviceByNetworkAddress(neighbor.networkAddress);
+                if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
+                    neighbor.ieeeAddr = neighborDevice.ieeeAddr;
+                }
+
                 const link = {
                     source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
                     target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},

lib/extension/onEvent.js

@@ -27,10 +27,10 @@ class OnEvent extends Extension {
     }
 
     callOnEvent(resolvedEntity, type, data) {
-        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
 
         if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
-            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
         }
     }
 }

lib/extension/otaUpdate.js

@@ -1,5 +1,6 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
 const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
 const topicRegex =
@@ -17,18 +18,6 @@ class OTAUpdate extends Extension {
     }
 
     onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
-        }
-
         for (const device of this.zigbee.getClients()) {
             // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
             // remove them.
@@ -67,7 +56,7 @@ class OTAUpdate extends Extension {
                     const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                        stringify({type: `ota_update`, message, meta}),
                     );
                 }
             }
@@ -140,7 +129,7 @@ class OTAUpdate extends Extension {
                 const meta = {status: `not_supported`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `ota_update`, message: error, meta}),
+                    stringify({type: `ota_update`, message: error, meta}),
                 );
             }
         } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
@@ -157,7 +146,7 @@ class OTAUpdate extends Extension {
                     const meta = {status: `checking_if_available`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message: msg, meta}),
+                        stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
@@ -173,7 +162,7 @@ class OTAUpdate extends Extension {
                         const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `ota_update`, message: msg, meta}),
+                            stringify({type: `ota_update`, message: msg, meta}),
                         );
                     }
 
@@ -189,7 +178,7 @@ class OTAUpdate extends Extension {
                         const meta = {status: `check_failed`, device: resolvedEntity.name};
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `ota_update`, message: error, meta}),
+                            stringify({type: `ota_update`, message: error, meta}),
                         );
                     }
                 }
@@ -202,7 +191,7 @@ class OTAUpdate extends Extension {
                     const meta = {status: `update_in_progress`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, msg, meta}),
+                        stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
@@ -221,14 +210,14 @@ class OTAUpdate extends Extension {
                         /* istanbul ignore else */
                         if (settings.get().advanced.legacy_api) {
                             const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
-                            this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message: msg, meta}));
+                            this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
                         }
                     };
 
                     const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
                     await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
                     const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
-                    const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
                     const msg = `Finished update of '${resolvedEntity.name}'` +
                         (to ? `, from '${fromS}' to '${toS}'` : ``);
                     logger.info(msg);
@@ -237,11 +226,12 @@ class OTAUpdate extends Extension {
                     this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
                     responseData.from = from_ ? utils.toSnakeCase(from_) : null;
                     responseData.to = to ? utils.toSnakeCase(to) : null;
+                    this.eventBus.emit(`devicesChanged`);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
                         const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
-                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
                     }
                 } catch (e) {
                     error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
@@ -253,7 +243,7 @@ class OTAUpdate extends Extension {
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
                         const meta = {status: `update_failed`, device: resolvedEntity.name};
-                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message: error, meta}));
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
                     }
                 }
             }
@@ -264,7 +254,7 @@ class OTAUpdate extends Extension {
         const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
         if (!triggeredViaLegacyApi) {
             const response = utils.getResponse(message, responseData, error);
-            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, JSON.stringify(response));
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
         }
 
         if (error) {

lib/extension/publish.js

@@ -5,35 +5,28 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
+    zigbeeHerdsmanConverters.toZigbeeConverters.effect,
     zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
     zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
     zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
 ];
 
 class EntityPublish extends Extension {
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < 20; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
     parseTopic(topic) {
         const match = topic.match(topicRegex);
         if (!match) {
@@ -64,7 +57,7 @@ class EntityPublish extends Extension {
                 const message = {friendly_name: entityKey};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `entity_not_found`, message}),
+                    stringify({type: `entity_not_found`, message}),
                 );
             }
 
@@ -78,6 +71,7 @@ class EntityPublish extends Extension {
         let options = {};
         let device = null;
         let definition = null;
+        let membersState = null;
 
         assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
         if (resolvedEntity.type === 'device') {
@@ -91,12 +85,22 @@ class EntityPublish extends Extension {
             definition = resolvedEntity.definition;
             target = resolvedEntity.endpoint;
             converters = resolvedEntity.definition.toZigbee;
-            options = {...settings.get().device_options, ...resolvedEntity.settings};
+            options = resolvedEntity.settings;
         } else {
-            converters = groupConverters;
             target = resolvedEntity.group;
-            options = {...settings.get().device_options, ...resolvedEntity.settings};
-            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
+            options = resolvedEntity.settings;
+            definition = resolvedEntity.group.members
+                .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
+            converters = new Set(groupConverters);
+            for (const d of definition) {
+                d.toZigbee.forEach(converters.add, converters);
+            }
+            converters = [...converters];
+            membersState = {};
+            for (const member of resolvedEntity.group.members) {
+                const ieeeAddr = member.getDevice().ieeeAddr;
+                membersState[ieeeAddr] = this.state.get(ieeeAddr);
+            }
         }
 
         // Convert the MQTT message to a Zigbee message.
@@ -111,23 +115,30 @@ class EntityPublish extends Extension {
             try {
                 json = JSON.parse(message);
             } catch (e) {
-                // Cannot be parsed to JSON, assume state message.
-                json = {state: message};
+                if (stateValues.includes(message.toLowerCase())) {
+                    json = {state: message};
+                } else {
+                    logger.error(`Invalid JSON '${message}', skipping...`);
+                    return false;
+                }
             }
         }
-
+        if (!json) {
+            logger.error(`Invalid JSON '${message}', skipping...`);
+            return;
+        }
 
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
+        const entityState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
             const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
+            const isOn = entityState && entityState.state === 'ON' ? true : false;
             if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
                 delete json.state;
                 logger.debug('Skipping state because of Home Assistant');
@@ -155,14 +166,13 @@ class EntityPublish extends Extension {
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const underscoreIndex = key.lastIndexOf('_');
-                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getEndpointNames().includes(possibleEndpointName)) {
-                    endpointName = possibleEndpointName;
-                    key = key.substring(0, underscoreIndex);
+                const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
+                if (endpointNameMatch) {
+                    endpointName = endpointNameMatch;
+                    key = key.replace(`_${endpointNameMatch}`, '');
+
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
-
                     if (!actualTarget) {
                         logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
                         continue;
@@ -174,8 +184,9 @@ class EntityPublish extends Extension {
             if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters[endpointOrGroupID].includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
+            if (topic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
 
@@ -191,7 +202,8 @@ class EntityPublish extends Extension {
                 message: json,
                 logger,
                 device,
-                state: deviceState,
+                state: entityState,
+                membersState,
                 mapped: definition,
             };
 
@@ -199,7 +211,10 @@ class EntityPublish extends Extension {
                 if (topic.type === 'set' && converter.convertSet) {
                     logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
-                    if (result && result.state) {
+                    const isGroup = actualTarget.constructor.name == 'Group';
+                    // Optimistic behaviour of groups and devices is different.
+                    const publishOptimistic = isGroup || (!options.hasOwnProperty('optimistic') || options.optimistic);
+                    if (result && result.state && publishOptimistic) {
                         const msg = result.state;
 
                         if (endpointName) {
@@ -214,6 +229,12 @@ class EntityPublish extends Extension {
                         this.publishEntityState(resolvedEntity.settings.ID, msg);
                     }
 
+                    if (result && result.membersState && publishOptimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            this.publishEntityState(ieeeAddr, state);
+                        }
+                    }
+
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
                     // So here we re-read the value after a specified timeout, this timeout could for example be the
                     // transition time of a color change or for forcing a state read for devices that don't
@@ -245,7 +266,7 @@ class EntityPublish extends Extension {
                     const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                        stringify({type: `zigbee_publish_error`, message, meta}),
                     );
                 }
             }

lib/extension/receive.js

@@ -3,6 +3,7 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
@@ -73,35 +74,13 @@ class Receive extends Extension {
             return false;
         }
 
-        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
-            logger.debug('Ignoring message from coordinator');
-            return false;
-        }
-
-        /**
-         * Don't handle re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The source address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the source address of the end device.
-         * As the same message is also received directly from the end device, it makes no sense
-         * to handle these messages.
-         */
-        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
-        if (utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && hasGroupID) {
-            logger.debug('Skipping re-transmitted Xiaomi message');
-            return false;
-        }
-
         if (!resolvedEntity.definition) {
             if (data.device.interviewing) {
                 logger.debug(`Skipping message, definition is undefined and still interviewing`);
             } else {
-                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.modelID}' ` +
+                    `and manufacturer name '${data.device.manufacturerName}'`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
             }
 
@@ -112,6 +91,24 @@ class Receive extends Extension {
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
+        /**
+         * Handling of re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The network address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the network address of the end device.
+         */
+        if (type === 'message' && utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && data.groupID) {
+            logger.debug('Handling re-transmitted Xiaomi message');
+            data.device = this.zigbee.getDeviceByNetworkAddress(data.groupID);
+            resolvedEntity = this.zigbee.resolveEntity(data.device);
+        }
+
         if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
@@ -125,25 +122,17 @@ class Receive extends Extension {
         if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
             logger.debug(
                 `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
-                `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
+                `and type '${data.type}' and data '${stringify(data.data)}'`,
             );
             return;
         }
 
-        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
-        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
-
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
         // - If a payload is returned publish it to the MQTT broker
         // - If NO payload is returned do nothing. This is for non-standard behaviour
         //   for e.g. click switches where we need to count number of clicks and detect long presses.
         const publish = (payload) => {
-            // Add device linkquality.
-            if (data.hasOwnProperty('linkquality')) {
-                payload.linkquality = data.linkquality;
-            }
-
             if (settings.get().advanced.elapsed) {
                 const now = Date.now();
                 if (this.elapsed[data.device.ieeeAddr]) {
@@ -154,18 +143,22 @@ class Receive extends Extension {
             }
 
             // Check if we have to debounce
-            if (debounce) {
-                this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
+            if (resolvedEntity.settings.debounce) {
+                this.publishDebounce(
+                    data.device.ieeeAddr, payload, resolvedEntity.settings.debounce,
+                    resolvedEntity.settings.debounce_ignore,
+                );
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
             }
         };
 
-        const meta = {device: data.device, logger};
+        const meta = {device: data.device, logger, state: this.state.get(data.device.ieeeAddr)};
         let payload = {};
         converters.forEach((converter) => {
-            const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
-            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
+            const converted = converter.convert(
+                resolvedEntity.definition, data, publish, resolvedEntity.settings, meta,
+            );
             if (converted) {
                 payload = {...payload, ...converted};
             }

lib/extension/report.js

@@ -16,6 +16,7 @@ const devicesNotSupportingReporting = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0115'),
 ];
 
 const reportKey = 1;
@@ -80,8 +81,13 @@ const pollOnMessage = [
         },
         // Read the following attributes
         read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
-        // When the bound devices/members of group have the following manufacturerID
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
     },
     {
         key: 2,
@@ -97,7 +103,12 @@ const pollOnMessage = [
             ],
         },
         read: {cluster: 'genOnOff', attributes: ['onOff']},
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+        manufacturerIDs: [
+            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
     },
 ];
 
@@ -187,6 +198,11 @@ class Report extends Extension {
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
         if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
 
+        // These do not support reproting.
+        // https://github.com/Koenkk/zigbee-herdsman/issues/110
+        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
+        if (device.manufacturerName === 'Philips' && philipsIgnoreSw.includes(device.softwareBuildID)) return false;
+
         if (resolvedEntity.device.interviewing === true) return false;
         if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
         // Gledopto devices don't support reporting.
@@ -258,7 +274,7 @@ class Report extends Extension {
 
             for (const endpoint of toPoll) {
                 for (const poll of polls) {
-                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
                         continue;
                     }
 

lib/mqtt.js

@@ -8,6 +8,7 @@ class MQTT extends events.EventEmitter {
     constructor() {
         super();
         this.onMessage = this.onMessage.bind(this);
+        this.publishedTopics = new Set();
     }
 
     async connect() {
@@ -18,7 +19,7 @@ class MQTT extends events.EventEmitter {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
                 payload: 'offline',
-                retain: true,
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
             },
         };
 
@@ -44,8 +45,11 @@ class MQTT extends events.EventEmitter {
         }
 
         if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
         }
 
         if (mqttSettings.client_id) {
@@ -69,9 +73,11 @@ class MQTT extends events.EventEmitter {
         return new Promise((resolve) => {
             this.client = mqtt.connect(mqttSettings.server, options);
 
+            const self = this;
             this.client.on('connect', () => {
                 logger.info('Connected to MQTT server');
-                this.publish('bridge/state', 'online', {retain: true, qos: 0});
+                self.subscribe(`${settings.get().mqtt.base_topic}/#`);
+                self.publish('bridge/state', 'online', {retain: true, qos: 0});
                 resolve();
             });
 
@@ -92,17 +98,24 @@ class MQTT extends events.EventEmitter {
     }
 
     onMessage(topic, message) {
-        this.emit('message', {topic, message: message.toString()});
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            this.emit('message', {topic, message: message + ''});
+        }
     }
 
     isConnected() {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -115,8 +128,16 @@ class MQTT extends events.EventEmitter {
             logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
         }
 
+        const actualOptions = {...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
+
         return new Promise((resolve) => {
-            this.client.publish(topic, payload, options, () => resolve());
+            this.client.publish(topic, payload, actualOptions, () => {
+                this.emit('publishedMessage', {topic, payload, options});
+                resolve();
+            });
         });
     }
 }

lib/state.js

@@ -57,7 +57,11 @@ class State {
         if (settings.get().advanced.cache_state_persistent) {
             logger.debug(`Saving state to file ${this.file}`);
             const json = JSON.stringify(this.state, null, 4);
-            fs.writeFileSync(this.file, json, 'utf8');
+            try {
+                fs.writeFileSync(this.file, json, 'utf8');
+            } catch (e) {
+                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
+            }
         } else {
             logger.debug(`Not saving state`);
         }

lib/util/logger.js

@@ -115,24 +115,27 @@ function cleanup() {
     }
 }
 
+// Print to user what logging is enabled
+function logOutput() {
+    if (output.includes('file')) {
+        if (output.includes('console')) {
+            logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+        } else {
+            logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+        }
+        logger.cleanup();
+    } else if (output.includes('console')) {
+        logger.info(`Logging to console only'`);
+    }
+}
+
 logger.cleanup = cleanup;
+logger.logOutput = logOutput;
 logger.getLevel = () => transportsToUse[0].level;
 logger.setLevel = (level) => {
     transportsToUse.forEach((transport) => transport.level = level);
 };
 
-// Print to user what logging is enabled
-if (output.includes('file')) {
-    if (output.includes('console')) {
-        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
-    } else {
-        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
-    }
-    logger.cleanup();
-} else if (output.includes('console')) {
-    logger.info(`Logging to console only'`);
-}
-
 // winston.config.syslog.levels doesnt have warn, but is required for syslog.
 /* istanbul ignore next */
 logger.warn = (message) => logger.warning(message);

lib/util/settings.js

@@ -1,5 +1,6 @@
 const data = require('./data');
 const utils = require('./utils');
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
 const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
 const objectAssignDeep = require(`object-assign-deep`);
 const path = require('path');
@@ -16,6 +17,11 @@ const defaults = {
     permit_join: false,
     mqtt: {
         include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
     },
     serial: {
         disable_led: false,
@@ -59,6 +65,7 @@ const defaults = {
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         adapter_concurrent: null,
+        adapter_delay: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
@@ -152,6 +159,7 @@ const schema = {
                 reject_unauthorized: {type: 'boolean'},
                 include_device_information: {type: 'boolean'},
                 version: {type: 'number'},
+                force_disable_retain: {type: 'boolean'},
             },
             required: ['base_topic', 'server'],
         },
@@ -160,7 +168,7 @@ const schema = {
             properties: {
                 port: {type: ['string', 'null']},
                 disable_led: {type: 'boolean'},
-                adapter: {type: 'string', enum: ['deconz', 'zstack']},
+                adapter: {type: 'string', enum: ['deconz', 'zstack', 'zigate']},
             },
         },
         blocklist: {type: 'array', items: {type: 'string'}},
@@ -207,6 +215,7 @@ const schema = {
                 homeassistant_status_topic: {type: 'string'},
                 timestamp_format: {type: 'string'},
                 adapter_concurrent: {type: 'number'},
+                adapter_delay: {type: 'number'},
             },
         },
         map_options: {
@@ -316,15 +325,27 @@ function write() {
         }
     }
 
-    if (typeof actual.devices === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
-        toWrite.devices = actual.devices;
-    }
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type) => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            const content = objectAssignDeep.noMutate(settings[type]);
 
-    if (typeof actual.groups === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
-        toWrite.groups = actual.groups;
-    }
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f, i) => i !== 0)
+                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c) => Object.keys(c))
+                    .forEach((k) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
 
     yaml.writeIfChanged(file, toWrite);
 
@@ -360,14 +381,9 @@ function validate() {
 
     // Verify that all friendly names are unique
     const names = [];
-    const postfixes = utils.getEndpointNames();
     const check = (name) => {
         if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
-        if (postfixes.includes(name)) errors.push(`Following friendly_name are not allowed: '${postfixes}'`);
-        if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
-        if (name.includes('#') || name.includes('+')) {
-            errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
-        }
+        errors.push(...utils.validateFriendlyName(name));
         names.push(name);
     };
 
@@ -424,23 +440,70 @@ function read() {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    // Read devices/groups configuration from separate file.
-    if (typeof s.devices === 'string') {
-        const file = data.joinPath(s.devices);
-        s.devices = yaml.readIfExists(file) || {};
-    }
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type) => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
 
-    if (typeof s.groups === 'string') {
-        const file = data.joinPath(s.groups);
-        s.groups = yaml.readIfExists(file) || {};
-    }
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
 
     return s;
 }
 
+function applyEnvironmentVariables(settings) {
+    const iterate = (obj, path) => {
+        Object.keys(obj).forEach((key) => {
+            if (key !== 'type') {
+                if (key !== 'properties') {
+                    const type = (obj[key].type || 'object').toString();
+                    const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
+                    const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
+                    if (process.env[envVariableName]) {
+                        const setting = path.reduce((acc, val, index) => {
+                            acc[val] = acc[val] || {};
+                            return acc[val];
+                        }, settings);
+
+                        if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
+                            setting[key] = JSON.parse(process.env[envVariableName]);
+                        }
+                        if (type.indexOf('number') >= 0) {
+                            setting[key] = process.env[envVariableName] * 1;
+                        }
+                        if (type.indexOf('boolean') >= 0) {
+                            setting[key] = process.env[envVariableName].toLowerCase() === 'true';
+                        }
+                        if (type.indexOf('string') >= 0) {
+                            setting[key] = process.env[envVariableName];
+                        }
+                    }
+                }
+
+                if (typeof obj[key] === 'object') {
+                    const newPath = [...path];
+                    if (key !== 'properties') {
+                        newPath.push(key);
+                    }
+                    iterate(obj[key], newPath);
+                }
+            }
+        });
+    };
+    iterate(schema.properties, []);
+}
+
 function get() {
     if (!_settings) {
         _settings = read();
+        applyEnvironmentVariables(_settings);
     }
 
     return _settings;
@@ -743,11 +806,14 @@ module.exports = {
     // For tests only
     _write: write,
     _reRead: () => {
-        _settings = read();
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+        _settings = null;
+        get();
+        _settingsWithDefaults = null;
+        getWithDefaults();
     },
     _clear: () => {
         _settings = null;
+        _settingsWithDefaults = null;
     },
     _getDefaults: () => defaults,
 };

lib/util/utils.js

@@ -1,6 +1,8 @@
 const equals = require('fast-deep-equal/es6');
 const humanizeDuration = require('humanize-duration');
 const data = require('./data');
+const vm = require('vm');
+const fs = require('fs');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -37,6 +39,8 @@ const endpointNames = [
     'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
     'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool',
 ];
 
 function capitalize(s) {
@@ -67,6 +71,14 @@ async function getZigbee2mqttVersion() {
     });
 }
 
+async function getDependencyVersion(depend) {
+    return new Promise((resolve, reject) => {
+        const packageJSON = require('../../node_modules/'+depend+'/package.json');
+        const version = packageJSON.version;
+        resolve({version});
+    });
+}
+
 function formatDate(date, type) {
     let result;
 
@@ -111,14 +123,8 @@ function equalsPartial(object, expected) {
     return true;
 }
 
-function getObjectsProperty(objects, key, defaultValue) {
-    for (const object of objects) {
-        if (object.hasOwnProperty(key)) {
-            return object[key];
-        }
-    }
-
-    return defaultValue;
+function getObjectProperty(object, key, defaultValue) {
+    return object.hasOwnProperty(key) ? object[key] : defaultValue;
 }
 
 function getResponse(request, data, error) {
@@ -142,13 +148,20 @@ function* getExternalConvertersDefinitions(settings) {
     const externalConverters = settings.get().external_converters;
 
     for (const moduleName of externalConverters) {
-        let converterModule = moduleName;
+        let converter;
 
         if (moduleName.endsWith('.js')) {
-            converterModule = data.joinPath(moduleName.split('.')[0]);
+            const sandbox = {
+                require,
+                module: {},
+            };
+            const converterCode = fs.readFileSync(data.joinPath(moduleName), {encoding: 'utf8'});
+            vm.runInNewContext(converterCode, sandbox);
+            converter = sandbox.module.exports;
+        } else {
+            converter = require(moduleName);
         }
 
-        const converter = require(converterModule);
         if (Array.isArray(converter)) {
             for (const item of converter) {
                 yield item;
@@ -174,13 +187,24 @@ function toSnakeCase(value) {
     }
 }
 
+function validateFriendlyName(name) {
+    const errors = [];
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+    return errors;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
+    getDependencyVersion,
     objectHasProperties,
     toSnakeCase,
-    getObjectsProperty,
+    getObjectProperty,
     getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
         return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
@@ -193,4 +217,5 @@ module.exports = {
     capitalize,
     parseJSON,
     getExternalConvertersDefinitions,
+    validateFriendlyName,
 };

lib/util/yaml.js

@@ -14,8 +14,8 @@ function read(file) {
     }
 }
 
-function readIfExists(file) {
-    return fs.existsSync(file) ? read(file) : null;
+function readIfExists(file, default_=null) {
+    return fs.existsSync(file) ? read(file) : default_;
 }
 
 function writeIfChanged(file, content) {

lib/zigbee.js

@@ -7,6 +7,7 @@ const utils = require('./util/utils');
 const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
@@ -18,7 +19,8 @@ class Zigbee extends events.EventEmitter {
     }
 
     async start() {
-        logger.info(`Starting zigbee-herdsman...`);
+        const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
+        logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
         const herdsmanSettings = {
             network: {
                 panID: settings.get().advanced.pan_id,
@@ -37,12 +39,13 @@ class Zigbee extends events.EventEmitter {
             },
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
+                delay: settings.get().advanced.adapter_delay,
             },
         };
 
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
-        logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
+        logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
 
         if (herdsmanSettings.network.networkKey === 'GENERATE') {
             const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
@@ -67,8 +70,8 @@ class Zigbee extends events.EventEmitter {
         this.herdsman.on('message', (data) => this.emit('event', 'message', data));
 
         logger.info('zigbee-herdsman started');
-        logger.info(`Coordinator firmware version: '${JSON.stringify(await this.getCoordinatorVersion())}'`);
-        logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
+        logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
         for (const device of this.getClients()) {
             // If a passlist is used, all other device will be removed from the network.
@@ -115,12 +118,16 @@ class Zigbee extends events.EventEmitter {
         logger.info('zigbee-herdsman stopped');
     }
 
-    async permitJoin(permit) {
+    async permitJoin(permit, resolvedEntity) {
         permit ?
-            logger.info('Zigbee: allowing new devices to join.') :
+            logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`) :
             logger.info('Zigbee: disabling joining new devices.');
 
-        await this.herdsman.permitJoin(permit);
+        if (resolvedEntity && permit) {
+            await this.herdsman.permitJoin(permit, resolvedEntity.device);
+        } else {
+            await this.herdsman.permitJoin(permit);
+        }
     }
 
     async getPermitJoin() {
@@ -139,6 +146,10 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
     }
 
+    getDeviceByNetworkAddress(networkAddress) {
+        return this.herdsman.getDeviceByNetworkAddress(networkAddress);
+    }
+
     getDevicesByType(type) {
         return this.herdsman.getDevicesByType(type);
     }
@@ -161,6 +172,7 @@ class Zigbee extends events.EventEmitter {
             `Wrong type '${typeof key}'`,
         );
 
+        const deviceOptions = settings.get().device_options;
         if (typeof key === 'string' || typeof key === 'number') {
             if (typeof key === 'number') {
                 key = key.toString();
@@ -215,12 +227,13 @@ class Zigbee extends events.EventEmitter {
                 }
 
                 return {
-                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
+                    type: 'device', device, endpoint, settings: {...deviceOptions, ...entity},
+                    name: entity.friendlyName, definition,
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
-                return {type: 'group', group, settings: entity, name: entity.friendlyName};
+                return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
             }
         } else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
@@ -228,7 +241,7 @@ class Zigbee extends events.EventEmitter {
                 type: 'device',
                 device: key,
                 endpoint: key.endpoints[0],
-                settings: setting,
+                settings: {...deviceOptions, ...(setting || {})},
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
                 definition: zigbeeHerdsmanConverters.findByDevice(key),
             };
@@ -237,7 +250,7 @@ class Zigbee extends events.EventEmitter {
             return {
                 type: 'group',
                 group: key,
-                settings: setting,
+                settings: {...deviceOptions, ...(setting || {})},
                 name: setting ? setting.friendlyName : key.groupID,
             };
         }
@@ -280,8 +293,20 @@ class Zigbee extends events.EventEmitter {
         }
     }
 
-    async touchlinkFactoryReset() {
-        return this.herdsman.touchlinkFactoryReset();
+    async touchlinkFactoryResetFirst() {
+        return this.herdsman.touchlinkFactoryResetFirst();
+    }
+
+    async touchlinkFactoryReset(ieeeAddr, channel) {
+        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
+    }
+
+    async touchlinkIdentify(ieeeAddr, channel) {
+        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
+    }
+
+    async touchlinkScan() {
+        return this.herdsman.touchlinkScan();
     }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.14.3",
+  "version": "1.16.2",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13 || ^14"
+    "node": "^10 || ^12 || ^14 || ^15"
   },
   "keywords": [
     "xiaomi",
@@ -33,22 +33,27 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^6.12.3",
+    "ajv": "^6.12.4",
     "debounce": "^1.2.0",
     "fast-deep-equal": "^3.1.3",
+    "finalhandler": "^1.1.2",
     "git-last-commit": "^1.0.0",
     "humanize-duration": "^3.23.1",
     "js-yaml": "^3.14.0",
+    "json-stable-stringify-without-jsonify": "=1.0.1",
     "mkdir-recursive": "^0.4.0",
     "moment": "^2.27.0",
-    "mqtt": "^4.1.0",
+    "mqtt": "4.2.1",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
     "semver": "^7.3.2",
+    "serve-static": "^1.14.1",
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
-    "zigbee-herdsman": "0.12.116",
-    "zigbee-herdsman-converters": "12.0.161"
+    "ws": "^7.3.1",
+    "zigbee-herdsman": "0.13.37",
+    "zigbee-herdsman-converters": "13.0.7",
+    "zigbee2mqtt-frontend": "0.2.100"
   },
   "devDependencies": {
     "eslint": "*",
@@ -69,5 +74,8 @@
     "collectCoverageFrom": [
       "lib/**"
     ]
+  },
+  "bin": {
+    "zigbee2mqtt": "./cli.js"
   }
 }

scripts/dumpHomeAssistantMapping.js

@@ -0,0 +1,35 @@
+const HomeAssistant = require('../lib/extension/homeassistant');
+const stringify = require('json-stable-stringify-without-jsonify');
+const homeassistant = new HomeAssistant(null, null, null, null, {on: () => {}});
+const mapping = Object.entries(homeassistant._getMapping());
+mapping.sort((a, b) => {
+    if (a[0] > b[0]) {
+        return -1;
+    }
+    if (b[0] > a[0]) {
+        return 1;
+    }
+    return 0;
+});
+
+for (const map of mapping) {
+    map[1].sort((a, b) => {
+        return JSON.stringify(a).localeCompare(JSON.stringify(b));
+    });
+
+    const toDelete = [];
+    for (const entry of map[1]) {
+        if (entry.type === 'light') {
+            if (entry.discovery_payload.brightness === false) delete entry.discovery_payload.brightness;
+            if (entry.discovery_payload.color_temp === false) delete entry.discovery_payload.color_temp;
+            if (entry.discovery_payload.xy === false) delete entry.discovery_payload.xy;
+            if (entry.discovery_payload.hs === false) delete entry.discovery_payload.hs;
+        } else if (entry.object_id === 'linkquality') {
+            toDelete.push(entry);
+        }
+    }
+
+    toDelete.forEach((d) => map[1].splice(map[1].indexOf(d), 1));
+
+    console.log(`${map[0]} - ${stringify(map[1])}`);
+}

scripts/zStackEraseAllNvMem.js

@@ -0,0 +1,74 @@
+const {ZnpCommandStatus} = require('zigbee-herdsman/dist/adapter/z-stack/constants/common');
+const {ZnpVersion} = require('zigbee-herdsman/dist/adapter/z-stack/adapter/tstype');
+const {Subsystem} = require('zigbee-herdsman/dist/adapter/z-stack/unpi/constants');
+const {Znp} = require('zigbee-herdsman/dist/adapter/z-stack/znp');
+
+
+class ZStackNvMemEraser {
+    constructor(device) {
+        this.znp = new Znp(device, 115200, false);
+    }
+
+    async start() {
+        await this.znp.open();
+        const attempts = 3;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                await this.znp.request(Subsystem.SYS, 'ping', {capabilities: 1});
+                break;
+            } catch (e) {
+                if (attempts - 1 === i) {
+                    throw new Error(`Failed to connect to the adapter (${e})`);
+                }
+            }
+        }
+        // Old firmware did not support version, assume it's Z-Stack 1.2 for now.
+        try {
+            this.version = (await this.znp.request(Subsystem.SYS, 'version', {})).payload;
+        } catch (e) {
+            console.log(`Failed to get zStack version, assuming 1.2`);
+            this.version = {'transportrev': 2, 'product': 0, 'majorrel': 2,
+                'minorrel': 0, 'maintrel': 0, 'revision': ''};
+        }
+
+        console.log(`Detected znp version '${ZnpVersion[this.version.product]}' (${JSON.stringify(this.version)})`);
+
+        await this.clearAllNvMemItems();
+
+        process.exit(0);
+    }
+
+    async clearAllNvMemItems() {
+        let maxNvMemId;
+        switch (this.version.product) {
+        case ZnpVersion.zStack12: maxNvMemId = 0x0302; break;
+        case ZnpVersion.zStack30x: maxNvMemId = 0x033F; break;
+        case ZnpVersion.zStack3x0: maxNvMemId = 0x032F; break;
+        }
+
+        console.log(`Clearing all NVMEM items, from 0 to ${maxNvMemId}`);
+        for (let id=0; id<=maxNvMemId; id++) {
+            const lengthRes = await this.znp.request(Subsystem.SYS, 'osalNvLength',{id: id});
+            if (lengthRes.payload['length']!=0) {
+                console.log(`NVMEM item #${id} - need to delete, size: ${lengthRes.payload['length']}`);
+                await this.znp.request(Subsystem.SYS, 'osalNvDelete',
+                    {id: id, len: lengthRes.payload['length']},
+                    null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+            } else {
+                console.log(`NVMEM item #${id} - is free`);
+            }
+        }
+        console.log('Clearing all NVMEM items finished');
+    }
+}
+const processArgs = process.argv.slice(2);
+if (processArgs.length != 1) {
+    console.log('ZStack NVMEM eraser.');
+    console.log('Usage:');
+    console.log('   node zStackEraseAllNvMem.js <SERIAL_PORT>');
+    process.exit(1);
+}
+
+const eraser = new ZStackNvMemEraser(processArgs[0]);
+
+eraser.start();

test/availability.test.js

@@ -27,7 +27,8 @@ describe('Availability', () => {
         settings._reRead();
         data.writeEmptyState();
         jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10)
+        settings.set(['advanced', 'availability_timeout'], 10);
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         mocksClear.forEach((m) => m.mockClear());
         await controller.start();
@@ -80,7 +81,7 @@ describe('Availability', () => {
     });
 
     it('Should publish availabilty online and query state on reconnect', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
+        const device = zigbeeHerdsman.devices.E11_G13;
         const endpoint = device.getEndpoint(1);
         device.ping.mockImplementationOnce(() => {throw new Error('failed')});
         jest.advanceTimersByTime(11 * 1000);
@@ -89,16 +90,15 @@ describe('Availability', () => {
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
+            'zigbee2mqtt/bulb_enddevice/availability',
           'online',
           { retain: true, qos: 0 },
           expect.any(Function)
         );
 
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
     });
 
     it('Should fail gracefully when quering state after reconnect fails', async () => {
@@ -172,10 +172,11 @@ describe('Availability', () => {
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(4);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorTemperature']);
     });
 
     it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
@@ -397,4 +398,15 @@ describe('Availability', () => {
           expect.any(Function)
         );
     });
+
+    it('Should clear retained availability topic when device is remove', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
 });

test/bind.test.js

@@ -5,6 +5,7 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
+const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('Bind', () => {
     let controller;
@@ -32,28 +33,30 @@ describe('Bind', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
-    });
-
     it('Should bind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
     });
 
     it('Should bind only specifief clusters', async () => {
@@ -61,13 +64,13 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color', clusters: ["genOnOff"]}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color', clusters: ["genOnOff"]}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -77,12 +80,12 @@ describe('Bind', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         logger.error.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'button'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'button'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(0);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
+            stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -92,7 +95,7 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'bulb_color'}));
         await flushPromises();
         expect(endpoint.unbind).toHaveBeenCalledTimes(3);
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
@@ -100,7 +103,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/unbind',
-            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -111,7 +114,7 @@ describe('Bind', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         endpoint.unbind.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: 'Coordinator'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'Coordinator'}));
         await flushPromises();
         expect(endpoint.unbind).toHaveBeenCalledTimes(3);
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
@@ -119,7 +122,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/unbind',
-            JSON.stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -129,7 +132,7 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'group_1'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'group_1'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
@@ -137,7 +140,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -147,7 +150,7 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: '1'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: '1'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
@@ -155,7 +158,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -166,12 +169,12 @@ describe('Bind', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         endpoint.bind.mockImplementation(() => {throw new Error('failed')});
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -181,13 +184,13 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
         const endpoint = device.getEndpoint(2);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -197,13 +200,13 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
         const endpoint = device.getEndpoint(2);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote/ep2', to: 'wall_switch'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch'}));
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -213,7 +216,7 @@ describe('Bind', () => {
         const target = 'default_bind_group';
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: target}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: target}));
         await flushPromises();
         expect(endpoint.unbind).toHaveBeenCalledTimes(3);
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
@@ -221,7 +224,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/unbind',
-            JSON.stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -231,11 +234,11 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote_not_existing', to: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote_not_existing', to: 'bulb_color'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
+            stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -245,11 +248,11 @@ describe('Bind', () => {
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color_not_existing'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color_not_existing'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            JSON.stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
+            stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -265,7 +268,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -296,7 +299,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -317,7 +320,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -337,7 +340,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -357,7 +360,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -411,7 +414,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');

test/configure.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
@@ -109,38 +110,38 @@ describe('Configure', () => {
         expectRemoteConfigured();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            JSON.stringify({"data":{"id": "remote"},"status":"ok"}),
+            stringify({"data":{"id": "remote"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
 
     it('Fail to configure via MQTT when device does not exist', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "not_existing_device"}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "not_existing_device"}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            JSON.stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
+            stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
 
     it('Fail to configure via MQTT when configure fails', async () => {
         zigbeeHerdsman.devices.remote.getEndpoint(1).bind.mockImplementationOnce(async () => {throw new Error('Bind timeout after 10s')});
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "remote"}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "remote"}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            JSON.stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
+            stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
 
     it('Fail to configure via MQTT when device has no configure', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "bulb", transaction: 20}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb", transaction: 20}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
+            stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });

test/controller.test.js

@@ -6,6 +6,7 @@ const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
 const flushPromises = () => new Promise(setImmediate);
 const tmp = require('tmp');
 const mocksClear = [
@@ -30,7 +31,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -42,8 +43,8 @@ describe('Controller', () => {
         expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',{ retain: true, qos: 0 }, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}),{ retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', stringify({"brightness":255}), { retain: true, qos: 0 }, expect.any(Function));
     });
 
     it('Start controller when permit join fails', async () => {
@@ -104,8 +105,8 @@ describe('Controller', () => {
         data.writeDefaultState();
         await controller.start();
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", stringify({"brightness":255}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Start controller should not publish cached states when disabled', async () => {
@@ -148,7 +149,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(2);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
-        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"state\":\"ON\",\"brightness\":50,\"color_temp\":370,\"linkquality\":99,\"color\":{\"r\":100,\"g\":50,\"b\":10},\"dummy\":{\"1\":\"yes\",\"2\":\"no\"}}");
+        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"brightness\":50,\"color\":{\"b\":10,\"g\":50,\"r\":100},\"color_temp\":370,\"dummy\":{\"1\":\"yes\",\"2\":\"no\"},\"linkquality\":99,\"state\":\"ON\"}");
         controller.mqtt.client.reconnecting = false;
     });
 
@@ -296,7 +297,7 @@ describe('Controller', () => {
         const payload = {device};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('acceptJoiningDeviceHandler reject device on blocklist', async () => {
@@ -354,7 +355,7 @@ describe('Controller', () => {
         zigbeeHerdsman.events.deviceJoined(payload);
         zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee deviceInterview started', async () => {
@@ -363,7 +364,7 @@ describe('Controller', () => {
         const payload = {device, status: 'started'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview failed', async () => {
@@ -372,7 +373,7 @@ describe('Controller', () => {
         const payload = {device, status: 'failed'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview successful supported', async () => {
@@ -381,7 +382,7 @@ describe('Controller', () => {
         const payload = {device, status: 'successful'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview successful not supported', async () => {
@@ -390,7 +391,7 @@ describe('Controller', () => {
         const payload = {device, status: 'successful'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee event device announce', async () => {
@@ -400,7 +401,7 @@ describe('Controller', () => {
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
         expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and settings)', async () => {
@@ -412,7 +413,7 @@ describe('Controller', () => {
         const payload = {ieeeAddr: device.ieeeAddr};
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and NOT settings)', async () => {
@@ -423,7 +424,7 @@ describe('Controller', () => {
         const payload = {ieeeAddr: device.ieeeAddr};
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Publish entity state attribute output', async () => {
@@ -436,8 +437,8 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color", '100,50,10', {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", 'yes', {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", 'no', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", "yes", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", "no", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test1", '', {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test", '', {"qos": 0, "retain": true}, expect.any(Function));
     });
@@ -453,7 +454,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
 
@@ -467,7 +468,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state attribute_json output filtered (device_options)', async () => {
@@ -480,7 +481,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state with device information', async () => {
@@ -489,12 +490,12 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": true}, expect.any(Function));
 
         // Unsupported device should have model "unknown"
         await controller.publishEntityState('unsupported2', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state without retain', async () => {
@@ -503,7 +504,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state with retain', async () => {
@@ -512,7 +513,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Should publish entity state with expiring retention', async () => {
@@ -523,7 +524,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
     });
 
     it('Publish entity state no empty messages', async () => {
@@ -544,12 +545,21 @@ describe('Controller', () => {
         await controller.publishEntityState('bulb', {brightness: 200});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", JSON.stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", JSON.stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
         await controller.stop();
         expect(data.stateExists()).toBeFalsy();
     });
 
+    it('Shouldnt crash when it cannot save state', async () => {
+        data.removeState();
+        await controller.start();
+        logger.error.mockClear();
+        controller.state.file = "/";
+        await controller.state.save();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to write state to '/' (EISDIR: illegal operation on a directory, open '/')`);
+    });
+
     it('Publish should not cache when set', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeEmptyState();
@@ -559,8 +569,8 @@ describe('Controller', () => {
         await controller.publishEntityState('bulb', {brightness: 200});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"state\":\"ON\"}", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"brightness\":200}", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish should not do anything for unknown entity', async () => {
@@ -588,4 +598,24 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
     });
 
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
+
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
 });

test/frontend.test.js

@@ -0,0 +1,215 @@
+const data = require('./stub/data');
+require('./stub/logger');
+require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mockHTTP = {
+    implementation: {
+        listen: jest.fn(),
+        on: (event, handler) => {mockHTTP.events[event] = handler},
+        close: jest.fn().mockImplementation((cb) => cb()),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockWSocket = {
+    close: jest.fn(),
+};
+
+const mockWS = {
+    implementation: {
+        clients: [],
+        on: (event, handler) => {mockWS.events[event] = handler},
+        handleUpgrade: jest.fn().mockImplementation((request, socket, head, cb) => {
+            cb(mockWSocket)
+        }),
+        emit: jest.fn(),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockNodeStatic = {
+    implementation: jest.fn(),
+    variables: {},
+    events: {},
+};
+
+jest.mock('http', () => ({
+    createServer: jest.fn().mockImplementation((onRequest) => {
+        mockHTTP.variables.onRequest = onRequest;
+        return mockHTTP.implementation;
+    }),
+}));
+
+jest.mock("serve-static", () =>
+    jest.fn().mockImplementation((path) => {
+        mockNodeStatic.variables.path = path
+        return mockNodeStatic.implementation
+    })
+);
+
+jest.mock('zigbee2mqtt-frontend', () => ({
+    getPath: () => 'my/dummy/path',
+}));
+
+jest.mock('ws', () => ({
+    OPEN: 'open',
+    Server: jest.fn().mockImplementation(() => {
+        return mockWS.implementation;
+    }),
+}));
+
+describe('Frontend', () => {
+    let controller;
+
+    beforeEach(async () => {
+        mockWS.implementation.clients = [];
+        data.writeDefaultConfiguration();
+        data.writeDefaultState();
+        settings._reRead();
+        settings.set(['experimental'], {new_api: true});
+        settings.set(['frontend'], {port: 8081, host: "127.0.0.1"});
+        settings.set(['homeassistant'], true);
+        zigbeeHerdsman.devices.bulb.linkquality = 10;
+    });
+
+    afterEach(async() => {
+        delete zigbeeHerdsman.devices.bulb.linkquality;
+    });
+
+    it('Start/stop', async () => {
+        controller = new Controller();
+        await controller.start();
+        expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+
+        const mockWSClient = {
+            implementation: {
+                close: jest.fn(),
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await controller.stop();
+        expect(mockWSClient.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
+    });
+
+    it('Websocket interaction', async () => {
+        controller = new Controller();
+        await controller.start();
+
+        // Connect
+        const mockWSClient = {
+            implementation: {
+                on: (event, handler) => {mockWSClient.events[event] = handler},
+                send: jest.fn(),
+                readyState: 'open',
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(48);
+
+        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[0])).toStrictEqual({topic: 'bridge/state', payload: 'online'});
+        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}});
+
+        // Message
+        MQTT.publish.mockClear();
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        mockWSClient.events.message(undefined);
+        mockWSClient.events.message("");
+        mockWSClient.events.message(null);
+        await flushPromises();
+
+        // Received message on socket
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(4);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON'}}));
+
+        // Shouldnt set when not ready
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'close';
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(0);
+
+        // Send last seen on connect
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'open';
+        settings.set(['advanced'], {last_seen: 'ISO_8601'});
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}});
+    });
+
+    it('onReques/onUpgrade', async () => {
+        controller = new Controller();
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: 'http://localhost:8080/api'}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "http://localhost:8080/api"}, mockSocket, 3, expect.any(Function));
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](99);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', 99, {"url": "http://localhost:8080/api"});
+
+        mockHTTP.variables.onRequest(1, 2);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledTimes(1);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledWith(1, 2, expect.any(Function));
+    });
+
+    it('Static server', async () => {
+        controller = new Controller();
+        await controller.start();
+
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+    });
+
+    it('Authentification', async () => {
+        const authToken = 'sample-secure-token'
+        settings.set(['frontend'], {auth_token: authToken});
+        controller = new Controller();
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: '/api'}, mockSocket, mockWSocket);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "/api"}, mockSocket, mockWSocket, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledWith(4401, "Unauthorized");
+
+        mockWSocket.close.mockClear();
+        mockWS.implementation.emit.mockClear();
+
+        const url = `/api?token=${authToken}`;
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: url}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({url}, mockSocket, 3, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledTimes(0);
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](mockWSocket);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', mockWSocket, {url});
+
+    });
+});

test/group.test.js

@@ -1,11 +1,15 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
@@ -107,7 +111,7 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Legacy api: Add to group with slashes via MQTT', async () => {
@@ -122,13 +126,7 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}', {"retain": false, qos: 0}, expect.any(Function));
-
-        // Test if subscribed to topics with slashes
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Legacy api: Add to group via MQTT with postfix', async () => {
@@ -171,7 +169,7 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Legacy api: Remove from group via MQTT when in zigbee but not in settings', async () => {
@@ -242,7 +240,7 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Remove from group all deprecated', async () => {
@@ -292,8 +290,26 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not republish identical optimistic group states', async () => {
+        const device1 = zigbeeHerdsman.devices.bulb_2;
+        const device2 = zigbeeHerdsman.devices.bulb_color_2;
+        const group = zigbeeHerdsman.groups.group_tradfri_remote;
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -306,11 +322,11 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state, filtered', async () => {
@@ -323,11 +339,11 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON', brightness: 100}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","brightness":100}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","brightness":100}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Shouldnt publish group state change when a group is not optimistic', async () => {
@@ -345,7 +361,7 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of another group with shared device when a group changes its state', async () => {
@@ -362,12 +378,12 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via device', async () => {
@@ -384,14 +400,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -409,15 +425,15 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -434,17 +450,17 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
-        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"OFF"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -461,17 +477,17 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
         controller.state.state = {};
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Add to group via MQTT', async () => {
@@ -483,14 +499,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -505,14 +521,14 @@ describe('Groups', () => {
         endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
         expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -526,14 +542,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group/with/slashes', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            JSON.stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
+            stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -546,14 +562,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            JSON.stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -566,16 +582,16 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            JSON.stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -589,14 +605,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove',
-            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -610,14 +626,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove',
-            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -631,14 +647,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove',
-            JSON.stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -652,14 +668,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'wall_switch_double/3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove',
-            JSON.stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -673,14 +689,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove',
-            JSON.stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -694,14 +710,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', JSON.stringify({device: '0x0017880104e45542/right'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove_all',
-            JSON.stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
+            stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -711,12 +727,12 @@ describe('Groups', () => {
         await flushPromises();
         logger.error.mockClear();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
         await flushPromises();
         expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/remove',
-            JSON.stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
+            stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -726,13 +742,13 @@ describe('Groups', () => {
         await flushPromises();
         logger.error.mockClear();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
         await flushPromises();
         expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            JSON.stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
+            stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
-});
+});

test/homeassistant.test.js

@@ -1,5 +1,6 @@
 const data = require('./stub/data');
 const settings = require('../lib/util/settings');
+const stringify = require('json-stable-stringify-without-jsonify');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const flushPromises = () => new Promise(setImmediate);
@@ -21,19 +22,6 @@ describe('HomeAssistant extension', () => {
         settings.set(['homeassistant'], true);
     });
 
-    it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
-        const missing = [];
-        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
-
-        require('zigbee-herdsman-converters').devices.forEach((d) => {
-            if (!ha._getMapping()[d.model]) {
-                missing.push(d.model);
-            }
-        });
-
-        expect(missing).toHaveLength(0);
-    });
-
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
@@ -76,12 +64,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -101,12 +89,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/humidity/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -126,12 +114,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/pressure/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -151,12 +139,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/battery/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -176,12 +164,120 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/linkquality/config',
-            JSON.stringify(payload),
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "command_topic":"zigbee2mqtt/wall_switch_double/left/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45542"
+                ],
+                "manufacturer":"Xiaomi",
+                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
+                "name":"wall_switch_double",
+                "sw_version":this.version
+            },
+            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
+            "name":"wall_switch_double_left",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/wall_switch_double",
+            "unique_id":"0x0017880104e45542_switch_left_zigbee2mqtt",
+            "value_template":"{{ value_json.state_left }}"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/0x0017880104e45542/switch_left/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "command_topic":"zigbee2mqtt/wall_switch_double/right/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45542"
+                ],
+                "manufacturer":"Xiaomi",
+                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
+                "name":"wall_switch_double",
+                "sw_version":this.version
+            },
+            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
+            "name":"wall_switch_double_right",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/wall_switch_double",
+            "unique_id":"0x0017880104e45542_switch_right_zigbee2mqtt",
+            "value_template":"{{ value_json.state_right }}"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/0x0017880104e45542/switch_right/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_temp":true,
+            "command_topic":"zigbee2mqtt/bulb/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x000b57fffec6a5b2"
+                ],
+                "manufacturer":"IKEA",
+                "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
+                "name":"bulb",
+                "sw_version":this.version,
+            },
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
+            "hs":false,
+            "json_attributes_topic":"zigbee2mqtt/bulb",
+            "name":"bulb",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/bulb",
+            "unique_id":"0x000b57fffec6a5b2_light_zigbee2mqtt",
+            "xy":false
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/0x000b57fffec6a5b2/light/config',
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -217,12 +313,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -242,12 +338,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/humidity/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -267,12 +363,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/pressure/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -323,14 +419,14 @@ describe('HomeAssistant extension', () => {
                 'model': 'custom model',
                 'manufacturer': 'From Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             'expire_after': 90,
             'icon': 'mdi:test',
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -349,19 +445,71 @@ describe('HomeAssistant extension', () => {
                 'model': 'custom model',
                 'manufacturer': 'Not from Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             'expire_after': 30,
             'icon': 'mdi:test',
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/humidity/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
     });
 
+    it('Should discover devices with overriden user configuration affecting type and object_id', async () => {
+        settings.set(['devices', '0x0017880104e45541'], {
+            friendly_name: 'my_switch',
+            homeassistant: {
+                switch: {
+                    type: 'light',
+                    object_id: 'light'
+                },
+                light: {
+                    type: 'this should be ignored',
+                    name: 'my_light_name_override'
+                }
+
+            },
+        })
+
+        controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            "command_topic": "zigbee2mqtt/my_switch/set",
+            "device": {
+              "identifiers": [
+                "zigbee2mqtt_0x0017880104e45541"
+              ],
+              "manufacturer": "Xiaomi",
+              "model": "Aqara single key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG04LM)",
+              "name": "my_switch",
+              "sw_version": this.version
+            },
+            "json_attributes_topic": "zigbee2mqtt/my_switch",
+            "name": "my_light_name_override",
+            "payload_off": "OFF",
+            "payload_on": "ON",
+            "state_topic": "zigbee2mqtt/my_switch",
+            "unique_id": "0x0017880104e45541_light_zigbee2mqtt",
+            "value_template": "{{ value_json.state }}"
+        }
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/0x0017880104e45541/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+    });
+
     it('Shouldnt discover devices when homeassistant null is set in device options', async () => {
         settings.set(['devices', '0x0017880104e45522'], {
             homeassistant: null,
@@ -403,7 +551,7 @@ describe('HomeAssistant extension', () => {
                "smart"
             ],
             "json_attributes_topic":"zigbee2mqtt/fan",
-            "name":"fan_fan",
+            "name":"fan",
             "unique_id":"0x0017880104e45548_fan_zigbee2mqtt",
             "device":{
                "identifiers":[
@@ -414,12 +562,74 @@ describe('HomeAssistant extension', () => {
                "model":"Universal wink enabled white ceiling fan premier remote control (99432)",
                "manufacturer":"Hampton Bay"
             },
-            "availability_topic":"zigbee2mqtt/bridge/state"
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
          };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/fan/0x0017880104e45548/fan/config',
-            JSON.stringify(payload),
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should discover thermostat devices', async () => {
+        controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            "action_template":"{% set values = {'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'} %}{{ values[value_json.running_state] }}",
+            "action_topic":"zigbee2mqtt/TS0601_thermostat",
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
+            "away_mode_state_template":"{{ value_json.away_mode }}",
+            "away_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "current_temperature_template":"{{ value_json.local_temperature }}",
+            "current_temperature_topic":"zigbee2mqtt/TS0601_thermostat",
+            "device":{
+               "identifiers":[
+                  "zigbee2mqtt_0x0017882104a44559"
+               ],
+               "manufacturer":"TuYa",
+               "model":"Radiator valve with thermostat (TS0601_thermostat)",
+               "name":"TS0601_thermostat",
+               "sw_version":  this.version,
+            },
+            "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
+            "hold_modes":[
+               "schedule",
+               "manual",
+               "boost",
+               "complex",
+               "comfort",
+               "eco"
+            ],
+            "hold_state_template":"{{ value_json.preset }}",
+            "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "json_attributes_topic":"zigbee2mqtt/TS0601_thermostat",
+            "max_temp":"35",
+            "min_temp":"5",
+            "mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/system_mode",
+            "mode_state_template":"{{ value_json.system_mode }}",
+            "mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "modes":[
+               "auto"
+            ],
+            "name":"TS0601_thermostat",
+            "temp_step":0.5,
+            "temperature_command_topic":"zigbee2mqtt/TS0601_thermostat/set/current_heating_setpoint",
+            "temperature_state_template":"{{ value_json.current_heating_setpoint }}",
+            "temperature_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "temperature_unit":"C",
+            "unique_id":"0x0017882104a44559_climate_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/climate/0x0017882104a44559/climate/config',
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -439,7 +649,7 @@ describe('HomeAssistant extension', () => {
             set_position_template: '{ "position": {{ position }} }',
             value_template: '{{ value_json.position }}',
             json_attributes_topic: 'zigbee2mqtt/smart vent',
-            name: 'smart vent cover',
+            name: 'smart vent',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
             device:
             {
@@ -449,12 +659,12 @@ describe('HomeAssistant extension', () => {
                 model: 'Smart vent (SV01)',
                 manufacturer: 'Keen Home'
             },
-            availability_topic: 'zigbee2mqtt/bridge/state'
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/cover/0x0017880104e45551/cover/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -483,12 +693,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'my_custom_discovery_topic/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -548,12 +758,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payloadHA),
+            stringify(payloadHA),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -587,13 +797,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            '{"brightness":255,"update_available":false}',
+            stringify({"brightness":255,"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -613,13 +823,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            '{"brightness":255,"update_available":false}',
+            stringify({"brightness":255,"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -676,12 +886,12 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/weather_sensor/availability',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -738,11 +948,14 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should refresh discovery when device is renamed', async () => {
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller(false);
         await controller.start();
         await flushPromises();
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45522/action_double/config', stringify({topic: 'zigbee2mqtt/weather_sensor/action'}));
+        await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', '{"old": "weather_sensor", "new": "weather_sensor_renamed"}');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":true}));
         await flushPromises();
 
         const payload = {
@@ -760,12 +973,83 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45522/action_double/config',
+            stringify({
+                "automation_type":"trigger",
+                "type":"action",
+                "subtype":"double",
+                "payload":"double",
+                "topic":"zigbee2mqtt/weather_sensor_renamed/action",
+                "device":{
+                    "identifiers":[
+                        "zigbee2mqtt_0x0017880104e45522"
+                    ],
+                    "name":"weather_sensor_renamed",
+                    "sw_version": this.version,
+                    "model":"Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)",
+                    "manufacturer":"Xiaomi"
+                }
+            }),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
+        settings.set(['experimental', 'new_api'], true);
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":false}));
+        await flushPromises();
+
+        expect(MQTT.publish).not.toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        const payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
+            'name': 'weather_sensor_renamed_temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'weather_sensor_renamed',
+                'sw_version': this.version,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -792,12 +1076,12 @@ describe('HomeAssistant extension', () => {
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "manufacturer":"IKEA"
             },
-            "availability_topic":"zigbee2mqtt/bridge/state"
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -816,12 +1100,11 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
 
         const device = zigbeeHerdsman.devices.WXKG11LM;
-        settings.set(['devices', device.ieeeAddr, 'legacy'], false);
-        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        const payload1 = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
         await flushPromises();
 
-        const discoverPayload = {
+        const discoverPayloadAction = {
             "automation_type":"trigger",
             "type":"action",
             "subtype":"single",
@@ -840,7 +1123,31 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/device_automation/0x0017880104e45520/action_single/config',
-            JSON.stringify(discoverPayload),
+            stringify(discoverPayloadAction),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        const discoverPayloadClick = {
+            "automation_type":"trigger",
+            "type":"click",
+            "subtype":"single",
+            "payload":"single",
+            "topic":"zigbee2mqtt/button/click",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45520"
+                ],
+                "name":"button",
+                "sw_version": this.version,
+                "model":"Aqara wireless switch (WXKG11LM)",
+                "manufacturer":"Xiaomi"
+            }
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
+            stringify(discoverPayloadClick),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -853,26 +1160,47 @@ describe('HomeAssistant extension', () => {
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/button',
-            JSON.stringify({action: "single", linkquality: 10}),
+            'zigbee2mqtt/button/click',
+            'single',
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            JSON.stringify({linkquality: 10, action: ""}),
+            stringify({action: "single", click: "single"}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button',
+            stringify({action: ""}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button',
+            stringify({click: ""}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         // Should only discover it once
         MQTT.publish.mockClear();
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.message(payload1);
         await flushPromises();
         expect(MQTT.publish).not.toHaveBeenCalledWith(
             'homeassistant/device_automation/0x0017880104e45520/action_single/config',
-            JSON.stringify(discoverPayload),
+            stringify(discoverPayloadAction),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).not.toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
+            stringify(discoverPayloadClick),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -883,6 +1211,33 @@ describe('HomeAssistant extension', () => {
             { retain: false, qos: 0 },
             expect.any(Function),
         );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button/click',
+            'single',
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        // Shouldn't rediscover when already discovered in previous session
+        controller.extensions.find((e) => e.constructor.name === 'HomeAssistant')._clearDiscoveredTrigger();
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45520/action_double/config', stringify({topic: 'zigbee2mqtt/button/action'}));
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45520/action_double/config', stringify({topic: 'zigbee2mqtt/button/action'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const payload2 = {data: {'32768': 2}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
+
+        // Should rediscover when already discovered in previous session but with diferent name
+        controller.extensions.find((e) => e.constructor.name === 'HomeAssistant')._clearDiscoveredTrigger();
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45520/action_double/config', stringify({topic: 'zigbee2mqtt/button_other_name/action'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload2);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
     });
 
     it('Should not discover sensor_click when legacy: false is set', async () => {
@@ -940,7 +1295,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/device_automation/0x0017880104e45520/action_single/config',
-            JSON.stringify(discoverPayload),
+            stringify(discoverPayload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -954,7 +1309,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            JSON.stringify({action: "single", linkquality: 10}),
+            stringify({action: "single"}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -968,11 +1323,11 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/U202DST600ZB/l2/set', JSON.stringify({state: 'ON', brightness: 20}));
+        await MQTT.events.message('zigbee2mqtt/U202DST600ZB/l2/set', stringify({state: 'ON', brightness: 20}));
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', JSON.stringify({state_l2:"ON", brightness_l2:20}), {"qos": 0, "retain": false}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l2', JSON.stringify({state:"ON", brightness:20}), {"qos": 0, "retain": false}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', JSON.stringify({}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', stringify({state_l2:"ON", brightness_l2:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l2', stringify({state:"ON", brightness:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', stringify({}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Shouldnt crash in onPublishEntityState on group publish', async () => {
@@ -982,7 +1337,7 @@ describe('HomeAssistant extension', () => {
         logger.error.mockClear();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
@@ -1000,10 +1355,10 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: ''});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/action_single/config');
         expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/action');
@@ -1030,6 +1385,56 @@ describe('HomeAssistant extension', () => {
                 command_topic: true,
             },
         };
-        expect(ha._getMapping()['external_converters_device']).toStrictEqual([homeassistantSwitch]);
+        expect(ha._getMapping()['external_converters_device']).toEqual([homeassistantSwitch]);
+    });
+
+    it('Should clear outdated configs', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+
+        // Non-existing device -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x123/temperature/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Existing device -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Non-existing device of different instance -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt_different/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Existing device but non-existing config -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Non-existing device but invalid payload -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', '1}3');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Existing device, device automation -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', stringify({topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Device automation of different instance -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/device_automation/0x000b57fffec6a5b2_not_existing/action_button_3_single/config', stringify({topic: 'zigbee2mqtt_different/0x000b57fffec6a5b2_not_existing/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 });

test/legacy/bridgeLegacy.test.js

@@ -2,6 +2,7 @@ const data = require('../stub/data');
 const logger = require('../stub/logger');
 const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
 const MQTT = require('../stub/mqtt');
+const stringify = require('json-stable-stringify-without-jsonify');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
@@ -29,7 +30,7 @@ describe('Bridge legacy', () => {
     it('Should publish bridge configuration on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/config',
-          JSON.stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
+          stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -56,7 +57,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -67,7 +68,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -85,12 +86,12 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {retain: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {retain: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
@@ -100,17 +101,17 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
@@ -177,7 +178,7 @@ describe('Bridge legacy', () => {
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
         expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
         expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
-        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "softwareBuildID": null, "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
         Date.now = now;
     });
 
@@ -187,25 +188,25 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","optimistic":true,"retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","optimistic":true,"retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","optimistic":true,"retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","optimistic":true,"retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","optimistic":true,"retain":false},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","optimistic":true,"retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {
         const bulb_color2 = {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color2", "friendly_name": "bulb_color2", "retain": false};
         MQTT.publish.mockClear();
         expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false});
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color2'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color2'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(null);
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
+            stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
 
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
 
@@ -213,7 +214,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
 
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color3'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
     });
@@ -221,12 +222,12 @@ describe('Bridge legacy', () => {
     it('Should allow rename groups', async () => {
         MQTT.publish.mockClear();
         expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'group_1', new: 'group_1_renamed'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
         await flushPromises();
         expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -243,7 +244,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb_new_name');
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
+            stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -265,7 +266,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -283,7 +284,7 @@ describe('Bridge legacy', () => {
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -298,7 +299,7 @@ describe('Bridge legacy', () => {
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -321,7 +322,7 @@ describe('Bridge legacy', () => {
         expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            stringify({type: 'group_removed', message: 'group_1'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -335,7 +336,7 @@ describe('Bridge legacy', () => {
         expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            stringify({type: 'group_removed', message: 'group_1'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -370,7 +371,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_removed', message: 'bulb_color'}),
+            stringify({type: 'device_removed', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -392,7 +393,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_force_removed', message: 'bulb_color'}),
+            stringify({type: 'device_force_removed', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -413,7 +414,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_banned', message: 'bulb_color'}),
+            stringify({type: 'device_banned', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -456,22 +457,22 @@ describe('Bridge legacy', () => {
     });
 
     it('Should allow to touchlink factory reset (OK)', async () => {
-        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
 
-        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(true);
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
-        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
         expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
     });
 
     it('Should allow to touchlink factory reset (FAILS)', async () => {
-        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
 
-        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(false);
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
-        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
         expect(logger.warn).toHaveBeenCalledWith('Failed to factory reset device through Touchlink');
     });
 });

test/logger.test.js

@@ -24,12 +24,14 @@ describe('Logger', () => {
 
     it('Create log directory', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const dirs = fs.readdirSync(dir.name);
         expect(dirs.length).toBe(1);
     });
 
     it('Should cleanup', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
 
         for (const d of fs.readdirSync(dir.name)) {
             rimraf.sync(path.join(dir.name, d));
@@ -46,6 +48,7 @@ describe('Logger', () => {
 
     it('Should not cleanup when there is no timestamp set', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         for (let i = 30; i < 40; i++) {
             fs.mkdirSync(path.join(dir.name, `log_${i}`));
         }
@@ -58,12 +61,14 @@ describe('Logger', () => {
 
     it('Set and get log level', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         logger.setLevel('debug');
         expect(logger.getLevel()).toBe('debug');
     });
 
     it('Logger should be console and file by default', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
@@ -75,6 +80,7 @@ describe('Logger', () => {
     it('Logger can be file only', () => {
         settings.set(['advanced', 'log_output'], ['file']);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
@@ -86,6 +92,7 @@ describe('Logger', () => {
     it('Logger can be console only', () => {
         settings.set(['advanced', 'log_output'], ['console']);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(false);
@@ -94,6 +101,7 @@ describe('Logger', () => {
     it('Logger can be nothing', () => {
         settings.set(['advanced', 'log_output'], []);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
@@ -102,6 +110,7 @@ describe('Logger', () => {
     it('Should allow to disable log rotation', () => {
         settings.set(['advanced', 'log_rotation'], false);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes[1].constructor.name).toBe('File');
         expect(pipes[1].maxFiles).toBeNull();

test/networkMap.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
 
@@ -36,6 +37,8 @@ describe('Networkmap', () => {
     beforeEach(async () => {
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = 1000;
     });
 
     function mock() {
@@ -67,7 +70,7 @@ describe('Networkmap', () => {
         bulb_color.routingTable = () => {return {table: []}};
 
         CC2530_ROUTER.lqi = () => {return {neighbors: [
-            {ieeeAddr: WXKG02LM.ieeeAddr, networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
+            {ieeeAddr: '0x0000000000000000', networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
             {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 4, depth: 2, linkquality: 130},
         ]}};
         CC2530_ROUTER.routingTable = () => {return {table: []}};
@@ -84,26 +87,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            "nodes":[
-                {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-            ],
-            "links":[
-                {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-            ]
-        };
-
+        const expected = {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch","model":"WXKG02LM","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -258,41 +242,14 @@ describe('Networkmap', () => {
     it('Should output raw networkmap', async () => {
         mock();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', JSON.stringify({type: 'raw', routes: true}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', stringify({type: 'raw', routes: true}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            status: 'ok',
-            data: {
-                type: 'raw',
-                routes: true,
-                value: {
-                    "nodes":[
-                        {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                        {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                        {"lastSeen": null,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                        {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                        {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                        {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-                    ],
-                    "links":[
-                        {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                        {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                        {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                        {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                        {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-                    ]
-                }
-            }
-        };
-
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch","model":"WXKG02LM","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
-        actual.data.value = JSON.parse(actual.data.value);
         expect(actual).toStrictEqual(expected);
     });
 
@@ -303,7 +260,7 @@ describe('Networkmap', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/networkmap',
-            JSON.stringify({"data":{},"status":"error","error":"Type 'not_existing' not supported, allowed are: raw,graphviz,plantuml"}),
+            stringify({"data":{},"status":"error","error":"Type 'not_existing' not supported, allowed are: raw,graphviz,plantuml"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });

test/otaUpdate.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('OTA update', () => {
     let controller;
@@ -26,13 +27,6 @@ describe('OTA update', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/update');
-    });
-
     it('Should OTA update a device', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
@@ -58,30 +52,36 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');
         expect(device.softwareBuildID).toBe(2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            JSON.stringify({"update_available":false,"update":{"state":"updating","progress":0}}),
+            stringify({"update_available":false,"update":{"state":"updating","progress":0}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            JSON.stringify({"update_available":false,"update":{"state":"updating","progress":10,"remaining":3600}}),
+            stringify({"update_available":false,"update":{"state":"updating","progress":10,"remaining":3600}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            JSON.stringify({"update_available":false,"update":{"state":"idle"}}),
+            stringify({"update_available":false,"update":{"state":"idle"}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/update',
-            JSON.stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
     });
 
     it('Should handle when OTA update fails', async () => {
@@ -95,16 +95,16 @@ describe('OTA update', () => {
             throw new Error('Update failed');
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', JSON.stringify({id: "bulb"}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', stringify({id: "bulb"}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            JSON.stringify({"update_available":true,"update":{"state":"available"}}),
+            stringify({"update_available":true,"update":{"state":"available"}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/update',
-            JSON.stringify({"data":{"id": "bulb"},"status":"error","error":"Update of 'bulb' failed (Update failed)"}),
+            stringify({"data":{"id": "bulb"},"status":"error","error":"Update of 'bulb' failed (Update failed)"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -121,7 +121,7 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            JSON.stringify({"data":{"id": "bulb","updateAvailable":false},"status":"ok"}),
+            stringify({"data":{"id": "bulb","updateAvailable":false},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
 
@@ -133,7 +133,7 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            JSON.stringify({"data":{"id": "bulb","updateAvailable":true},"status":"ok"}),
+            stringify({"data":{"id": "bulb","updateAvailable":true},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -150,7 +150,7 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": `Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`}),
+            stringify({"data":{"id": "bulb"},"status":"error","error": `Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -160,7 +160,7 @@ describe('OTA update', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            JSON.stringify({"data":{"id": "not_existing_deviceooo"},"status":"error","error": `Device 'not_existing_deviceooo' does not exist`}),
+            stringify({"data":{"id": "not_existing_deviceooo"},"status":"error","error": `Device 'not_existing_deviceooo' does not exist`}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -170,7 +170,7 @@ describe('OTA update', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            JSON.stringify({"data":{"id": "ZNLDP12LM"},"status":"error","error": `Device 'ZNLDP12LM' does not support OTA updates`}),
+            stringify({"data":{"id": "ZNLDP12LM"},"status":"error","error": `Device 'ZNLDP12LM' does not support OTA updates`}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -193,7 +193,7 @@ describe('OTA update', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": `Update or check for update already in progress for 'bulb'`}),
+            stringify({"data":{"id": "bulb"},"status":"error","error": `Update or check for update already in progress for 'bulb'`}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -210,7 +210,7 @@ describe('OTA update', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/update',
-            JSON.stringify({"data":{"id":"bulb","from":null,"to":null},"status":"ok"}),
+            stringify({"data":{"id":"bulb","from":null,"to":null},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -245,7 +245,7 @@ describe('OTA update', () => {
         expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            JSON.stringify({"update_available":true,"update":{"state":"available"}}),
+            stringify({"update_available":true,"update":{"state":"available"}}),
             {retain: true, qos: 0}, expect.any(Function)
         );
     });
@@ -297,7 +297,7 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
         expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');

test/receive.test.js

@@ -1,5 +1,6 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
@@ -19,16 +20,18 @@ describe('Receive', () => {
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
+        delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
     });
 
     it('Should handle a zigbee message', async () => {
         const device = zigbeeHerdsman.devices.WXKG11LM;
+        device.linkquality = 10;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', JSON.stringify({action: 'single', click: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single', click: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should handle a zigbee message which uses ep (left)', async () => {
@@ -39,7 +42,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', action: 'single_left', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', action: 'single_left'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -51,7 +54,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', action: 'single_right', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', action: 'single_right'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -63,7 +66,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -74,7 +77,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', JSON.stringify({position: 10, tilt: 20, linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20}), {retain: false, qos: 0}, expect.any(Function));
 
         // Inverted
         MQTT.publish.mockClear();
@@ -82,7 +85,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', JSON.stringify({position: 90, tilt: 80, linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should allow to disable the legacy integration', async () => {
@@ -93,7 +96,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', JSON.stringify({action: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single'}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should debounce messages', async () => {
@@ -116,7 +119,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -142,7 +145,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": true});
     });
 
@@ -162,11 +165,11 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2, linkquality: 13});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2});
         jest.runAllTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03});
     });
 
     it('Shouldnt republish old state', async () => {
@@ -175,12 +178,12 @@ describe('Receive', () => {
         const device = zigbeeHerdsman.devices.bulb;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
-        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON'}));
         await flushPromises();
         jest.runAllTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
     });
 
     it('Should handle a zigbee message with 1 precision', async () => {
@@ -192,7 +195,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -205,7 +208,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -218,7 +221,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -232,7 +235,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -245,14 +248,14 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5,"linkquality":10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5});
 
         MQTT.publish.mockClear();
         payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4,"linkquality":10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4});
     });
 
     it('Should handle a zigbee message with voltage 3010', async () => {
@@ -263,7 +266,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -275,7 +278,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -287,7 +290,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -299,7 +302,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -311,7 +314,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -363,13 +366,18 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-    it('Should not handle messages forwarded Xiaomi messages', async () => {
+    it('Should handle forwarded Xiaomi messages', async () => {
         const device = zigbeeHerdsman.devices.ZNCZ02LM;
-        const data = {onOff: 1};
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 599};
+        const payload = {data: {measuredValue: -85}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 6539};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/weather_sensor',
+            stringify({temperature: -0.85}),
+            {"qos": 1, "retain": false},
+            expect.any(Function),
+        )
     });
 
     it('Should handle messages from Xiaomi router devices', async () => {
@@ -378,10 +386,20 @@ describe('Receive', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/power_plug');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'state': 'ON', linkquality: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/power_plug',
+            stringify({state: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/switch_group',
+            stringify({'state': 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+
     });
 
     it('Should not handle messages from coordinator', async () => {
@@ -400,7 +418,7 @@ describe('Receive', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID' and manufacturer name 'notSupportedMfg'`);
         expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
@@ -424,7 +442,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -471,7 +489,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6});
 
         MQTT.publish.mockClear();
         const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
@@ -479,6 +497,6 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496});
     });
 });

test/report.test.js

@@ -267,7 +267,7 @@ describe('Report', () => {
         expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
 
         // Should only call Hue bulb, not e.g. tradfri
-        expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
     });
 
     it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {

test/settings.test.js

@@ -6,9 +6,11 @@ const settings = require('../lib/util/settings.js');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
+const devicesFile2 = data.joinPath('devices2.yaml');
 const groupsFile = data.joinPath('groups.yaml');
 const secretFile = data.joinPath('secret.yaml');
 const yaml = require('js-yaml');
+const objectAssignDeep = require(`object-assign-deep`);
 
 const minimalConfig = {
     permit_join: true,
@@ -27,11 +29,19 @@ describe('Settings', () => {
     const remove = (file) => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
+    const clearEnvironmentVariables = () => {
+        Object.keys(process.env).forEach((key) => {
+            if(key.indexOf('ZIGBEE2MQTT_CONFIG_') >= 0) {
+                delete process.env[key];
+            }
+        });
+    }
 
     beforeEach(() => {
         remove(configurationFile);
         remove(devicesFile);
         remove(groupsFile);
+        clearEnvironmentVariables();
     });
 
     it('Should return default settings', () => {
@@ -53,6 +63,29 @@ describe('Settings', () => {
         expect(s).toStrictEqual(expected);
     });
 
+    it('Should apply environment variables', () => {
+        process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
+        process.env['ZIGBEE2MQTT_CONFIG_EXPERIMENTAL_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_AVAILABILITY_BLOCKLIST'] = '["0x43597f0dac781b1e", "x223b0aef2ae8d1b0"]';
+        process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
+        process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
+
+        write(configurationFile, {});
+        const s = settings.get();
+        const expected = objectAssignDeep.noMutate({}, settings._getDefaults());
+        expected.devices = {};
+        expected.groups = {};
+        expected.serial.disable_led = true;
+        expected.advanced.soft_reset_timeout = 1;
+        expected.experimental.output = 'csvtest';
+        expected.advanced.availability_blocklist = ['0x43597f0dac781b1e', 'x223b0aef2ae8d1b0'];
+        expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
+        expected.mqtt.base_topic = 'testtopic';
+
+        expect(s).toStrictEqual(expected);
+    });
+
     it('Should add devices', () => {
         write(configurationFile, {});
         settings.addDevice('0x12345678');
@@ -121,6 +154,7 @@ describe('Settings', () => {
 
         const expected = {
             include_device_information: false,
+            force_disable_retain: false,
             password: "mysecretpassword",
             server: "my.mqtt.server",
             user: "mysecretusername",
@@ -164,6 +198,32 @@ describe('Settings', () => {
         expect(device).toStrictEqual(expected);
     });
 
+    it('Should read devices form 2 separate files', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+        expect(settings.getDevice('0x12345678').friendly_name).toStrictEqual('0x12345678');
+        expect(settings.getDevice('0x87654321').friendly_name).toStrictEqual('0x87654321');
+    });
+
     it('Should add devices to a separate file', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',
@@ -196,6 +256,47 @@ describe('Settings', () => {
         expect(read(devicesFile)).toStrictEqual(expected);
     });
 
+    it('Should add devices for first file when using 2 separates file', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual({devices: ['devices.yaml', 'devices2.yaml']});
+
+        const expected = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expected);
+        expect(read(devicesFile2)).toStrictEqual(contentDevices2);
+    });
+
     it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',

test/stub/data.js

@@ -2,6 +2,7 @@ const tmp = require('tmp');
 const yaml = require('../../lib/util/yaml');
 const path = require('path');
 const fs = require('fs');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const mockDir = tmp.dirSync().name;
 const mockDirStorage = tmp.dirSync().name;
@@ -158,6 +159,9 @@ function writeDefaultConfiguration() {
             '0x0017880104a44559': {
                 friendly_name: 'J1_cover',
             },
+            '0x0017882104a44559': {
+                friendly_name: 'TS0601_thermostat',
+            },
         },
         groups: {
             '1': {
@@ -178,6 +182,16 @@ function writeDefaultConfiguration() {
                 retain: false,
                 devices: ['bulb_2']
             },
+            '12': {
+                friendly_name: 'thermostat_group',
+                retain: false,
+                devices: ['TS0601_thermostat'],
+            },
+            '14': {
+                friendly_name: 'switch_group',
+                retain: false,
+                devices: ['power_plug'],
+            }
         },
         external_converters: [],
     };
@@ -186,7 +200,7 @@ function writeDefaultConfiguration() {
 }
 
 function writeEmptyState() {
-    fs.writeFileSync(stateFile, JSON.stringify({}));
+    fs.writeFileSync(stateFile, stringify({}));
 }
 
 function removeState() {
@@ -212,7 +226,7 @@ function writeDefaultState() {
         },
     }
 
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
+    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(state));
 }
 
 jest.mock('../../lib/util/data', () => ({

test/stub/logger.js

@@ -17,6 +17,7 @@ const mock = {
     error: jest.fn().mockImplementation((msg) => callTransports('error', msg)),
     debug: jest.fn().mockImplementation((msg) => callTransports('debug', msg)),
     cleanup: jest.fn(),
+    logOutput: jest.fn(),
     add: (transport) => transports.push(transport),
     setLevel: (newLevel) => {level = newLevel},
     getLevel: () => level,

test/stub/zigbeeHerdsman.js

@@ -1,6 +1,11 @@
 const events = {};
 const assert = require('assert');
 
+function getKeyByValue(object, value, fallback) {
+    const key = Object.keys(object).find((k) => object[k] === value);
+    return key != null ? key : fallback;
+}
+
 class Group {
     constructor(groupID, members) {
         this.groupID = groupID;
@@ -36,8 +41,18 @@ class Endpoint {
         this.write = jest.fn();
         this.bind = jest.fn();
         this.unbind = jest.fn();
+        this.save = jest.fn();
         this.configureReporting = jest.fn();
+        this.meta = {};
         this.binds = binds;
+        this.getInputClusters = () => inputClusters.map((c) => {
+            return {ID: c, name: getKeyByValue(clusters, c)};
+        }).filter((c) => c.name);
+
+        this.getOutputClusters = () => outputClusters.map((c) => {
+            return {ID: c, name: getKeyByValue(clusters, c)};
+        }).filter((c) => c.name);
+
         this.supportsInputCluster = (cluster) => {
             assert(clusters[cluster] !== undefined, `Undefined '${cluster}'`);
             return this.inputClusters.includes(clusters[cluster]);
@@ -76,7 +91,7 @@ class Endpoint {
 }
 
 class Device {
-    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null) {
+    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null, softwareBuildID=null) {
         this.type = type;
         this.ieeeAddr = ieeeAddr;
         this.dateCode = dateCode;
@@ -84,6 +99,7 @@ class Device {
         this.manufacturerID = manufacturerID;
         this.endpoints = endpoints;
         this.powerSource = powerSource;
+        this.softwareBuildID = softwareBuildID;
         this.interviewCompleted = interviewCompleted;
         this.modelID = modelID;
         this.interviewing = interviewing;
@@ -104,8 +120,9 @@ class Device {
 const returnDevices = [];
 
 const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
-const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
+const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020", false, 'Philips', '2019.09', '5.127.1.26581');
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
+const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 
 const devices = {
     'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
@@ -113,8 +130,8 @@ const devices = {
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
-    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0]}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
-    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: new Group(1, []), cluster: {ID: 6, name: 'genOnOff'}}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
+    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID", false, "notSupportedMfg"),
     'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
     'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
     'notInSettings': new Device('EndDevice', '0x0017880104e45519', 6537, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
@@ -122,13 +139,13 @@ const devices = {
     'WXKG02LM': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
     'WSDCGQ11LM': new Device('EndDevice', '0x0017880104e45522', 6539,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.weather"),
     'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
-    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", "lumi.plug"),
+    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug"),
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
     'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
     'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
     'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
-    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
+    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45545')], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
     'ZNCLDJ11LM': new Device('Router', '0x0017880104e45547', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'lumi.curtain'),
     'HAMPTON99432':  new Device('Router', '0x0017880104e45548', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'HDC52EastwindFan'),
     'HS2WD': new Device('Router', '0x0017880104e45549', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'WarningDevice'),
@@ -150,18 +167,25 @@ const devices = {
     'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
     '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
     'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
+    'TS0601_thermostat': TS0601_thermostat,
 }
 
 const groups = {
     'group_1': new Group(1, []),
     'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
     'group/with/slashes': new Group(99, []),
-    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]])
+    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
+    'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
+    'group_with_switch': new Group(14, [devices.ZNCZ02LM.endpoints[0]]),
+    'default_bind_group': new Group(901, []),
 }
 
 const mock = {
     setTransmitPower: jest.fn(),
     touchlinkFactoryReset: jest.fn(),
+    touchlinkFactoryResetFirst: jest.fn(),
+    touchlinkScan: jest.fn(),
+    touchlinkIdentify: jest.fn(),
     start: jest.fn(),
     permitJoin: jest.fn(),
     getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
@@ -180,6 +204,9 @@ const mock = {
     getDeviceByIeeeAddr: jest.fn().mockImplementation((ieeeAddr) => {
         return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.ieeeAddr === ieeeAddr);
     }),
+    getDeviceByNetworkAddress: jest.fn().mockImplementation((networkAddress) => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.networkAddress === networkAddress);
+    }),
     getGroups: jest.fn().mockImplementation((query) => {
         return Object.values(groups);
     }),

test/utils.test.js

@@ -1,5 +1,6 @@
 const utils = require('../lib/util/utils.js');
 const version = require('../package.json').version;
+const versionHerdsman = require('../node_modules/zigbee-herdsman/package.json').version;
 
 describe('Utils', () => {
     describe('Is xiaomi device', () => {
@@ -41,6 +42,11 @@ describe('Utils', () => {
         expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
     })
 
+    it('Check dependency version', async () => {
+        var dependency = 'zigbee-herdsman';
+        expect(await utils.getDependencyVersion(dependency)).toStrictEqual({"version": versionHerdsman});
+    })
+    
     it('To local iso string', async () => {
         var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
         var getTimezoneOffset = Date.prototype.getTimezoneOffset;

update.sh

@@ -1,5 +1,12 @@
 #!/bin/bash -e
 
+if [ -d data-backup ]; then
+   echo "ERROR: Backup directory exists. May be previous restoring was failed?"
+   echo "1. Save 'data-backup' and 'data' dirs to safe location to make possibility to restore config later."
+   echo "2. Manually delete 'data-backup' dir and try again."
+   exit 1
+fi
+
 echo "Stopping Zigbee2MQTT..."
 sudo systemctl stop zigbee2mqtt