1.16.2

Co-authored-by: nurikk <nurikk@users.noreply.github.com>

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,14 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Frontend issues
+    url: https://github.com/nurikk/z2m-frontend/issues
+    about: Issues related to the frontend.
+  - name: Home Assistant addon issues
+    url: https://github.com/danielwelch/hassio-zigbee2mqtt/issues
+    about: Issues related to the Home Assistant addon.
+  - name: FAQ
+    url: https://www.zigbee2mqtt.io/information/FAQ.html
+    about: Frequently asked questions.
+  - name: Support Chat
+    url: https://discord.gg/NyseBeK
+    about: Chat for feedback, questions and troubleshooting.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature request
+assignees: ''
+
+---
+
+## Is your feature request related to a problem? Please describe
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -0,0 +1,18 @@
+---
+name: New device support request
+about: Request support for a new device
+title: ''
+labels: new device support
+assignees: ''
+
+---
+
+<!--
+IMPORTANT: Before submitting a request, first follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+-->
+
+## Information about the device + link
+..
+
+## data/database.db entry of the device
+..

.github/ISSUE_TEMPLATE/problem_report.md

@@ -0,0 +1,54 @@
+---
+name: Problem report
+about: Create a report to help us improve
+title: ''
+labels: problem
+assignees: ''
+
+---
+
+<!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
+Did you read the FAQ?
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
+
+Unsupported device?
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
+
+Bug report?
+- If applicable, provide steps how to reproduce the problem.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
+-->
+
+<!--Start your bug report below this line-->
+
+## What happened
+
+## What did you expect to happen
+
+## How to reproduce it (minimal and precise)
+
+## Debug info
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1, CC1352P-2, Conbee II
+Adapter firmware version:

.github/branch-switcher.yml

@@ -1,3 +1 @@
-preferredBranch: dev
-exclude:
-  label: branch-switcher-ignore
+preferredBranch: dev

.github/workflows/ci.yml

@@ -31,20 +31,23 @@ jobs:
     - name: Docker login
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
       run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
-    - name: Docker setup
+    - name: Docker setup - QEMU
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
-      run: |
-        sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
-        sudo chmod a+x /usr/local/bin/buildx
-        docker run --rm --privileged hypriot/qemu-register:v2.7.0
-        buildx create --use
-        buildx ls
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+    - name: Docker setup - Buildx
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
     - name: Docker build dev
       if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
       run: |
-        buildx build \
+        docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
           -t koenkk/zigbee2mqtt:latest-dev \
@@ -53,9 +56,9 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
       run: |
         TAG="$(git describe --tags)"
-        buildx build \
+        docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
           -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
@@ -73,4 +76,13 @@ jobs:
           --header "Content-Type: application/json" \
           --data '{"definition": {"id": 1}}' \
           -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 9}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/stale.yml

@@ -16,3 +16,4 @@ jobs:
         days-before-stale: 30
         days-before-close: 7
         operations-per-run: 500
+        exempt-issue-labels: 'breaking change'

.github/workflows/update_deps.yml

@@ -16,9 +16,9 @@ jobs:
       with:
         node-version: 12
     - run: npm install
-    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee-herdsman" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
+    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
     - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies
         branch: "update_deps"

.github/workflows/update_frontend.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-frontend
+on:
+  repository_dispatch:
+    types: update_frontend
+
+jobs:
+  update_frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
+        branch: "update_frontend"
+        title: Update zigbee-frontend to ${{ github.event.client_payload.version }}

.github/workflows/update_zh.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
         branch: "update_zh"

.github/workflows/update_zhc.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
         branch: "update_zhc"

.gitignore

@@ -70,4 +70,7 @@ data/database.db.backup
 data/extension
 
 # MacOS indexing file
-.DS_Store
+.DS_Store
+
+# Ignore config
+data/*.yaml

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "files.associations": {
-        "*.yaml": "home-assistant"
-    }
-}

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12-alpine as base
+FROM node:12-alpine3.12 as base
 
 WORKDIR /app
 RUN apk add --no-cache tzdata eudev
@@ -10,7 +10,7 @@ FROM base as dependencies
 
 COPY npm-shrinkwrap.json ./
 
-RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
     npm ci --production && \
     apk del .buildtools
 

lib/controller.js

@@ -6,13 +6,14 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const fs = require('fs');
 const data = require('./util/data');
 const path = require('path');
 const assert = require('assert');
 
 // Extensions
+const ExtensionFrontend = require('./extension/frontend');
 const ExtensionPublish = require('./extension/publish');
 const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
@@ -67,6 +68,10 @@ class Controller {
             this.extensions.push(new ExtensionBridge(...args, this.enableDisableExtension));
         }
 
+        if (settings.get().frontend) {
+            this.extensions.push(new ExtensionFrontend(...args));
+        }
+
         if (settings.get().advanced.legacy_api) {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
@@ -99,6 +104,7 @@ class Controller {
 
     async start() {
         this.state.start();
+        logger.logOutput();
 
         const info = await utils.getZigbee2mqttVersion();
         logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
@@ -205,7 +211,12 @@ class Controller {
 
     async onZigbeeEvent(type, data) {
         const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !settings.getDevice(data.device.ieeeAddr) && data.device.type !== 'Coordinator') {
+        if (data.device && data.device.type === 'Coordinator') {
+            logger.debug('Ignoring message from coordinator');
+            return;
+        }
+
+        if (data.device && !settings.getDevice(data.device.ieeeAddr)) {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
             resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
         }
@@ -229,7 +240,8 @@ class Controller {
                     logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
-                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' and manufacturer name ` +
+                        `'${data.device.manufacturerName}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
                 }
@@ -247,6 +259,7 @@ class Controller {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
                 logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                this.state.remove(data.ieeeAddr);
             }
         }
 
@@ -307,11 +320,17 @@ class Controller {
             attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
         }
 
+        // Add lastseen
         const lastSeen = settings.get().advanced.last_seen;
         if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
             messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
         }
 
+        // Add device linkquality.
+        if (resolvedEntity.type === 'device' && resolvedEntity.device.linkquality !== undefined) {
+            messagePayload.linkquality = resolvedEntity.device.linkquality;
+        }
+
         // filter mqtt message attributes
         if (resolvedEntity.settings.filtered_attributes) {
             resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);

lib/eventBus.js

@@ -9,6 +9,7 @@ const allowedEvents = [
     'stateChange', // Entity changes its state
     'groupMembersChanged', // Members of a group has been changed
     'reportingDisabled', // Reporting is disabled for a device
+    'devicesChanged', // Devices changed
 ];
 
 class EventBus extends events.EventEmitter {

lib/extension/availability.js

@@ -24,6 +24,8 @@ class Availability extends Extension {
         this.timers = {};
         this.state = {};
 
+        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+
         this.blocklist = settings.get().advanced.availability_blocklist
             .concat(settings.get().advanced.availability_blacklist)
             .map((e) => settings.getEntity(e).ID);
@@ -33,6 +35,12 @@ class Availability extends Extension {
             .map((e) => settings.getEntity(e).ID);
     }
 
+    onDeviceRemoved(resolvedEntity) {
+        this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
+        delete this.state[resolvedEntity.device.ieeeAddr];
+        clearTimeout(this.timers[resolvedEntity.device.ieeeAddr]);
+    }
+
     inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
         const deviceSettings = settings.getDevice(ieeeAddr);
@@ -114,10 +122,7 @@ class Availability extends Extension {
     }
 
     setTimerPingable(device) {
-        if (this.timers[device.ieeeAddr]) {
-            clearTimeout(this.timers[device.ieeeAddr]);
-        }
-
+        clearTimeout(this.timers[device.ieeeAddr]);
         this.timers[device.ieeeAddr] = setTimeout(async () => {
             await this.handleIntervalPingable(device);
         }, utils.secondsToMilliseconds(this.availability_timeout));
@@ -135,13 +140,11 @@ class Availability extends Extension {
     async onReconnect(device) {
         const resolvedEntity = this.zigbee.resolveEntity(device);
         if (resolvedEntity && resolvedEntity.definition) {
-            const used = [];
             try {
                 for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                     const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
-                    if (converter && !used.includes(converter)) {
+                    if (converter) {
                         await converter.convertGet(device.endpoints[0], key, {});
-                        used.push(converter);
                     }
                 }
             } catch (error) {

lib/extension/bind.js

@@ -4,7 +4,7 @@ const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const Extension = require('./extension');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
 
@@ -17,20 +17,6 @@ class Bind extends Extension {
         this.legacyApi = settings.get().advanced.legacy_api;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
-        }
-    }
-
     parseMQTTMessage(topic, message) {
         let type = null;
         let sourceKey = null;
@@ -157,6 +143,8 @@ class Bind extends Extension {
 
         if (error) {
             logger.error(error);
+        } else {
+            this.eventBus.emit(`devicesChanged`);
         }
     }
 }

lib/extension/bridge.js

@@ -4,7 +4,8 @@ const Extension = require('./extension');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../util/settings');
 const Transport = require('winston-transport');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
+const objectAssignDeep = require(`object-assign-deep`);
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
@@ -29,6 +30,8 @@ class Bridge extends Extension {
             'config/elapsed': this.configElapsed.bind(this),
             'config/log_level': this.configLogLevel.bind(this),
             'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'touchlink/identify': this.touchlinkIdentify.bind(this),
+            'touchlink/scan': this.touchlinkScan.bind(this),
             'health_check': this.healthCheck.bind(this),
         };
     }
@@ -38,7 +41,7 @@ class Bridge extends Extension {
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
         this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        this.eventBus.on(`devicesChanged`, () => this.publishDevices());
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
@@ -88,13 +91,9 @@ class Bridge extends Extension {
                     friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
                 };
                 if (data.status === 'successful') {
-                    payload.supported = !!resolvedEntity.definition;
-                    payload.definition = resolvedEntity.definition ? {
-                        model: resolvedEntity.definition.model,
-                        vendor: resolvedEntity.definition.vendor,
-                        description: resolvedEntity.definition.description,
-                        supports: resolvedEntity.definition.supports,
-                    } : null;
+                    const definition = resolvedEntity.definition;
+                    payload.supported = !!definition;
+                    payload.definition = this.getDefinitionPayload(definition);
                 }
             } else if (type === 'deviceAnnounce') {
                 payload = {
@@ -160,10 +159,29 @@ class Bridge extends Extension {
     }
 
     async permitJoin(message) {
-        const value = this.getValue(message);
-        await this.zigbee.permitJoin(value);
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        let value;
+        let resolvedEntity;
+        if (typeof message === 'object') {
+            value = message.value;
+            if (message.device) {
+                resolvedEntity = this.zigbee.resolveEntity(message.device);
+                if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                    throw new Error(`Device '${message.device}' does not exist`);
+                }
+            }
+        } else {
+            value = message;
+        }
+
+        await this.zigbee.permitJoin(value, resolvedEntity);
         await this.publishInfo();
-        return utils.getResponse(message, {value: value}, null);
+        return utils.getResponse(
+            message, resolvedEntity ? {value: value, device: message.device} : {value: value}, null,
+        );
     }
 
     configLastSeen(message) {
@@ -174,6 +192,7 @@ class Bridge extends Extension {
         }
 
         settings.set(['advanced', 'last_seen'], message);
+        this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
@@ -186,6 +205,7 @@ class Bridge extends Extension {
 
         this.enableDisableExtension(value, 'HomeAssistant');
         settings.set(['homeassistant'], value);
+        this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
@@ -197,6 +217,7 @@ class Bridge extends Extension {
         }
 
         settings.set(['advanced', 'elapsed'], value);
+        this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
@@ -212,18 +233,50 @@ class Bridge extends Extension {
         return utils.getResponse(message, {value}, null);
     }
 
+    async touchlinkIdentify(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
+            !message.hasOwnProperty('channel')) {
+            throw new Error('Invalid payload');
+        }
+
+        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
+        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
+        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
+    }
+
     async touchlinkFactoryReset(message) {
-        logger.info('Start touchlink factory reset');
-        const result = await this.zigbee.touchlinkFactoryReset();
+        let result = false;
+        const payload = {};
+        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
+            message.hasOwnProperty('channel')) {
+            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
+            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
+            payload.ieee_address = message.ieee_address;
+            payload.channel = message.channel;
+        } else {
+            logger.info('Start Touchlink factory reset of first found device');
+            result = await this.zigbee.touchlinkFactoryResetFirst();
+        }
+
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
-            return utils.getResponse(message, {}, null);
+            return utils.getResponse(message, payload, null);
         } else {
             logger.error('Failed to factory reset device through Touchlink');
             throw new Error('Failed to factory reset device through Touchlink');
         }
     }
 
+    async touchlinkScan(message) {
+        logger.info('Start Touchlink scan');
+        const result = await this.zigbee.touchlinkScan();
+        const found = result.map((r) => {
+            return {ieee_address: r.ieeeAddr, channel: r.channel};
+        });
+        logger.info('Finished Touchlink scan');
+        return utils.getResponse(message, {found}, null);
+    }
+
     /**
      * Utils
      */
@@ -268,21 +321,38 @@ class Bridge extends Extension {
             throw new Error('No device has joined since start');
         }
 
+        const validationErrors = utils.validateFriendlyName(message.to);
+        if (validationErrors.length !== 0) {
+            throw new Error(validationErrors[0]);
+        }
+
         const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
         const to = message.to;
+        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
+            message.homeassistant_rename : false;
         const entity = this.getEntity(entityType, from);
 
         settings.changeFriendlyName(from, to);
 
+        // Clear retained messages
+        this.mqtt.publish(entity.name, '', {retain: true});
+
         if (entity.type === 'device') {
             this.publishDevices();
-            this.eventBus.emit(`deviceRenamed`, {device: entity.device});
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename});
         } else {
             this.publishGroups();
-            this.eventBus.emit(`groupRenamed`, {group: entity.group});
+            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename});
         }
 
-        return utils.getResponse(message, {from: entity.settings.friendlyName, to}, null);
+        // Repulish entity state
+        this.publishEntityState(to, {});
+
+        return utils.getResponse(
+            message,
+            {from: entity.settings.friendlyName, to, homeassistant_rename: homeAssisantRename},
+            null,
+        );
     }
 
     async removeEntity(entityType, message) {
@@ -324,7 +394,7 @@ class Bridge extends Extension {
 
             // Fire event
             if (entity.type === 'device') {
-                this.eventBus.emit('deviceRemoved', {device: entity.device});
+                this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
             }
 
             // Remove from configuration.yaml
@@ -337,9 +407,13 @@ class Bridge extends Extension {
             // Remove from state
             this.state.remove(entity.settings.ID);
 
+            // Clear any retained messages
+            this.mqtt.publish(entity.name, '', {retain: true});
+
             logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
 
             if (entity.type === 'device') {
+                this.publishGroups();
                 this.publishDevices();
                 return utils.getResponse(message, {id: ID, block, force}, null);
             } else {
@@ -362,6 +436,10 @@ class Bridge extends Extension {
     }
 
     async publishInfo() {
+        const config = objectAssignDeep.noMutate({}, settings.get());
+        delete config.advanced.network_key;
+        delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
         const payload = {
             version: this.zigbee2mqttVersion.version,
             commit: this.zigbee2mqttVersion.commitHash,
@@ -369,6 +447,7 @@ class Bridge extends Extension {
             network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
             log_level: logger.getLevel(),
             permit_join: await this.zigbee.getPermitJoin(),
+            config,
         };
 
         await this.mqtt.publish('bridge/info', stringify(payload), {retain: true, qos: 0});
@@ -378,13 +457,6 @@ class Bridge extends Extension {
         const devices = this.zigbee.getDevices().map((device) => {
             const definition = zigbeeHerdsmanConverters.findByDevice(device);
             const resolved = this.zigbee.resolveEntity(device);
-            const definitionPayload = definition ? {
-                model: definition.model,
-                vendor: definition.vendor,
-                description: definition.description,
-                supports: definition.supports,
-            } : null;
-
             const endpoints = {};
             for (const endpoint of device.endpoints) {
                 const data = {
@@ -418,7 +490,7 @@ class Bridge extends Extension {
                 network_address: device.networkAddress,
                 supported: !!definition,
                 friendly_name: resolved.name,
-                definition: definitionPayload,
+                definition: this.getDefinitionPayload(definition),
                 power_source: device.powerSource,
                 software_build_id: device.softwareBuildID,
                 date_code: device.dateCode,
@@ -436,7 +508,7 @@ class Bridge extends Extension {
             const resolved = this.zigbee.resolveEntity(group);
             return {
                 id: group.groupID,
-                friendly_name: resolved.name,
+                friendly_name: group.groupID === 901 ? 'default_bind_group' : resolved.name,
                 members: group.members.map((m) => {
                     return {
                         ieee_address: m.deviceIeeeAddress,
@@ -448,6 +520,19 @@ class Bridge extends Extension {
 
         await this.mqtt.publish('bridge/groups', stringify(groups), {retain: true, qos: 0});
     }
+
+    getDefinitionPayload(definition) {
+        if (definition) {
+            return {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                exposes: definition.exposes,
+            };
+        } else {
+            return null;
+        }
+    }
 }
 
 module.exports = Bridge;

lib/extension/configure.js

@@ -2,7 +2,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 /**
  * This extension calls the zigbee-herdsman-converters definition configure() method
@@ -53,18 +53,6 @@ class Configure extends Extension {
         return true;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (settings.get().advanced.legacy_api) {
-            this.mqtt.subscribe(this.legacyTopic);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         if (topic === this.legacyTopic) {
             const resolvedEntity = this.zigbee.resolveEntity(message);

lib/extension/frontend.js

@@ -0,0 +1,127 @@
+const http = require('http');
+const serveStatic = require('serve-static');
+const finalhandler = require('finalhandler');
+const Extension = require('./extension');
+const logger = require('../util/logger');
+const frontend = require('zigbee2mqtt-frontend');
+const WebSocket = require('ws');
+const url = require('url');
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+/**
+ * This extension servers the frontend
+ */
+class Frontend extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.onRequest = this.onRequest.bind(this);
+        this.onUpgrade = this.onUpgrade.bind(this);
+        this.mqttBaseTopic = settings.get().mqtt.base_topic;
+        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
+        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
+        this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+        this.host = settings.get().frontend.host || '0.0.0.0';
+        this.port = settings.get().frontend.port || 8080;
+        this.authToken = settings.get().frontend.auth_token || false;
+        this.retainedMessages = new Map();
+        /* istanbul ignore next */
+        const options = {setHeaders: (res, path) => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+    }
+
+    onZigbeeStarted() {
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    async stop() {
+        for (const client of this.wss.clients) {
+            client.close();
+        }
+
+        return new Promise((resolve) => {
+            this.server.close(resolve);
+        });
+    }
+    onRequest(request, response) {
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+    authenticate(request, cb) {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    onUpgrade(request, socket, head) {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    onWebSocketConnection(ws) {
+        ws.on('message', (message) => {
+            if (message) {
+                const {topic, payload} = utils.parseJSON(message, message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            let payload = {};
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.state.exists(device.ieeeAddr)) {
+                payload = {...payload, ...this.state.get(device.ieeeAddr)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+            }
+
+            if (resolvedEntity.device.linkquality !== undefined) {
+                payload.linkquality = resolvedEntity.device.linkquality;
+            }
+
+            ws.send(stringify({topic: resolvedEntity.name, payload}));
+        }
+    }
+
+    onMQTTPublishedMessage(data) {
+        let {topic, payload, options} = data;
+        if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            topic = topic.substring(this.mqttBaseTopic.length + 1);
+            payload = utils.parseJSON(payload, payload);
+            if (options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            for (const client of this.wss.clients) {
+                if (client.readyState === WebSocket.OPEN) {
+                    client.send(stringify({topic, payload}));
+                }
+            }
+        }
+    }
+}
+
+module.exports = Frontend;

lib/extension/groups.js

@@ -3,7 +3,8 @@ const logger = require('../util/logger');
 const Extension = require('./extension');
 const utils = require('../util/utils');
 const postfixes = utils.getEndpointNames();
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
+const equals = require('fast-deep-equal/es6');
 
 const topicRegex =
     new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
@@ -15,27 +16,7 @@ class Groups extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
         this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-
-            for (let step = 1; step < 20; step++) {
-                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-                this.mqtt.subscribe(`${topic}remove`);
-                this.mqtt.subscribe(`${topic}add`);
-                this.mqtt.subscribe(`${topic}remove_all`);
-            }
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
-        }
+        this.lastOptimisticState = {};
     }
 
     async onZigbeeStarted() {
@@ -119,8 +100,10 @@ class Groups extends Extension {
 
             if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint) &&
+                        !equals(this.lastOptimisticState[zigbeeGroup.groupID], payload)) {
                         if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                            this.lastOptimisticState[zigbeeGroup.groupID] = payload;
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }

lib/extension/legacy/bridgeLegacy.js

@@ -4,7 +4,7 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const utils = require('../../util/utils');
 const assert = require('assert');
 const Extension = require('../extension');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
@@ -230,6 +230,7 @@ class BridgeLegacy extends Extension {
             logger.info(`Successfully renamed - ${from} to ${to} `);
             const entity = this.zigbee.resolveEntity(to);
             const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            eventData.homeAssisantRename = false;
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
 
             this.mqtt.publish(
@@ -315,7 +316,7 @@ class BridgeLegacy extends Extension {
 
         const cleanup = () => {
             // Fire event
-            this.eventBus.emit('deviceRemoved', {device: entity.device});
+            this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
 
             // Remove from configuration.yaml
             settings.removeDevice(entity.settings.ID);
@@ -350,8 +351,6 @@ class BridgeLegacy extends Extension {
     }
 
     async onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
         await this.publish();
     }
 
@@ -451,7 +450,7 @@ class BridgeLegacy extends Extension {
             'bridge/log',
             stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
         );
-        const result = await this.zigbee.touchlinkFactoryReset();
+        const result = await this.zigbee.touchlinkFactoryResetFirst();
 
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');

lib/extension/legacy/deviceGroupMembership.js

@@ -6,13 +6,6 @@ const Extension = require('../extension');
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
 class DeviceGroupMembership extends Extension {
-    onMQTTConnected() {
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
-            this.mqtt.subscribe(topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         const match = topic.match(topicRegex);
         if (!match) {

lib/extension/networkMap.js

@@ -2,7 +2,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 /**
  * This extension creates a network map
@@ -29,18 +29,6 @@ class NetworkMap extends Extension {
         };
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(this.legacyTopic);
-            this.mqtt.subscribe(this.legacyTopicRoutes);
-        }
-
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         /* istanbul ignore else */
         if (this.legacyApi) {
@@ -48,7 +36,8 @@ class NetworkMap extends Extension {
                 this.supportedFormats.hasOwnProperty(message)) {
                 const includeRoutes = topic === this.legacyTopicRoutes;
                 const topology = await this.networkScan(includeRoutes);
-                const converted = this.supportedFormats[message](topology);
+                let converted = this.supportedFormats[message](topology);
+                converted = message === 'raw' ? stringify(converted) : converted;
                 this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
             }
         }
@@ -78,7 +67,7 @@ class NetworkMap extends Extension {
     }
 
     raw(topology) {
-        return stringify(topology);
+        return topology;
     }
 
     graphviz(topology) {
@@ -101,9 +90,8 @@ class NetworkMap extends Extension {
 
             // Add the device model
             if (node.type !== 'Coordinator') {
-                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
-                if (definition) {
-                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                if (node.definition) {
+                    labels.push(`${node.definition.vendor} ${node.definition.description} (${node.definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
                     labels.push(`${node.manufacturerName} ${node.modelID}`);
@@ -252,10 +240,22 @@ class NetworkMap extends Extension {
         // Add nodes
         for (const device of devices) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
+            const definition = resolvedEntity.definition ? {
+                model: resolvedEntity.definition.model,
+                vendor: resolvedEntity.definition.vendor,
+                description: resolvedEntity.definition.description,
+                supports: Array.from(new Set(resolvedEntity.definition.exposes.map((e) => {
+                    return e.hasOwnProperty('name') ?
+                        e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
+            } : null;
+
             networkMap.nodes.push({
                 ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+                definition,
             });
         }
 
@@ -267,6 +267,13 @@ class NetworkMap extends Extension {
                     continue;
                 }
 
+                // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
+                // Determine the correct ieeeAddr based on the networkAddress.
+                const neighborDevice = this.zigbee.getDeviceByNetworkAddress(neighbor.networkAddress);
+                if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
+                    neighbor.ieeeAddr = neighborDevice.ieeeAddr;
+                }
+
                 const link = {
                     source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
                     target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},

lib/extension/onEvent.js

@@ -27,10 +27,10 @@ class OnEvent extends Extension {
     }
 
     callOnEvent(resolvedEntity, type, data) {
-        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
 
         if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
-            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
         }
     }
 }

lib/extension/otaUpdate.js

@@ -1,6 +1,6 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
 const topicRegex =
@@ -18,18 +18,6 @@ class OTAUpdate extends Extension {
     }
 
     onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
-        }
-
         for (const device of this.zigbee.getClients()) {
             // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
             // remove them.
@@ -203,7 +191,7 @@ class OTAUpdate extends Extension {
                     const meta = {status: `update_in_progress`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        stringify({type: `ota_update`, msg, meta}),
+                        stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
@@ -238,6 +226,7 @@ class OTAUpdate extends Extension {
                     this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
                     responseData.from = from_ ? utils.toSnakeCase(from_) : null;
                     responseData.to = to ? utils.toSnakeCase(to) : null;
+                    this.eventBus.emit(`devicesChanged`);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {

lib/extension/publish.js

@@ -5,7 +5,7 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
 const Extension = require('./extension');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
 const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
@@ -13,7 +13,7 @@ const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'un
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
+    zigbeeHerdsmanConverters.toZigbeeConverters.effect,
     zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
     zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
@@ -27,18 +27,6 @@ const groupConverters = [
 ];
 
 class EntityPublish extends Extension {
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < 20; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
     parseTopic(topic) {
         const match = topic.match(topicRegex);
         if (!match) {
@@ -83,6 +71,7 @@ class EntityPublish extends Extension {
         let options = {};
         let device = null;
         let definition = null;
+        let membersState = null;
 
         assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
         if (resolvedEntity.type === 'device') {
@@ -107,6 +96,11 @@ class EntityPublish extends Extension {
                 d.toZigbee.forEach(converters.add, converters);
             }
             converters = [...converters];
+            membersState = {};
+            for (const member of resolvedEntity.group.members) {
+                const ieeeAddr = member.getDevice().ieeeAddr;
+                membersState[ieeeAddr] = this.state.get(ieeeAddr);
+            }
         }
 
         // Convert the MQTT message to a Zigbee message.
@@ -125,22 +119,26 @@ class EntityPublish extends Extension {
                     json = {state: message};
                 } else {
                     logger.error(`Invalid JSON '${message}', skipping...`);
+                    return false;
                 }
             }
         }
-
+        if (!json) {
+            logger.error(`Invalid JSON '${message}', skipping...`);
+            return;
+        }
 
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
+        const entityState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
             const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
+            const isOn = entityState && entityState.state === 'ON' ? true : false;
             if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
                 delete json.state;
                 logger.debug('Skipping state because of Home Assistant');
@@ -168,14 +166,13 @@ class EntityPublish extends Extension {
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const underscoreIndex = key.lastIndexOf('_');
-                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getEndpointNames().includes(possibleEndpointName)) {
-                    endpointName = possibleEndpointName;
-                    key = key.substring(0, underscoreIndex);
+                const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
+                if (endpointNameMatch) {
+                    endpointName = endpointNameMatch;
+                    key = key.replace(`_${endpointNameMatch}`, '');
+
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
-
                     if (!actualTarget) {
                         logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
                         continue;
@@ -187,8 +184,9 @@ class EntityPublish extends Extension {
             if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters[endpointOrGroupID].includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
+            if (topic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
 
@@ -204,7 +202,8 @@ class EntityPublish extends Extension {
                 message: json,
                 logger,
                 device,
-                state: deviceState,
+                state: entityState,
+                membersState,
                 mapped: definition,
             };
 
@@ -212,7 +211,10 @@ class EntityPublish extends Extension {
                 if (topic.type === 'set' && converter.convertSet) {
                     logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
-                    if (result && result.state) {
+                    const isGroup = actualTarget.constructor.name == 'Group';
+                    // Optimistic behaviour of groups and devices is different.
+                    const publishOptimistic = isGroup || (!options.hasOwnProperty('optimistic') || options.optimistic);
+                    if (result && result.state && publishOptimistic) {
                         const msg = result.state;
 
                         if (endpointName) {
@@ -227,6 +229,12 @@ class EntityPublish extends Extension {
                         this.publishEntityState(resolvedEntity.settings.ID, msg);
                     }
 
+                    if (result && result.membersState && publishOptimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            this.publishEntityState(ieeeAddr, state);
+                        }
+                    }
+
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
                     // So here we re-read the value after a specified timeout, this timeout could for example be the
                     // transition time of a color change or for forcing a state read for devices that don't

lib/extension/receive.js

@@ -3,7 +3,7 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
 const Extension = require('./extension');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
@@ -74,16 +74,13 @@ class Receive extends Extension {
             return false;
         }
 
-        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
-            logger.debug('Ignoring message from coordinator');
-            return false;
-        }
-
         if (!resolvedEntity.definition) {
             if (data.device.interviewing) {
                 logger.debug(`Skipping message, definition is undefined and still interviewing`);
             } else {
-                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.modelID}' ` +
+                    `and manufacturer name '${data.device.manufacturerName}'`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
             }
 
@@ -136,11 +133,6 @@ class Receive extends Extension {
         // - If NO payload is returned do nothing. This is for non-standard behaviour
         //   for e.g. click switches where we need to count number of clicks and detect long presses.
         const publish = (payload) => {
-            // Add device linkquality.
-            if (data.hasOwnProperty('linkquality')) {
-                payload.linkquality = data.linkquality;
-            }
-
             if (settings.get().advanced.elapsed) {
                 const now = Date.now();
                 if (this.elapsed[data.device.ieeeAddr]) {

lib/extension/report.js

@@ -16,6 +16,7 @@ const devicesNotSupportingReporting = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0115'),
 ];
 
 const reportKey = 1;
@@ -80,8 +81,13 @@ const pollOnMessage = [
         },
         // Read the following attributes
         read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
-        // When the bound devices/members of group have the following manufacturerID
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
     },
     {
         key: 2,
@@ -97,7 +103,12 @@ const pollOnMessage = [
             ],
         },
         read: {cluster: 'genOnOff', attributes: ['onOff']},
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+        manufacturerIDs: [
+            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
     },
 ];
 
@@ -187,6 +198,11 @@ class Report extends Extension {
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
         if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
 
+        // These do not support reproting.
+        // https://github.com/Koenkk/zigbee-herdsman/issues/110
+        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
+        if (device.manufacturerName === 'Philips' && philipsIgnoreSw.includes(device.softwareBuildID)) return false;
+
         if (resolvedEntity.device.interviewing === true) return false;
         if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
         // Gledopto devices don't support reporting.
@@ -258,7 +274,7 @@ class Report extends Extension {
 
             for (const endpoint of toPoll) {
                 for (const poll of polls) {
-                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
                         continue;
                     }
 

lib/mqtt.js

@@ -8,6 +8,7 @@ class MQTT extends events.EventEmitter {
     constructor() {
         super();
         this.onMessage = this.onMessage.bind(this);
+        this.publishedTopics = new Set();
     }
 
     async connect() {
@@ -18,7 +19,7 @@ class MQTT extends events.EventEmitter {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
                 payload: 'offline',
-                retain: true,
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
             },
         };
 
@@ -44,8 +45,11 @@ class MQTT extends events.EventEmitter {
         }
 
         if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
         }
 
         if (mqttSettings.client_id) {
@@ -69,9 +73,11 @@ class MQTT extends events.EventEmitter {
         return new Promise((resolve) => {
             this.client = mqtt.connect(mqttSettings.server, options);
 
+            const self = this;
             this.client.on('connect', () => {
                 logger.info('Connected to MQTT server');
-                this.publish('bridge/state', 'online', {retain: true, qos: 0});
+                self.subscribe(`${settings.get().mqtt.base_topic}/#`);
+                self.publish('bridge/state', 'online', {retain: true, qos: 0});
                 resolve();
             });
 
@@ -92,17 +98,24 @@ class MQTT extends events.EventEmitter {
     }
 
     onMessage(topic, message) {
-        this.emit('message', {topic, message: message.toString()});
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            this.emit('message', {topic, message: message + ''});
+        }
     }
 
     isConnected() {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -115,8 +128,16 @@ class MQTT extends events.EventEmitter {
             logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
         }
 
+        const actualOptions = {...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
+
         return new Promise((resolve) => {
-            this.client.publish(topic, payload, options, () => resolve());
+            this.client.publish(topic, payload, actualOptions, () => {
+                this.emit('publishedMessage', {topic, payload, options});
+                resolve();
+            });
         });
     }
 }

lib/state.js

@@ -3,7 +3,6 @@ const data = require('./util/data');
 const settings = require('./util/settings');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
-const stringify = require('json-stable-stringify');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
@@ -57,8 +56,12 @@ class State {
     save() {
         if (settings.get().advanced.cache_state_persistent) {
             logger.debug(`Saving state to file ${this.file}`);
-            const json = stringify(this.state, null, 4);
-            fs.writeFileSync(this.file, json, 'utf8');
+            const json = JSON.stringify(this.state, null, 4);
+            try {
+                fs.writeFileSync(this.file, json, 'utf8');
+            } catch (e) {
+                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
+            }
         } else {
             logger.debug(`Not saving state`);
         }

lib/util/logger.js

@@ -115,24 +115,27 @@ function cleanup() {
     }
 }
 
+// Print to user what logging is enabled
+function logOutput() {
+    if (output.includes('file')) {
+        if (output.includes('console')) {
+            logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+        } else {
+            logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+        }
+        logger.cleanup();
+    } else if (output.includes('console')) {
+        logger.info(`Logging to console only'`);
+    }
+}
+
 logger.cleanup = cleanup;
+logger.logOutput = logOutput;
 logger.getLevel = () => transportsToUse[0].level;
 logger.setLevel = (level) => {
     transportsToUse.forEach((transport) => transport.level = level);
 };
 
-// Print to user what logging is enabled
-if (output.includes('file')) {
-    if (output.includes('console')) {
-        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
-    } else {
-        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
-    }
-    logger.cleanup();
-} else if (output.includes('console')) {
-    logger.info(`Logging to console only'`);
-}
-
 // winston.config.syslog.levels doesnt have warn, but is required for syslog.
 /* istanbul ignore next */
 logger.warn = (message) => logger.warning(message);

lib/util/settings.js

@@ -1,5 +1,6 @@
 const data = require('./data');
 const utils = require('./utils');
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
 const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
 const objectAssignDeep = require(`object-assign-deep`);
 const path = require('path');
@@ -16,6 +17,11 @@ const defaults = {
     permit_join: false,
     mqtt: {
         include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
     },
     serial: {
         disable_led: false,
@@ -59,6 +65,7 @@ const defaults = {
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         adapter_concurrent: null,
+        adapter_delay: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
@@ -152,6 +159,7 @@ const schema = {
                 reject_unauthorized: {type: 'boolean'},
                 include_device_information: {type: 'boolean'},
                 version: {type: 'number'},
+                force_disable_retain: {type: 'boolean'},
             },
             required: ['base_topic', 'server'],
         },
@@ -160,7 +168,7 @@ const schema = {
             properties: {
                 port: {type: ['string', 'null']},
                 disable_led: {type: 'boolean'},
-                adapter: {type: 'string', enum: ['deconz', 'zstack']},
+                adapter: {type: 'string', enum: ['deconz', 'zstack', 'zigate']},
             },
         },
         blocklist: {type: 'array', items: {type: 'string'}},
@@ -207,6 +215,7 @@ const schema = {
                 homeassistant_status_topic: {type: 'string'},
                 timestamp_format: {type: 'string'},
                 adapter_concurrent: {type: 'number'},
+                adapter_delay: {type: 'number'},
             },
         },
         map_options: {
@@ -316,15 +325,27 @@ function write() {
         }
     }
 
-    if (typeof actual.devices === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
-        toWrite.devices = actual.devices;
-    }
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type) => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            const content = objectAssignDeep.noMutate(settings[type]);
 
-    if (typeof actual.groups === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
-        toWrite.groups = actual.groups;
-    }
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f, i) => i !== 0)
+                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c) => Object.keys(c))
+                    .forEach((k) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
 
     yaml.writeIfChanged(file, toWrite);
 
@@ -360,14 +381,9 @@ function validate() {
 
     // Verify that all friendly names are unique
     const names = [];
-    const postfixes = utils.getEndpointNames();
     const check = (name) => {
         if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
-        if (postfixes.includes(name)) errors.push(`Following friendly_name are not allowed: '${postfixes}'`);
-        if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
-        if (name.includes('#') || name.includes('+')) {
-            errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
-        }
+        errors.push(...utils.validateFriendlyName(name));
         names.push(name);
     };
 
@@ -424,16 +440,20 @@ function read() {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    // Read devices/groups configuration from separate file.
-    if (typeof s.devices === 'string') {
-        const file = data.joinPath(s.devices);
-        s.devices = yaml.readIfExists(file) || {};
-    }
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type) => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
 
-    if (typeof s.groups === 'string') {
-        const file = data.joinPath(s.groups);
-        s.groups = yaml.readIfExists(file) || {};
-    }
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
 
     return s;
 }

lib/util/utils.js

@@ -1,6 +1,8 @@
 const equals = require('fast-deep-equal/es6');
 const humanizeDuration = require('humanize-duration');
 const data = require('./data');
+const vm = require('vm');
+const fs = require('fs');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -37,6 +39,8 @@ const endpointNames = [
     'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
     'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool',
 ];
 
 function capitalize(s) {
@@ -67,6 +71,14 @@ async function getZigbee2mqttVersion() {
     });
 }
 
+async function getDependencyVersion(depend) {
+    return new Promise((resolve, reject) => {
+        const packageJSON = require('../../node_modules/'+depend+'/package.json');
+        const version = packageJSON.version;
+        resolve({version});
+    });
+}
+
 function formatDate(date, type) {
     let result;
 
@@ -136,13 +148,20 @@ function* getExternalConvertersDefinitions(settings) {
     const externalConverters = settings.get().external_converters;
 
     for (const moduleName of externalConverters) {
-        let converterModule = moduleName;
+        let converter;
 
         if (moduleName.endsWith('.js')) {
-            converterModule = data.joinPath(moduleName.split('.')[0]);
+            const sandbox = {
+                require,
+                module: {},
+            };
+            const converterCode = fs.readFileSync(data.joinPath(moduleName), {encoding: 'utf8'});
+            vm.runInNewContext(converterCode, sandbox);
+            converter = sandbox.module.exports;
+        } else {
+            converter = require(moduleName);
         }
 
-        const converter = require(converterModule);
         if (Array.isArray(converter)) {
             for (const item of converter) {
                 yield item;
@@ -168,10 +187,21 @@ function toSnakeCase(value) {
     }
 }
 
+function validateFriendlyName(name) {
+    const errors = [];
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+    return errors;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
+    getDependencyVersion,
     objectHasProperties,
     toSnakeCase,
     getObjectProperty,
@@ -187,4 +217,5 @@ module.exports = {
     capitalize,
     parseJSON,
     getExternalConvertersDefinitions,
+    validateFriendlyName,
 };

lib/util/yaml.js

@@ -14,8 +14,8 @@ function read(file) {
     }
 }
 
-function readIfExists(file) {
-    return fs.existsSync(file) ? read(file) : null;
+function readIfExists(file, default_=null) {
+    return fs.existsSync(file) ? read(file) : default_;
 }
 
 function writeIfChanged(file, content) {

lib/zigbee.js

@@ -7,7 +7,7 @@ const utils = require('./util/utils');
 const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
@@ -19,7 +19,8 @@ class Zigbee extends events.EventEmitter {
     }
 
     async start() {
-        logger.info(`Starting zigbee-herdsman...`);
+        const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
+        logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
         const herdsmanSettings = {
             network: {
                 panID: settings.get().advanced.pan_id,
@@ -38,6 +39,7 @@ class Zigbee extends events.EventEmitter {
             },
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
+                delay: settings.get().advanced.adapter_delay,
             },
         };
 
@@ -116,12 +118,16 @@ class Zigbee extends events.EventEmitter {
         logger.info('zigbee-herdsman stopped');
     }
 
-    async permitJoin(permit) {
+    async permitJoin(permit, resolvedEntity) {
         permit ?
-            logger.info('Zigbee: allowing new devices to join.') :
+            logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`) :
             logger.info('Zigbee: disabling joining new devices.');
 
-        await this.herdsman.permitJoin(permit);
+        if (resolvedEntity && permit) {
+            await this.herdsman.permitJoin(permit, resolvedEntity.device);
+        } else {
+            await this.herdsman.permitJoin(permit);
+        }
     }
 
     async getPermitJoin() {
@@ -287,8 +293,20 @@ class Zigbee extends events.EventEmitter {
         }
     }
 
-    async touchlinkFactoryReset() {
-        return this.herdsman.touchlinkFactoryReset();
+    async touchlinkFactoryResetFirst() {
+        return this.herdsman.touchlinkFactoryResetFirst();
+    }
+
+    async touchlinkFactoryReset(ieeeAddr, channel) {
+        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
+    }
+
+    async touchlinkIdentify(ieeeAddr, channel) {
+        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
+    }
+
+    async touchlinkScan() {
+        return this.herdsman.touchlinkScan();
     }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.14.4",
+  "version": "1.16.2",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13 || ^14"
+    "node": "^10 || ^12 || ^14 || ^15"
   },
   "keywords": [
     "xiaomi",
@@ -36,20 +36,24 @@
     "ajv": "^6.12.4",
     "debounce": "^1.2.0",
     "fast-deep-equal": "^3.1.3",
+    "finalhandler": "^1.1.2",
     "git-last-commit": "^1.0.0",
     "humanize-duration": "^3.23.1",
     "js-yaml": "^3.14.0",
-    "json-stable-stringify": "^1.0.1",
+    "json-stable-stringify-without-jsonify": "=1.0.1",
     "mkdir-recursive": "^0.4.0",
     "moment": "^2.27.0",
-    "mqtt": "^4.2.1",
+    "mqtt": "4.2.1",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
     "semver": "^7.3.2",
+    "serve-static": "^1.14.1",
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
-    "zigbee-herdsman": "0.12.127",
-    "zigbee-herdsman-converters": "12.0.178"
+    "ws": "^7.3.1",
+    "zigbee-herdsman": "0.13.37",
+    "zigbee-herdsman-converters": "13.0.7",
+    "zigbee2mqtt-frontend": "0.2.100"
   },
   "devDependencies": {
     "eslint": "*",
@@ -70,5 +74,8 @@
     "collectCoverageFrom": [
       "lib/**"
     ]
+  },
+  "bin": {
+    "zigbee2mqtt": "./cli.js"
   }
 }

scripts/dumpHomeAssistantMapping.js

@@ -0,0 +1,35 @@
+const HomeAssistant = require('../lib/extension/homeassistant');
+const stringify = require('json-stable-stringify-without-jsonify');
+const homeassistant = new HomeAssistant(null, null, null, null, {on: () => {}});
+const mapping = Object.entries(homeassistant._getMapping());
+mapping.sort((a, b) => {
+    if (a[0] > b[0]) {
+        return -1;
+    }
+    if (b[0] > a[0]) {
+        return 1;
+    }
+    return 0;
+});
+
+for (const map of mapping) {
+    map[1].sort((a, b) => {
+        return JSON.stringify(a).localeCompare(JSON.stringify(b));
+    });
+
+    const toDelete = [];
+    for (const entry of map[1]) {
+        if (entry.type === 'light') {
+            if (entry.discovery_payload.brightness === false) delete entry.discovery_payload.brightness;
+            if (entry.discovery_payload.color_temp === false) delete entry.discovery_payload.color_temp;
+            if (entry.discovery_payload.xy === false) delete entry.discovery_payload.xy;
+            if (entry.discovery_payload.hs === false) delete entry.discovery_payload.hs;
+        } else if (entry.object_id === 'linkquality') {
+            toDelete.push(entry);
+        }
+    }
+
+    toDelete.forEach((d) => map[1].splice(map[1].indexOf(d), 1));
+
+    console.log(`${map[0]} - ${stringify(map[1])}`);
+}

scripts/zStackEraseAllNvMem.js

@@ -0,0 +1,74 @@
+const {ZnpCommandStatus} = require('zigbee-herdsman/dist/adapter/z-stack/constants/common');
+const {ZnpVersion} = require('zigbee-herdsman/dist/adapter/z-stack/adapter/tstype');
+const {Subsystem} = require('zigbee-herdsman/dist/adapter/z-stack/unpi/constants');
+const {Znp} = require('zigbee-herdsman/dist/adapter/z-stack/znp');
+
+
+class ZStackNvMemEraser {
+    constructor(device) {
+        this.znp = new Znp(device, 115200, false);
+    }
+
+    async start() {
+        await this.znp.open();
+        const attempts = 3;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                await this.znp.request(Subsystem.SYS, 'ping', {capabilities: 1});
+                break;
+            } catch (e) {
+                if (attempts - 1 === i) {
+                    throw new Error(`Failed to connect to the adapter (${e})`);
+                }
+            }
+        }
+        // Old firmware did not support version, assume it's Z-Stack 1.2 for now.
+        try {
+            this.version = (await this.znp.request(Subsystem.SYS, 'version', {})).payload;
+        } catch (e) {
+            console.log(`Failed to get zStack version, assuming 1.2`);
+            this.version = {'transportrev': 2, 'product': 0, 'majorrel': 2,
+                'minorrel': 0, 'maintrel': 0, 'revision': ''};
+        }
+
+        console.log(`Detected znp version '${ZnpVersion[this.version.product]}' (${JSON.stringify(this.version)})`);
+
+        await this.clearAllNvMemItems();
+
+        process.exit(0);
+    }
+
+    async clearAllNvMemItems() {
+        let maxNvMemId;
+        switch (this.version.product) {
+        case ZnpVersion.zStack12: maxNvMemId = 0x0302; break;
+        case ZnpVersion.zStack30x: maxNvMemId = 0x033F; break;
+        case ZnpVersion.zStack3x0: maxNvMemId = 0x032F; break;
+        }
+
+        console.log(`Clearing all NVMEM items, from 0 to ${maxNvMemId}`);
+        for (let id=0; id<=maxNvMemId; id++) {
+            const lengthRes = await this.znp.request(Subsystem.SYS, 'osalNvLength',{id: id});
+            if (lengthRes.payload['length']!=0) {
+                console.log(`NVMEM item #${id} - need to delete, size: ${lengthRes.payload['length']}`);
+                await this.znp.request(Subsystem.SYS, 'osalNvDelete',
+                    {id: id, len: lengthRes.payload['length']},
+                    null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+            } else {
+                console.log(`NVMEM item #${id} - is free`);
+            }
+        }
+        console.log('Clearing all NVMEM items finished');
+    }
+}
+const processArgs = process.argv.slice(2);
+if (processArgs.length != 1) {
+    console.log('ZStack NVMEM eraser.');
+    console.log('Usage:');
+    console.log('   node zStackEraseAllNvMem.js <SERIAL_PORT>');
+    process.exit(1);
+}
+
+const eraser = new ZStackNvMemEraser(processArgs[0]);
+
+eraser.start();

test/availability.test.js

@@ -27,7 +27,8 @@ describe('Availability', () => {
         settings._reRead();
         data.writeEmptyState();
         jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10)
+        settings.set(['advanced', 'availability_timeout'], 10);
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         mocksClear.forEach((m) => m.mockClear());
         await controller.start();
@@ -80,7 +81,7 @@ describe('Availability', () => {
     });
 
     it('Should publish availabilty online and query state on reconnect', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
+        const device = zigbeeHerdsman.devices.E11_G13;
         const endpoint = device.getEndpoint(1);
         device.ping.mockImplementationOnce(() => {throw new Error('failed')});
         jest.advanceTimersByTime(11 * 1000);
@@ -89,16 +90,15 @@ describe('Availability', () => {
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
+            'zigbee2mqtt/bulb_enddevice/availability',
           'online',
           { retain: true, qos: 0 },
           expect.any(Function)
         );
 
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
     });
 
     it('Should fail gracefully when quering state after reconnect fails', async () => {
@@ -172,10 +172,11 @@ describe('Availability', () => {
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(4);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorTemperature']);
     });
 
     it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
@@ -397,4 +398,15 @@ describe('Availability', () => {
           expect.any(Function)
         );
     });
+
+    it('Should clear retained availability topic when device is remove', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
 });

test/bind.test.js

@@ -5,7 +5,7 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('Bind', () => {
     let controller;
@@ -33,12 +33,6 @@ describe('Bind', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
-    });
-
     it('Should bind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
@@ -50,11 +44,19 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
             stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
     });
 
     it('Should bind only specifief clusters', async () => {
@@ -266,7 +268,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -297,7 +299,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -318,7 +320,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -338,7 +340,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -358,7 +360,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -412,7 +414,7 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');

test/configure.test.js

@@ -6,7 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 

test/controller.test.js

@@ -6,7 +6,7 @@ const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const flushPromises = () => new Promise(setImmediate);
 const tmp = require('tmp');
 const mocksClear = [
@@ -31,7 +31,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -490,12 +490,12 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": true}, expect.any(Function));
 
         // Unsupported device should have model "unknown"
         await controller.publishEntityState('unsupported2', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state without retain', async () => {
@@ -551,6 +551,15 @@ describe('Controller', () => {
         expect(data.stateExists()).toBeFalsy();
     });
 
+    it('Shouldnt crash when it cannot save state', async () => {
+        data.removeState();
+        await controller.start();
+        logger.error.mockClear();
+        controller.state.file = "/";
+        await controller.state.save();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to write state to '/' (EISDIR: illegal operation on a directory, open '/')`);
+    });
+
     it('Publish should not cache when set', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeEmptyState();
@@ -589,4 +598,24 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
     });
 
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
+
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
 });

test/frontend.test.js

@@ -0,0 +1,215 @@
+const data = require('./stub/data');
+require('./stub/logger');
+require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mockHTTP = {
+    implementation: {
+        listen: jest.fn(),
+        on: (event, handler) => {mockHTTP.events[event] = handler},
+        close: jest.fn().mockImplementation((cb) => cb()),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockWSocket = {
+    close: jest.fn(),
+};
+
+const mockWS = {
+    implementation: {
+        clients: [],
+        on: (event, handler) => {mockWS.events[event] = handler},
+        handleUpgrade: jest.fn().mockImplementation((request, socket, head, cb) => {
+            cb(mockWSocket)
+        }),
+        emit: jest.fn(),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockNodeStatic = {
+    implementation: jest.fn(),
+    variables: {},
+    events: {},
+};
+
+jest.mock('http', () => ({
+    createServer: jest.fn().mockImplementation((onRequest) => {
+        mockHTTP.variables.onRequest = onRequest;
+        return mockHTTP.implementation;
+    }),
+}));
+
+jest.mock("serve-static", () =>
+    jest.fn().mockImplementation((path) => {
+        mockNodeStatic.variables.path = path
+        return mockNodeStatic.implementation
+    })
+);
+
+jest.mock('zigbee2mqtt-frontend', () => ({
+    getPath: () => 'my/dummy/path',
+}));
+
+jest.mock('ws', () => ({
+    OPEN: 'open',
+    Server: jest.fn().mockImplementation(() => {
+        return mockWS.implementation;
+    }),
+}));
+
+describe('Frontend', () => {
+    let controller;
+
+    beforeEach(async () => {
+        mockWS.implementation.clients = [];
+        data.writeDefaultConfiguration();
+        data.writeDefaultState();
+        settings._reRead();
+        settings.set(['experimental'], {new_api: true});
+        settings.set(['frontend'], {port: 8081, host: "127.0.0.1"});
+        settings.set(['homeassistant'], true);
+        zigbeeHerdsman.devices.bulb.linkquality = 10;
+    });
+
+    afterEach(async() => {
+        delete zigbeeHerdsman.devices.bulb.linkquality;
+    });
+
+    it('Start/stop', async () => {
+        controller = new Controller();
+        await controller.start();
+        expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+
+        const mockWSClient = {
+            implementation: {
+                close: jest.fn(),
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await controller.stop();
+        expect(mockWSClient.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
+    });
+
+    it('Websocket interaction', async () => {
+        controller = new Controller();
+        await controller.start();
+
+        // Connect
+        const mockWSClient = {
+            implementation: {
+                on: (event, handler) => {mockWSClient.events[event] = handler},
+                send: jest.fn(),
+                readyState: 'open',
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(48);
+
+        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[0])).toStrictEqual({topic: 'bridge/state', payload: 'online'});
+        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}});
+
+        // Message
+        MQTT.publish.mockClear();
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        mockWSClient.events.message(undefined);
+        mockWSClient.events.message("");
+        mockWSClient.events.message(null);
+        await flushPromises();
+
+        // Received message on socket
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(4);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON'}}));
+
+        // Shouldnt set when not ready
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'close';
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(0);
+
+        // Send last seen on connect
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'open';
+        settings.set(['advanced'], {last_seen: 'ISO_8601'});
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}});
+    });
+
+    it('onReques/onUpgrade', async () => {
+        controller = new Controller();
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: 'http://localhost:8080/api'}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "http://localhost:8080/api"}, mockSocket, 3, expect.any(Function));
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](99);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', 99, {"url": "http://localhost:8080/api"});
+
+        mockHTTP.variables.onRequest(1, 2);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledTimes(1);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledWith(1, 2, expect.any(Function));
+    });
+
+    it('Static server', async () => {
+        controller = new Controller();
+        await controller.start();
+
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+    });
+
+    it('Authentification', async () => {
+        const authToken = 'sample-secure-token'
+        settings.set(['frontend'], {auth_token: authToken});
+        controller = new Controller();
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: '/api'}, mockSocket, mockWSocket);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "/api"}, mockSocket, mockWSocket, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledWith(4401, "Unauthorized");
+
+        mockWSocket.close.mockClear();
+        mockWS.implementation.emit.mockClear();
+
+        const url = `/api?token=${authToken}`;
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: url}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({url}, mockSocket, 3, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledTimes(0);
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](mockWSocket);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', mockWSocket, {url});
+
+    });
+});

test/group.test.js

@@ -1,12 +1,15 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
@@ -124,12 +127,6 @@ describe('Groups', () => {
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}), {"retain": false, qos: 0}, expect.any(Function));
-
-        // Test if subscribed to topics with slashes
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
     });
 
     it('Legacy api: Add to group via MQTT with postfix', async () => {
@@ -293,10 +290,28 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","linkquality":10}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
+    it('Should not republish identical optimistic group states', async () => {
+        const device1 = zigbeeHerdsman.devices.bulb_2;
+        const device2 = zigbeeHerdsman.devices.bulb_color_2;
+        const group = zigbeeHerdsman.groups.group_tradfri_remote;
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
     it('Should publish state change of all members when a group changes its state', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -346,7 +361,7 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","linkquality":10}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of another group with shared device when a group changes its state', async () => {
@@ -392,7 +407,7 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -417,8 +432,8 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -443,9 +458,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"OFF","brightness":0}), {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"OFF"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -471,8 +486,8 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Add to group via MQTT', async () => {
@@ -736,4 +751,4 @@ describe('Groups', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
-});
+});

test/homeassistant.test.js

@@ -1,6 +1,6 @@
 const data = require('./stub/data');
 const settings = require('../lib/util/settings');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const flushPromises = () => new Promise(setImmediate);
@@ -22,19 +22,6 @@ describe('HomeAssistant extension', () => {
         settings.set(['homeassistant'], true);
     });
 
-    it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
-        const missing = [];
-        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
-
-        require('zigbee-herdsman-converters').devices.forEach((d) => {
-            if (!ha._getMapping()[d.model]) {
-                missing.push(d.model);
-            }
-        });
-
-        expect(missing).toHaveLength(0);
-    });
-
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
@@ -77,7 +64,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -102,7 +89,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -127,7 +114,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -152,7 +139,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -177,7 +164,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -186,6 +173,114 @@ describe('HomeAssistant extension', () => {
             { retain: true, qos: 0 },
             expect.any(Function),
         );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "command_topic":"zigbee2mqtt/wall_switch_double/left/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45542"
+                ],
+                "manufacturer":"Xiaomi",
+                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
+                "name":"wall_switch_double",
+                "sw_version":this.version
+            },
+            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
+            "name":"wall_switch_double_left",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/wall_switch_double",
+            "unique_id":"0x0017880104e45542_switch_left_zigbee2mqtt",
+            "value_template":"{{ value_json.state_left }}"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/0x0017880104e45542/switch_left/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "command_topic":"zigbee2mqtt/wall_switch_double/right/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45542"
+                ],
+                "manufacturer":"Xiaomi",
+                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
+                "name":"wall_switch_double",
+                "sw_version":this.version
+            },
+            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
+            "name":"wall_switch_double_right",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/wall_switch_double",
+            "unique_id":"0x0017880104e45542_switch_right_zigbee2mqtt",
+            "value_template":"{{ value_json.state_right }}"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/0x0017880104e45542/switch_right/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_temp":true,
+            "command_topic":"zigbee2mqtt/bulb/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x000b57fffec6a5b2"
+                ],
+                "manufacturer":"IKEA",
+                "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
+                "name":"bulb",
+                "sw_version":this.version,
+            },
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
+            "hs":false,
+            "json_attributes_topic":"zigbee2mqtt/bulb",
+            "name":"bulb",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/bulb",
+            "unique_id":"0x000b57fffec6a5b2_light_zigbee2mqtt",
+            "xy":false
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/0x000b57fffec6a5b2/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
     });
 
     it('Should discover devices with precision', async () => {
@@ -218,7 +313,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -243,7 +338,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -268,7 +363,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -324,7 +419,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'custom model',
                 'manufacturer': 'From Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             'expire_after': 90,
             'icon': 'mdi:test',
         };
@@ -350,7 +445,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'custom model',
                 'manufacturer': 'Not from Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             'expire_after': 30,
             'icon': 'mdi:test',
         };
@@ -363,6 +458,58 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should discover devices with overriden user configuration affecting type and object_id', async () => {
+        settings.set(['devices', '0x0017880104e45541'], {
+            friendly_name: 'my_switch',
+            homeassistant: {
+                switch: {
+                    type: 'light',
+                    object_id: 'light'
+                },
+                light: {
+                    type: 'this should be ignored',
+                    name: 'my_light_name_override'
+                }
+
+            },
+        })
+
+        controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            "command_topic": "zigbee2mqtt/my_switch/set",
+            "device": {
+              "identifiers": [
+                "zigbee2mqtt_0x0017880104e45541"
+              ],
+              "manufacturer": "Xiaomi",
+              "model": "Aqara single key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG04LM)",
+              "name": "my_switch",
+              "sw_version": this.version
+            },
+            "json_attributes_topic": "zigbee2mqtt/my_switch",
+            "name": "my_light_name_override",
+            "payload_off": "OFF",
+            "payload_on": "ON",
+            "state_topic": "zigbee2mqtt/my_switch",
+            "unique_id": "0x0017880104e45541_light_zigbee2mqtt",
+            "value_template": "{{ value_json.state }}"
+        }
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/0x0017880104e45541/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+    });
+
     it('Shouldnt discover devices when homeassistant null is set in device options', async () => {
         settings.set(['devices', '0x0017880104e45522'], {
             homeassistant: null,
@@ -404,7 +551,7 @@ describe('HomeAssistant extension', () => {
                "smart"
             ],
             "json_attributes_topic":"zigbee2mqtt/fan",
-            "name":"fan_fan",
+            "name":"fan",
             "unique_id":"0x0017880104e45548_fan_zigbee2mqtt",
             "device":{
                "identifiers":[
@@ -415,7 +562,7 @@ describe('HomeAssistant extension', () => {
                "model":"Universal wink enabled white ceiling fan premier remote control (99432)",
                "manufacturer":"Hampton Bay"
             },
-            "availability_topic":"zigbee2mqtt/bridge/state"
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
          };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -436,7 +583,10 @@ describe('HomeAssistant extension', () => {
         payload = {
             "action_template":"{% set values = {'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'} %}{{ values[value_json.running_state] }}",
             "action_topic":"zigbee2mqtt/TS0601_thermostat",
-            "availability_topic":"zigbee2mqtt/bridge/state",
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+            "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
+            "away_mode_state_template":"{{ value_json.away_mode }}",
+            "away_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "current_temperature_template":"{{ value_json.local_temperature }}",
             "current_temperature_topic":"zigbee2mqtt/TS0601_thermostat",
             "device":{
@@ -452,7 +602,6 @@ describe('HomeAssistant extension', () => {
             "hold_modes":[
                "schedule",
                "manual",
-               "away",
                "boost",
                "complex",
                "comfort",
@@ -461,7 +610,7 @@ describe('HomeAssistant extension', () => {
             "hold_state_template":"{{ value_json.preset }}",
             "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "json_attributes_topic":"zigbee2mqtt/TS0601_thermostat",
-            "max_temp":"30",
+            "max_temp":"35",
             "min_temp":"5",
             "mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/system_mode",
             "mode_state_template":"{{ value_json.system_mode }}",
@@ -469,7 +618,7 @@ describe('HomeAssistant extension', () => {
             "modes":[
                "auto"
             ],
-            "name":"TS0601_thermostat_climate",
+            "name":"TS0601_thermostat",
             "temp_step":0.5,
             "temperature_command_topic":"zigbee2mqtt/TS0601_thermostat/set/current_heating_setpoint",
             "temperature_state_template":"{{ value_json.current_heating_setpoint }}",
@@ -500,7 +649,7 @@ describe('HomeAssistant extension', () => {
             set_position_template: '{ "position": {{ position }} }',
             value_template: '{{ value_json.position }}',
             json_attributes_topic: 'zigbee2mqtt/smart vent',
-            name: 'smart vent cover',
+            name: 'smart vent',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
             device:
             {
@@ -510,7 +659,7 @@ describe('HomeAssistant extension', () => {
                 model: 'Smart vent (SV01)',
                 manufacturer: 'Keen Home'
             },
-            availability_topic: 'zigbee2mqtt/bridge/state'
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -544,7 +693,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -609,7 +758,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -737,7 +886,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/weather_sensor/availability',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -799,11 +948,14 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should refresh discovery when device is renamed', async () => {
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller(false);
         await controller.start();
         await flushPromises();
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45522/action_double/config', stringify({topic: 'zigbee2mqtt/weather_sensor/action'}));
+        await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', '{"old": "weather_sensor", "new": "weather_sensor_renamed"}');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":true}));
         await flushPromises();
 
         const payload = {
@@ -821,7 +973,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability_topic': 'zigbee2mqtt/bridge/state',
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -837,6 +989,70 @@ describe('HomeAssistant extension', () => {
             { retain: true, qos: 0 },
             expect.any(Function),
         );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45522/action_double/config',
+            stringify({
+                "automation_type":"trigger",
+                "type":"action",
+                "subtype":"double",
+                "payload":"double",
+                "topic":"zigbee2mqtt/weather_sensor_renamed/action",
+                "device":{
+                    "identifiers":[
+                        "zigbee2mqtt_0x0017880104e45522"
+                    ],
+                    "name":"weather_sensor_renamed",
+                    "sw_version": this.version,
+                    "model":"Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)",
+                    "manufacturer":"Xiaomi"
+                }
+            }),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
+        settings.set(['experimental', 'new_api'], true);
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "weather_sensor", "to": "weather_sensor_renamed","homeassistant_rename":false}));
+        await flushPromises();
+
+        expect(MQTT.publish).not.toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            null,
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        const payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
+            'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
+            'name': 'weather_sensor_renamed_temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'weather_sensor_renamed',
+                'sw_version': this.version,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
     });
 
     it('Should discover update_available sensor when device supports it', async () => {
@@ -860,7 +1076,7 @@ describe('HomeAssistant extension', () => {
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "manufacturer":"IKEA"
             },
-            "availability_topic":"zigbee2mqtt/bridge/state"
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -884,8 +1100,8 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
 
         const device = zigbeeHerdsman.devices.WXKG11LM;
-        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        const payload1 = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
         await flushPromises();
 
         const discoverPayloadAction = {
@@ -952,28 +1168,28 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "single", linkquality: 10, click: "single"}),
+            stringify({action: "single", click: "single"}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({linkquality: 10, action: ""}),
+            stringify({action: ""}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({linkquality: 10, click: ""}),
+            stringify({click: ""}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         // Should only discover it once
         MQTT.publish.mockClear();
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.message(payload1);
         await flushPromises();
         expect(MQTT.publish).not.toHaveBeenCalledWith(
             'homeassistant/device_automation/0x0017880104e45520/action_single/config',
@@ -1002,6 +1218,26 @@ describe('HomeAssistant extension', () => {
             { retain: false, qos: 0 },
             expect.any(Function),
         );
+
+        // Shouldn't rediscover when already discovered in previous session
+        controller.extensions.find((e) => e.constructor.name === 'HomeAssistant')._clearDiscoveredTrigger();
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45520/action_double/config', stringify({topic: 'zigbee2mqtt/button/action'}));
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45520/action_double/config', stringify({topic: 'zigbee2mqtt/button/action'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const payload2 = {data: {'32768': 2}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
+
+        // Should rediscover when already discovered in previous session but with diferent name
+        controller.extensions.find((e) => e.constructor.name === 'HomeAssistant')._clearDiscoveredTrigger();
+        await MQTT.events.message('homeassistant/device_automation/0x0017880104e45520/action_double/config', stringify({topic: 'zigbee2mqtt/button_other_name/action'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload2);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
     });
 
     it('Should not discover sensor_click when legacy: false is set', async () => {
@@ -1073,7 +1309,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "single", linkquality: 10}),
+            stringify({action: "single"}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -1119,10 +1355,10 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: ''});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/action_single/config');
         expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/action');
@@ -1149,7 +1385,7 @@ describe('HomeAssistant extension', () => {
                 command_topic: true,
             },
         };
-        expect(ha._getMapping()['external_converters_device']).toStrictEqual([homeassistantSwitch]);
+        expect(ha._getMapping()['external_converters_device']).toEqual([homeassistantSwitch]);
     });
 
     it('Should clear outdated configs', async () => {
@@ -1159,26 +1395,26 @@ describe('HomeAssistant extension', () => {
 
         // Non-existing device -> clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability_topic: 'zigbee2mqtt/0x123/availability'}));
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x123/temperature/config', null, {qos: 0, retain: true}, expect.any(Function));
 
         // Existing device -> don't clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability_topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await MQTT.events.message('homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
 
         // Non-existing device of different instance -> don't clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability_topic: 'zigbee2mqtt_different/0x123/availability'}));
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt_different/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
 
         // Existing device but non-existing config -> don't clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability_topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', null, {qos: 0, retain: true}, expect.any(Function));
@@ -1188,5 +1424,17 @@ describe('HomeAssistant extension', () => {
         await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', '1}3');
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Existing device, device automation -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', stringify({topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Device automation of different instance -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/device_automation/0x000b57fffec6a5b2_not_existing/action_button_3_single/config', stringify({topic: 'zigbee2mqtt_different/0x000b57fffec6a5b2_not_existing/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 });

test/legacy/bridgeLegacy.test.js

@@ -2,7 +2,7 @@ const data = require('../stub/data');
 const logger = require('../stub/logger');
 const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
 const MQTT = require('../stub/mqtt');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
@@ -178,7 +178,7 @@ describe('Bridge legacy', () => {
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
         expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
         expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
-        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "softwareBuildID": null, "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
         Date.now = now;
     });
 
@@ -188,7 +188,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 12, "friendly_name": "thermostat_group", "retain": false, "devices": ['TS0601_thermostat'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","optimistic":true,"retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","optimistic":true,"retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","optimistic":true,"retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","optimistic":true,"retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","optimistic":true,"retain":false},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","optimistic":true,"retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {
@@ -457,22 +457,22 @@ describe('Bridge legacy', () => {
     });
 
     it('Should allow to touchlink factory reset (OK)', async () => {
-        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
 
-        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(true);
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
-        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
         expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
     });
 
     it('Should allow to touchlink factory reset (FAILS)', async () => {
-        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
 
-        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(false);
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
-        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
         expect(logger.warn).toHaveBeenCalledWith('Failed to factory reset device through Touchlink');
     });
 });

test/logger.test.js

@@ -24,12 +24,14 @@ describe('Logger', () => {
 
     it('Create log directory', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const dirs = fs.readdirSync(dir.name);
         expect(dirs.length).toBe(1);
     });
 
     it('Should cleanup', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
 
         for (const d of fs.readdirSync(dir.name)) {
             rimraf.sync(path.join(dir.name, d));
@@ -46,6 +48,7 @@ describe('Logger', () => {
 
     it('Should not cleanup when there is no timestamp set', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         for (let i = 30; i < 40; i++) {
             fs.mkdirSync(path.join(dir.name, `log_${i}`));
         }
@@ -58,12 +61,14 @@ describe('Logger', () => {
 
     it('Set and get log level', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         logger.setLevel('debug');
         expect(logger.getLevel()).toBe('debug');
     });
 
     it('Logger should be console and file by default', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
@@ -75,6 +80,7 @@ describe('Logger', () => {
     it('Logger can be file only', () => {
         settings.set(['advanced', 'log_output'], ['file']);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
@@ -86,6 +92,7 @@ describe('Logger', () => {
     it('Logger can be console only', () => {
         settings.set(['advanced', 'log_output'], ['console']);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(false);
@@ -94,6 +101,7 @@ describe('Logger', () => {
     it('Logger can be nothing', () => {
         settings.set(['advanced', 'log_output'], []);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
@@ -102,6 +110,7 @@ describe('Logger', () => {
     it('Should allow to disable log rotation', () => {
         settings.set(['advanced', 'log_rotation'], false);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes[1].constructor.name).toBe('File');
         expect(pipes[1].maxFiles).toBeNull();

test/networkMap.test.js

@@ -1,7 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
 
@@ -37,6 +37,8 @@ describe('Networkmap', () => {
     beforeEach(async () => {
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = 1000;
     });
 
     function mock() {
@@ -68,7 +70,7 @@ describe('Networkmap', () => {
         bulb_color.routingTable = () => {return {table: []}};
 
         CC2530_ROUTER.lqi = () => {return {neighbors: [
-            {ieeeAddr: WXKG02LM.ieeeAddr, networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
+            {ieeeAddr: '0x0000000000000000', networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
             {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 4, depth: 2, linkquality: 130},
         ]}};
         CC2530_ROUTER.routingTable = () => {return {table: []}};
@@ -85,26 +87,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            "nodes":[
-                {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-            ],
-            "links":[
-                {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-            ]
-        };
-
+        const expected = {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch","model":"WXKG02LM","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -265,35 +248,8 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            status: 'ok',
-            data: {
-                type: 'raw',
-                routes: true,
-                value: {
-                    "nodes":[
-                        {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                        {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                        {"lastSeen": null,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                        {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                        {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                        {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-                    ],
-                    "links":[
-                        {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                        {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                        {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                        {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                        {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-                    ]
-                }
-            }
-        };
-
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch","model":"WXKG02LM","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
-        actual.data.value = JSON.parse(actual.data.value);
         expect(actual).toStrictEqual(expected);
     });
 

test/otaUpdate.test.js

@@ -6,7 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('OTA update', () => {
     let controller;
@@ -27,13 +27,6 @@ describe('OTA update', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/update');
-    });
-
     it('Should OTA update a device', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
@@ -83,6 +76,12 @@ describe('OTA update', () => {
             stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
     });
 
     it('Should handle when OTA update fails', async () => {

test/publish.test.js

@@ -2,7 +2,7 @@ const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -66,6 +66,18 @@ describe('Publish', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
+    it('Should corretly handle mallformed messages', async () => {
+        await MQTT.events.message('zigbee2mqtt/foo', undefined);
+        await MQTT.events.message('zigbee2mqtt/foo', null);
+        await MQTT.events.message('zigbee2mqtt/foo', "");
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', undefined);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', null);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', "");
+        await flushPromises();
+        expectNothingPublished();
+    });
+
     it('Should publish messages to zigbee devices when there is no converters', async () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness_no: '200'}));
         await flushPromises();
@@ -260,7 +272,7 @@ describe('Publish', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF', brightness: 0});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -270,10 +282,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -282,10 +294,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -327,7 +339,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/thermostat_group/set', stringify({child_lock: 'LOCK'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
-        expect(group.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 263, fn: 0, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 7, length_hi: 0, length_lo: 1, status: 0, transid: 0}, {disableDefaultResponse: true});
     });
 
     it('Should publish messages to groups with on and brightness', async () => {
@@ -349,7 +361,7 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
     });
 
     it('Should publish messages to groups color', async () => {
@@ -376,14 +388,25 @@ describe('Publish', () => {
 
     it('Should create and publish to group which is in configuration.yaml but not in zigbee-herdsman', async () => {
         delete zigbeeHerdsman.groups.group_2;
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(5);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(7);
         await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'ON'}));
         await flushPromises();
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(6);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(8);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
     });
 
+    it('Shouldnt publish new state when optimistic = false', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        settings.set(['devices', device.ieeeAddr, 'optimistic'], false);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {"level": 200, "transtime": 0}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
     it('Should handle non-valid topics', async () => {
         await MQTT.events.message('zigbee2mqtt1/bulb_color/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -417,10 +440,11 @@ describe('Publish', () => {
     it('Should handle get', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        await MQTT.events.message('zigbee2mqtt/bulb_color/get', stringify({state: ''}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/get', stringify({state: '', brightness: ''}));
         await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(1);
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
     });
 
     it('Should handle get with multiple endpoints', async () => {
@@ -512,10 +536,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y: 0.6584}});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -639,7 +663,36 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColor", {"colorx": 45940, "colory": 19595, "transtime": 0}, {}]);
         expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read.mock.calls[0]).toEqual(["genOnOff", ["onOff"]]);
-        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]]);
+        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["currentX", "currentY"]]);
+    });
+
+    it('Should also use on/off cluster when controlling group with switch', async () => {
+        const group = zigbeeHerdsman.groups.group_with_switch;
+
+        MQTT.publish.mockClear();
+        group.command.mockClear();
+        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'ON', brightness: 100}));
+        await flushPromises();
+        expect(group.command).toHaveBeenCalledTimes(2);
+        expect(group.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 100, transtime: 0}, {});
+        expect(group.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/switch_group',
+            stringify({state: 'ON', brightness: 100}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        MQTT.publish.mockClear();
+        group.command.mockClear();
+        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF', brightness: 100}));
+        await flushPromises();
+        expect(group.command).toHaveBeenCalledTimes(1);
+        expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/switch_group',
+            stringify({state: 'OFF', brightness: 100}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should use transition when brightness with group', async () => {
@@ -686,7 +739,7 @@ describe('Publish', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
             'zigbee2mqtt/bulb_color',
-            stringify({state: 'OFF', brightness: 0}),
+            stringify({state: 'OFF'}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(2,
@@ -779,12 +832,12 @@ describe('Publish', () => {
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
-        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 0}, {}]);
         expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should turn device off when brightness 0 is send with transition', async () => {
@@ -799,11 +852,11 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
-        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should allow to set color via hue and saturation', async () => {
@@ -889,7 +942,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 10}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
     });
 
     it('When device is turned off and on with transition with report enabled it should restore correct brightness', async () => {
@@ -907,13 +960,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Bulb reports brightness while decreasing brightness
         await zigbeeHerdsman.events.message({data: {currentLevel: 1}, cluster: 'genLevelCtrl', device, endpoint, type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0, linkquality: 10}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 1}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Turn on again
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON', transition: 3}));
@@ -921,7 +974,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 200, linkquality: 10}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'ON', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('When device is turned off with transition and turned on WITHOUT transition it should restore the brightness', async () => {
@@ -939,7 +992,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Turn on again
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
@@ -1028,11 +1081,11 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
             'zigbee2mqtt/bulb_color',
-            stringify({state: 'OFF', brightness: 0}),
+            stringify({state: 'OFF'}),
             {retain: false, qos: 0}, expect.any(Function)
         );
 
-        // Toggle again, now that we have state it should publish state ON and retrieve the brightness from the bulb.
+        // Toggle again, now that we have state it should publish state ON
         endpoint.read.mockImplementationOnce((cluster, attrs) => {
             if (cluster === 'genLevelCtrl' && attrs.includes('currentLevel')) {
                 return {currentLevel: 100};
@@ -1044,32 +1097,11 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish).toHaveBeenNthCalledWith(2,
             'zigbee2mqtt/bulb_color',
-            stringify({state: 'ON', brightness: 100}),
+            stringify({state: 'ON'}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
 
-    it('Should restore brightness when state hass brightness 0 and bulb is turned ON', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const endpoint = device.getEndpoint(1);
-        endpoint.read.mockImplementationOnce((cluster, attrs) => {
-            if (cluster === 'genLevelCtrl' && attrs.includes('currentLevel')) {
-                return {currentLevel: 100};
-            }
-        });
-        controller.state.state = {[device.ieeeAddr]: {state: "OFF", brightness: 0,}};
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: "OFF"}));
-        await flushPromises();
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: "ON"}));
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenNthCalledWith(1, 'zigbee2mqtt/bulb_color', stringify({state: 'OFF', brightness: 0}), {retain: false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenNthCalledWith(2, 'zigbee2mqtt/bulb_color', stringify({state: 'ON', brightness: 100}), {retain: false, qos: 0}, expect.any(Function));
-        expect(endpoint.command).toHaveBeenCalledTimes(2);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "off", {}, {}]);
-        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "on", {}, {}]);
-    });
-
     it('Should publish messages with options disableDefaultResponse', async () => {
         const device = zigbeeHerdsman.devices.GLEDOPTO1112;
         const endpoint = device.getEndpoint(11);
@@ -1079,7 +1111,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/led_controller_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -1171,7 +1203,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 3, length_hi: 0, length_lo: 1, status: 0, transid: 1}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});
@@ -1256,7 +1288,58 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/bulb_color', stringify({"state":"ON"}), { qos: 0, retain: false }, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[1]).toEqual([ 'zigbee2mqtt/bulb_color', stringify({"state":"ON","brightness":150}), { qos: 0, retain: false }, expect.any(Function)]);
-        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', stringify({"state":"OFF","brightness":0}), { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', stringify({"state":"OFF","brightness":150}), { qos: 0, retain: false }, expect.any(Function)]);
         expect(MQTT.publish.mock.calls[3]).toEqual([ 'zigbee2mqtt/bulb_color', stringify({"state":"ON","brightness":150}), { qos: 0, retain: false }, expect.any(Function)]);
     });
+
+    it('Scenes', async () => {
+        const bulb_color_2 = zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1);
+        const bulb_2 = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_tradfri_remote;
+        await MQTT.events.message('zigbee2mqtt/bulb_color_2/set', stringify({"state": "ON", "brightness": 50, "color_temp": 290}));
+        await MQTT.events.message('zigbee2mqtt/bulb_2/set', stringify({"state": "ON", "brightness": 100}));
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_tradfri_remote/set', stringify({"scene_store": 1}));
+        await flushPromises();
+        expect(group.command).toHaveBeenCalledTimes(1);
+        expect(group.command).toHaveBeenCalledWith('genScenes', 'store', { groupid: 15071, sceneid: 1 }, {});
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color_2/set', stringify({"state": "ON", "brightness": 250, "color_temp": 20}));
+        await MQTT.events.message('zigbee2mqtt/bulb_2/set', stringify({"state": "ON", "brightness": 110}));
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        group.command.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_tradfri_remote/set', stringify({"scene_recall": 1}));
+        await flushPromises();
+        expect(group.command).toHaveBeenCalledTimes(1);
+        expect(group.command).toHaveBeenCalledWith('genScenes', 'recall', { groupid: 15071, sceneid: 1 }, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/group_tradfri_remote',
+            stringify({"brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2,
+            'zigbee2mqtt/bulb_color_2',
+            stringify({"brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(3,
+            'zigbee2mqtt/group_tradfri_remote',
+            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(4,
+            'zigbee2mqtt/bulb_2',
+            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(5,
+            'zigbee2mqtt/group_with_tradfri',
+            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
 });

test/receive.test.js

@@ -1,6 +1,6 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
@@ -20,10 +20,12 @@ describe('Receive', () => {
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
+        delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
     });
 
     it('Should handle a zigbee message', async () => {
         const device = zigbeeHerdsman.devices.WXKG11LM;
+        device.linkquality = 10;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -40,7 +42,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', action: 'single_left', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', action: 'single_left'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -52,7 +54,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', action: 'single_right', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', action: 'single_right'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -64,7 +66,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -75,7 +77,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20, linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20}), {retain: false, qos: 0}, expect.any(Function));
 
         // Inverted
         MQTT.publish.mockClear();
@@ -83,7 +85,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80, linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should allow to disable the legacy integration', async () => {
@@ -94,7 +96,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single'}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should debounce messages', async () => {
@@ -117,7 +119,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -143,7 +145,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": true});
     });
 
@@ -163,11 +165,11 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2, linkquality: 13});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2});
         jest.runAllTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03});
     });
 
     it('Shouldnt republish old state', async () => {
@@ -181,7 +183,7 @@ describe('Receive', () => {
         jest.runAllTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
     });
 
     it('Should handle a zigbee message with 1 precision', async () => {
@@ -193,7 +195,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -206,7 +208,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -219,7 +221,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -233,7 +235,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -246,14 +248,14 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5,"linkquality":10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5});
 
         MQTT.publish.mockClear();
         payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4,"linkquality":10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4});
     });
 
     it('Should handle a zigbee message with voltage 3010', async () => {
@@ -264,7 +266,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -276,7 +278,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -288,7 +290,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -300,7 +302,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -312,7 +314,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -372,7 +374,7 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/weather_sensor',
-            stringify({temperature: -0.85, linkquality: 10}),
+            stringify({temperature: -0.85}),
             {"qos": 1, "retain": false},
             expect.any(Function),
         )
@@ -384,10 +386,20 @@ describe('Receive', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/power_plug');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'state': 'ON', linkquality: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/power_plug',
+            stringify({state: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/switch_group',
+            stringify({'state': 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+
     });
 
     it('Should not handle messages from coordinator', async () => {
@@ -406,7 +418,7 @@ describe('Receive', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID' and manufacturer name 'notSupportedMfg'`);
         expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
@@ -430,7 +442,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -477,7 +489,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6});
 
         MQTT.publish.mockClear();
         const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
@@ -485,6 +497,6 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496});
     });
 });

test/report.test.js

@@ -267,7 +267,7 @@ describe('Report', () => {
         expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
 
         // Should only call Hue bulb, not e.g. tradfri
-        expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
     });
 
     it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {

test/settings.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings.js');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
+const devicesFile2 = data.joinPath('devices2.yaml');
 const groupsFile = data.joinPath('groups.yaml');
 const secretFile = data.joinPath('secret.yaml');
 const yaml = require('js-yaml');
@@ -29,7 +30,7 @@ describe('Settings', () => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
     const clearEnvironmentVariables = () => {
-        Object.keys(process.env).forEach((key) => { 
+        Object.keys(process.env).forEach((key) => {
             if(key.indexOf('ZIGBEE2MQTT_CONFIG_') >= 0) {
                 delete process.env[key];
             }
@@ -153,6 +154,7 @@ describe('Settings', () => {
 
         const expected = {
             include_device_information: false,
+            force_disable_retain: false,
             password: "mysecretpassword",
             server: "my.mqtt.server",
             user: "mysecretusername",
@@ -196,6 +198,32 @@ describe('Settings', () => {
         expect(device).toStrictEqual(expected);
     });
 
+    it('Should read devices form 2 separate files', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+        expect(settings.getDevice('0x12345678').friendly_name).toStrictEqual('0x12345678');
+        expect(settings.getDevice('0x87654321').friendly_name).toStrictEqual('0x87654321');
+    });
+
     it('Should add devices to a separate file', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',
@@ -228,6 +256,47 @@ describe('Settings', () => {
         expect(read(devicesFile)).toStrictEqual(expected);
     });
 
+    it('Should add devices for first file when using 2 separates file', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual({devices: ['devices.yaml', 'devices2.yaml']});
+
+        const expected = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expected);
+        expect(read(devicesFile2)).toStrictEqual(contentDevices2);
+    });
+
     it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',

test/stub/data.js

@@ -2,7 +2,7 @@ const tmp = require('tmp');
 const yaml = require('../../lib/util/yaml');
 const path = require('path');
 const fs = require('fs');
-const stringify = require('json-stable-stringify');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const mockDir = tmp.dirSync().name;
 const mockDirStorage = tmp.dirSync().name;
@@ -186,6 +186,11 @@ function writeDefaultConfiguration() {
                 friendly_name: 'thermostat_group',
                 retain: false,
                 devices: ['TS0601_thermostat'],
+            },
+            '14': {
+                friendly_name: 'switch_group',
+                retain: false,
+                devices: ['power_plug'],
             }
         },
         external_converters: [],

test/stub/logger.js

@@ -17,6 +17,7 @@ const mock = {
     error: jest.fn().mockImplementation((msg) => callTransports('error', msg)),
     debug: jest.fn().mockImplementation((msg) => callTransports('debug', msg)),
     cleanup: jest.fn(),
+    logOutput: jest.fn(),
     add: (transport) => transports.push(transport),
     setLevel: (newLevel) => {level = newLevel},
     getLevel: () => level,

test/stub/zigbeeHerdsman.js

@@ -41,7 +41,9 @@ class Endpoint {
         this.write = jest.fn();
         this.bind = jest.fn();
         this.unbind = jest.fn();
+        this.save = jest.fn();
         this.configureReporting = jest.fn();
+        this.meta = {};
         this.binds = binds;
         this.getInputClusters = () => inputClusters.map((c) => {
             return {ID: c, name: getKeyByValue(clusters, c)};
@@ -89,7 +91,7 @@ class Endpoint {
 }
 
 class Device {
-    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null) {
+    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null, softwareBuildID=null) {
         this.type = type;
         this.ieeeAddr = ieeeAddr;
         this.dateCode = dateCode;
@@ -97,6 +99,7 @@ class Device {
         this.manufacturerID = manufacturerID;
         this.endpoints = endpoints;
         this.powerSource = powerSource;
+        this.softwareBuildID = softwareBuildID;
         this.interviewCompleted = interviewCompleted;
         this.modelID = modelID;
         this.interviewing = interviewing;
@@ -117,7 +120,7 @@ class Device {
 const returnDevices = [];
 
 const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
-const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
+const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020", false, 'Philips', '2019.09', '5.127.1.26581');
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 
@@ -128,7 +131,7 @@ const devices = {
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
     'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: new Group(1, []), cluster: {ID: 6, name: 'genOnOff'}}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
-    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID", false, "notSupportedMfg"),
     'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
     'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
     'notInSettings': new Device('EndDevice', '0x0017880104e45519', 6537, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
@@ -136,7 +139,7 @@ const devices = {
     'WXKG02LM': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
     'WSDCGQ11LM': new Device('EndDevice', '0x0017880104e45522', 6539,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.weather"),
     'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
-    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", "lumi.plug"),
+    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug"),
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
     'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
     'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
@@ -172,12 +175,17 @@ const groups = {
     'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
     'group/with/slashes': new Group(99, []),
     'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
-    'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]])
+    'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
+    'group_with_switch': new Group(14, [devices.ZNCZ02LM.endpoints[0]]),
+    'default_bind_group': new Group(901, []),
 }
 
 const mock = {
     setTransmitPower: jest.fn(),
     touchlinkFactoryReset: jest.fn(),
+    touchlinkFactoryResetFirst: jest.fn(),
+    touchlinkScan: jest.fn(),
+    touchlinkIdentify: jest.fn(),
     start: jest.fn(),
     permitJoin: jest.fn(),
     getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
@@ -197,7 +205,7 @@ const mock = {
         return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.ieeeAddr === ieeeAddr);
     }),
     getDeviceByNetworkAddress: jest.fn().mockImplementation((networkAddress) => {
-        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.networkAddress)).find((d) => d.networkAddress === networkAddress);
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.networkAddress === networkAddress);
     }),
     getGroups: jest.fn().mockImplementation((query) => {
         return Object.values(groups);

test/utils.test.js

@@ -1,5 +1,6 @@
 const utils = require('../lib/util/utils.js');
 const version = require('../package.json').version;
+const versionHerdsman = require('../node_modules/zigbee-herdsman/package.json').version;
 
 describe('Utils', () => {
     describe('Is xiaomi device', () => {
@@ -41,6 +42,11 @@ describe('Utils', () => {
         expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
     })
 
+    it('Check dependency version', async () => {
+        var dependency = 'zigbee-herdsman';
+        expect(await utils.getDependencyVersion(dependency)).toStrictEqual({"version": versionHerdsman});
+    })
+    
     it('To local iso string', async () => {
         var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
         var getTimezoneOffset = Date.prototype.getTimezoneOffset;

update.sh

@@ -1,5 +1,12 @@
 #!/bin/bash -e
 
+if [ -d data-backup ]; then
+   echo "ERROR: Backup directory exists. May be previous restoring was failed?"
+   echo "1. Save 'data-backup' and 'data' dirs to safe location to make possibility to restore config later."
+   echo "2. Manually delete 'data-backup' dir and try again."
+   exit 1
+fi
+
 echo "Stopping Zigbee2MQTT..."
 sudo systemctl stop zigbee2mqtt