1.17.0

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,8 @@
 blank_issues_enabled: false
 contact_links:
+  - name: Questions/discussion
+    url: https://github.com/Koenkk/zigbee2mqtt/discussions/new
+    about: Ask questions, discuss about devices or show things you made
   - name: Frontend issues
     url: https://github.com/nurikk/z2m-frontend/issues
     about: Issues related to the frontend.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -8,7 +8,10 @@ assignees: ''
 ---
 
 <!--
-IMPORTANT: Before submitting a request, first follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+VERY IMPORTANT: Before submitting a request, first (try to) follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Dumping just the info about this device without actually attemping adding support yourself will likely lead into this issue being ignored.
+Clearly indicate what you tried and where you got stuck.
 -->
 
 ## Information about the device + link

.github/workflows/ci.yml

@@ -76,4 +76,13 @@ jobs:
           --header "Content-Type: application/json" \
           --data '{"definition": {"id": 1}}' \
           -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 9}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/update_deps.yml

@@ -18,7 +18,7 @@ jobs:
     - run: npm install
     - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
     - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies
         branch: "update_deps"

.github/workflows/update_frontend.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
         branch: "update_frontend"

.github/workflows/update_zh.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
         branch: "update_zh"

.github/workflows/update_zhc.yml

@@ -15,7 +15,7 @@ jobs:
       with:
         node-version: 12
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
         branch: "update_zhc"

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "files.associations": {
-        "*.yaml": "home-assistant"
-    }
-}

README.md

@@ -67,7 +67,7 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 ![Architecture](images/architecture.png)
 
 ### Internal Architecture
-Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
 
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

lib/controller.js

@@ -52,6 +52,7 @@ class Controller {
         // Initialize extensions.
         const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
+            new ExtensionBridge(...args, this.enableDisableExtension),
             new ExtensionPublish(...args),
             new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
@@ -64,10 +65,6 @@ class Controller {
             new ExtensionReport(...args),
         ];
 
-        if (settings.get().experimental.new_api) {
-            this.extensions.push(new ExtensionBridge(...args, this.enableDisableExtension));
-        }
-
         if (settings.get().frontend) {
             this.extensions.push(new ExtensionFrontend(...args));
         }
@@ -149,7 +146,6 @@ class Controller {
         // MQTT
         this.mqtt.on('message', this.onMQTTMessage.bind(this));
         await this.mqtt.connect();
-        this.mqtt.publish('bridge/state', 'online', {retain: true, qos: 0});
 
         // Send all cached states.
         if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
@@ -212,7 +208,12 @@ class Controller {
 
     async onZigbeeEvent(type, data) {
         const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !settings.getDevice(data.device.ieeeAddr) && data.device.type !== 'Coordinator') {
+        if (data.device && data.device.type === 'Coordinator') {
+            logger.debug('Ignoring message from coordinator');
+            return;
+        }
+
+        if (data.device && !settings.getDevice(data.device.ieeeAddr)) {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
             resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
         }
@@ -236,7 +237,8 @@ class Controller {
                     logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
-                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' and manufacturer name ` +
+                        `'${data.device.manufacturerName}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
                 }
@@ -254,6 +256,7 @@ class Controller {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
                 logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                this.state.remove(data.ieeeAddr);
             }
         }
 

lib/extension/availability.js

@@ -3,6 +3,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const Extension = require('./extension');
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/(.*)/availability`);
 
 // Pingable end devices, some end devices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
@@ -25,6 +26,7 @@ class Availability extends Extension {
         this.state = {};
 
         this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+        this.eventBus.on('deviceRenamed', (data) => this.onDeviceRenamed(data), this.constructor.name);
 
         this.blocklist = settings.get().advanced.availability_blocklist
             .concat(settings.get().advanced.availability_blacklist)
@@ -35,6 +37,10 @@ class Availability extends Extension {
             .map((e) => settings.getEntity(e).ID);
     }
 
+    onDeviceRenamed(data) {
+        this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
+    }
+
     onDeviceRemoved(resolvedEntity) {
         this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
         delete this.state[resolvedEntity.device.ieeeAddr];
@@ -86,6 +92,14 @@ class Availability extends Extension {
         }
     }
 
+    async onMQTTMessage(topic, message) {
+        // Clear topics for non-existing devices
+        const match = topic.match(topicRegex);
+        if (match && (!this.zigbee.resolveEntity(match[1]) || this.zigbee.resolveEntity(match[1]).name !== match[1])) {
+            this.mqtt.publish(`${match[1]}/availability`, null, {retain: true, qos: 0});
+        }
+    }
+
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
         const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);

lib/extension/bind.js

@@ -17,20 +17,6 @@ class Bind extends Extension {
         this.legacyApi = settings.get().advanced.legacy_api;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
-        }
-    }
-
     parseMQTTMessage(topic, message) {
         let type = null;
         let sourceKey = null;
@@ -42,7 +28,7 @@ class Bind extends Extension {
             type = topic.split('/')[0];
             sourceKey = topic.replace(`${type}/`, '');
             targetKey = message;
-        } else if (settings.get().experimental.new_api && topic.match(topicRegex)) {
+        } else if (topic.match(topicRegex)) {
             type = topic.endsWith('unbind') ? 'unbind' : 'bind';
             message = JSON.parse(message);
             sourceKey = message.from;

lib/extension/bridge.js

@@ -18,6 +18,7 @@ class Bridge extends Extension {
 
         this.requestLookup = {
             'device/options': this.deviceOptions.bind(this),
+            'device/configure_reporting': this.deviceConfigureReporting.bind(this),
             'device/remove': this.deviceRemove.bind(this),
             'device/rename': this.deviceRename.bind(this),
             'group/add': this.groupAdd.bind(this),
@@ -42,7 +43,6 @@ class Bridge extends Extension {
 
         this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
         this.eventBus.on(`devicesChanged`, () => this.publishDevices());
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
@@ -58,7 +58,7 @@ class Bridge extends Extension {
             }
         }
 
-        logger.add(new EventTransport());
+        logger.add(new EventTransport({level: logger.getLevel()}));
     }
 
     async onMQTTMessage(topic, message) {
@@ -311,6 +311,31 @@ class Bridge extends Extension {
         return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
     }
 
+    async deviceConfigureReporting(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('cluster') ||
+            !message.hasOwnProperty('maximum_report_interval') || !message.hasOwnProperty('minimum_report_interval') ||
+            !message.hasOwnProperty('reportable_change') || !message.hasOwnProperty('attribute')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const ID = message.id;
+        const endpoint = this.getEntity('device', ID).endpoint;
+        await endpoint.configureReporting(message.cluster, [{
+            attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
+            maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
+        }]);
+
+        this.publishDevices();
+
+        logger.info(`Configured reporting for '${message.id}', '${message.cluster}.${message.attribute}'`);
+
+        return utils.getResponse(message, {
+            id: message.id, cluster: message.cluster, maximum_report_interval: message.maximum_report_interval,
+            minimum_report_interval: message.minimum_report_interval, reportable_change: message.reportable_change,
+            attribute: message.attribute,
+        }, null);
+    }
+
     renameEntity(entityType, message) {
         const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
         if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
@@ -338,12 +363,13 @@ class Bridge extends Extension {
         // Clear retained messages
         this.mqtt.publish(entity.name, '', {retain: true});
 
+        const oldFriendlyName = entity.settings.friendlyName;
         if (entity.type === 'device') {
             this.publishDevices();
-            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename});
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename, from: oldFriendlyName, to});
         } else {
             this.publishGroups();
-            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename});
+            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename, from: oldFriendlyName, to});
         }
 
         // Repulish entity state
@@ -351,7 +377,7 @@ class Bridge extends Extension {
 
         return utils.getResponse(
             message,
-            {from: entity.settings.friendlyName, to, homeassistant_rename: homeAssisantRename},
+            {from: oldFriendlyName, to, homeassistant_rename: homeAssisantRename},
             null,
         );
     }
@@ -440,6 +466,7 @@ class Bridge extends Extension {
         const config = objectAssignDeep.noMutate({}, settings.get());
         delete config.advanced.network_key;
         delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
         const payload = {
             version: this.zigbee2mqttVersion.version,
             commit: this.zigbee2mqttVersion.commitHash,
@@ -461,6 +488,7 @@ class Bridge extends Extension {
             for (const endpoint of device.endpoints) {
                 const data = {
                     bindings: [],
+                    configured_reportings: [],
                     clusters: {
                         input: endpoint.getInputClusters().map((c) => c.name),
                         output: endpoint.getOutputClusters().map((c) => c.name),
@@ -481,6 +509,16 @@ class Bridge extends Extension {
                     data.bindings.push({cluster: bind.cluster.name, target});
                 }
 
+                for (const configuredReporting of endpoint.configuredReportings) {
+                    data.configured_reportings.push({
+                        cluster: configuredReporting.cluster.name,
+                        attribute: configuredReporting.attribute.name,
+                        minimum_report_interval: configuredReporting.minimumReportInterval,
+                        maximum_report_interval: configuredReporting.maximumReportInterval,
+                        reportable_change: configuredReporting.reportableChange,
+                    });
+                }
+
                 endpoints[endpoint.ID] = data;
             }
 
@@ -494,6 +532,7 @@ class Bridge extends Extension {
                 power_source: device.powerSource,
                 software_build_id: device.softwareBuildID,
                 date_code: device.dateCode,
+                model_id: device.modelID,
                 interviewing: device.interviewing,
                 interview_completed: device.interviewCompleted,
                 endpoints,
@@ -508,7 +547,7 @@ class Bridge extends Extension {
             const resolved = this.zigbee.resolveEntity(group);
             return {
                 id: group.groupID,
-                friendly_name: resolved.name,
+                friendly_name: group.groupID === 901 ? 'default_bind_group' : resolved.name,
                 members: group.members.map((m) => {
                     return {
                         ieee_address: m.deviceIeeeAddress,
@@ -527,8 +566,7 @@ class Bridge extends Extension {
                 model: definition.model,
                 vendor: definition.vendor,
                 description: definition.description,
-                [definition.hasOwnProperty('exposes') ? 'exposes' : 'supports']:
-                    definition.hasOwnProperty('exposes') ? definition.exposes : definition.supports,
+                exposes: definition.exposes,
             };
         } else {
             return null;

lib/extension/configure.js

@@ -53,18 +53,6 @@ class Configure extends Extension {
         return true;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (settings.get().advanced.legacy_api) {
-            this.mqtt.subscribe(this.legacyTopic);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         if (topic === this.legacyTopic) {
             const resolvedEntity = this.zigbee.resolveEntity(message);
@@ -140,10 +128,11 @@ class Configure extends Extension {
 
         logger.info(`Configuring '${resolvedEntity.name}'`);
         try {
-            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint, logger);
             logger.info(`Successfully configured '${resolvedEntity.name}'`);
             device.meta.configured = resolvedEntity.definition.meta.configureKey;
             device.save();
+            this.eventBus.emit(`devicesChanged`);
         } catch (error) {
             this.attempts[device.ieeeAddr]++;
             const attempt = this.attempts[device.ieeeAddr];

lib/extension/frontend.js

@@ -1,5 +1,4 @@
 const http = require('http');
-const httpProxy = require('http-proxy');
 const serveStatic = require('serve-static');
 const finalhandler = require('finalhandler');
 const Extension = require('./extension');
@@ -25,34 +24,24 @@ class Frontend extends Extension {
         this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
         this.server = http.createServer(this.onRequest);
         this.server.on('upgrade', this.onUpgrade);
-        this.developmentServer = settings.get().frontend.development_server;
-        this.development = !!this.developmentServer;
+        this.host = settings.get().frontend.host || '0.0.0.0';
         this.port = settings.get().frontend.port || 8080;
+        this.authToken = settings.get().frontend.auth_token || false;
         this.retainedMessages = new Map();
-
-        if (this.development) {
-            this.proxy = httpProxy.createProxyServer({ws: true});
-        } else {
-            /* istanbul ignore next */
-            const options = {setHeaders: (res, path) => {
-                if (path.endsWith('index.html')) {
-                    res.setHeader('Cache-Control', 'no-store');
-                }
-            }};
-            this.fileServer = serveStatic(frontend.getPath(), options);
-        }
-
+        /* istanbul ignore next */
+        const options = {setHeaders: (res, path) => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+        this.fileServer = serveStatic(frontend.getPath(), options);
         this.wss = new WebSocket.Server({noServer: true});
         this.wss.on('connection', this.onWebSocketConnection);
     }
 
     onZigbeeStarted() {
-        if (this.development) {
-            logger.info(`Running frontend in development mode (${this.developmentServer})`);
-        }
-
-        this.server.listen(this.port);
-        logger.info(`Started frontend on port ${this.port}`);
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
     }
 
     async stop() {
@@ -64,25 +53,24 @@ class Frontend extends Extension {
             this.server.close(resolve);
         });
     }
-
     onRequest(request, response) {
-        if (this.development) {
-            this.proxy.web(request, response, {target: `http://${this.developmentServer}`});
-        } else {
-            this.fileServer(request, response, finalhandler(request, response));
-        }
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+    authenticate(request, cb) {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
     }
 
     onUpgrade(request, socket, head) {
-        const pathname = url.parse(request.url).pathname;
-        if (pathname === '/api') {
-            const wss = this.wss;
-            wss.handleUpgrade(request, socket, head, (ws) => wss.emit('connection', ws, request));
-        } else if (this.development && pathname === '/sockjs-node') {
-            this.proxy.ws(request, socket, head, {target: `ws://${this.developmentServer}`});
-        } else {
-            socket.destroy();
-        }
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
     }
 
     onWebSocketConnection(ws) {
@@ -109,6 +97,10 @@ class Frontend extends Extension {
                 payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
             }
 
+            if (resolvedEntity.device.linkquality !== undefined) {
+                payload.linkquality = resolvedEntity.device.linkquality;
+            }
+
             ws.send(stringify({topic: resolvedEntity.name, payload}));
         }
     }

lib/extension/groups.js

@@ -19,27 +19,6 @@ class Groups extends Extension {
         this.lastOptimisticState = {};
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-
-            for (let step = 1; step < 20; step++) {
-                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-                this.mqtt.subscribe(`${topic}remove`);
-                this.mqtt.subscribe(`${topic}add`);
-                this.mqtt.subscribe(`${topic}remove_all`);
-            }
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
-        }
-    }
-
     async onZigbeeStarted() {
         this.eventBus.on('stateChange', this.onStateChange);
         await this.syncGroupsWithSettings();

lib/extension/homeassistant.js

@@ -7,368 +7,13 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const assert = require('assert');
 
-const cfg = {
-    // Binary sensor
-    'binary_sensor_water_leak': {
-        type: 'binary_sensor',
-        object_id: 'water_leak',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.water_leak }}',
-            device_class: 'moisture',
-        },
+const sensorClick = {
+    type: 'sensor',
+    object_id: 'click',
+    discovery_payload: {
+        icon: 'mdi:toggle-switch',
+        value_template: '{{ value_json.click }}',
     },
-    'binary_sensor_battery_low': {
-        type: 'binary_sensor',
-        object_id: 'battery_low',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.battery_low}}',
-            device_class: 'battery',
-        },
-    },
-    'binary_sensor_update_available': {
-        type: 'binary_sensor',
-        object_id: 'update_available',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.update_available}}',
-        },
-    },
-
-    // Sensor
-    'sensor_update_state': {
-        type: 'sensor',
-        object_id: 'update_state',
-        discovery_payload: {
-            icon: 'mdi:update',
-            value_template: `{{ value_json['update']['state'] }}`,
-        },
-    },
-    'sensor_local_temperature': {
-        type: 'sensor',
-        object_id: 'local_temperature',
-        discovery_payload: {
-            unit_of_measurement: '°C',
-            device_class: 'temperature',
-            value_template: '{{ value_json.local_temperature }}',
-        },
-    },
-    'sensor_click': {
-        type: 'sensor',
-        object_id: 'click',
-        discovery_payload: {
-            icon: 'mdi:toggle-switch',
-            value_template: '{{ value_json.click }}',
-        },
-    },
-    'sensor_power': {
-        type: 'sensor',
-        object_id: 'power',
-        discovery_payload: {
-            unit_of_measurement: 'W',
-            icon: 'mdi:flash',
-            value_template: '{{ value_json.power }}',
-        },
-    },
-    'sensor_energy': {
-        type: 'sensor',
-        object_id: 'energy',
-        discovery_payload: {
-            unit_of_measurement: 'kWh',
-            icon: 'mdi:power-plug',
-            value_template: '{{ value_json.energy }}',
-        },
-    },
-    'sensor_brightness': {
-        type: 'sensor',
-        object_id: 'brightness',
-        discovery_payload: {
-            unit_of_measurement: 'brightness',
-            icon: 'mdi:brightness-5',
-            value_template: '{{ value_json.brightness }}',
-        },
-    },
-    'sensor_battery': {
-        type: 'sensor',
-        object_id: 'battery',
-        discovery_payload: {
-            unit_of_measurement: '%',
-            device_class: 'battery',
-            value_template: '{{ value_json.battery }}',
-        },
-    },
-    'sensor_linkquality': {
-        type: 'sensor',
-        object_id: 'linkquality',
-        discovery_payload: {
-            icon: 'mdi:signal',
-            unit_of_measurement: 'lqi',
-            value_template: '{{ value_json.linkquality }}',
-        },
-    },
-
-    // Switch
-    'switch_window_detection': {
-        type: 'switch',
-        object_id: 'window_detection',
-        discovery_payload: {
-            state_topic: true,
-            command_topic: true,
-            command_topic_postfix: 'window_detection',
-            payload_off: 'OFF',
-            payload_on: 'ON',
-            state_off: 'OFF',
-            state_on: 'ON',
-            value_template: '{{ value_json.window_detection }}',
-            icon: 'mdi:window-open-variant',
-        },
-    },
-    'switch_valve_detection': {
-        type: 'switch',
-        object_id: 'valve_detection',
-        discovery_payload: {
-            state_topic: true,
-            command_topic: true,
-            command_topic_postfix: 'valve_detection',
-            payload_off: 'OFF',
-            payload_on: 'ON',
-            state_off: 'OFF',
-            state_on: 'ON',
-            value_template: '{{ value_json.valve_detection }}',
-        },
-    },
-
-    // Lock
-    'lock_keypad_lockout': {
-        type: 'lock',
-        object_id: 'keypad_lock',
-        discovery_payload: {
-            state_topic: true,
-            command_topic: true,
-            command_topic_postfix: 'keypad_lockout',
-            payload_unlock: '0',
-            payload_lock: '1',
-            value_template: '{{ value_json.keypad_lockout }}',
-        },
-    },
-    'lock_child_lock': {
-        type: 'lock',
-        object_id: 'child_lock',
-        discovery_payload: {
-            state_topic: true,
-            command_topic: true,
-            command_topic_postfix: 'child_lock',
-            payload_lock: 'LOCK',
-            payload_unlock: 'UNLOCK',
-            state_locked: 'LOCKED',
-            state_unlocked: 'UNLOCKED',
-            value_template: '{{ value_json.child_lock }}',
-        },
-    },
-
-    // Trigger
-    'trigger_action': {
-        type: 'device_automation',
-        discovery_payload: {
-            automation_type: 'trigger',
-            type: 'action',
-        },
-    },
-    'trigger_click': {
-        type: 'device_automation',
-        discovery_payload: {
-            automation_type: 'trigger',
-            type: 'click',
-        },
-    },
-};
-
-const climate = (minTemp=7, maxTemp=30, temperatureStateProperty='occupied_heating_setpoint',
-    tempStep=1, systemModes=['off', 'auto', 'heat'], fanModes=[], holdModes=[],
-    temperatureLowStateTopic=false, temperatureHighStateTopic=false, endpoint=null ) => {
-    const jsonProperty = (key) => `value_json.${key}${endpoint ? `_${endpoint}` : ''}`;
-    const retVal = {
-        type: 'climate',
-        object_id: endpoint ? `climate_${endpoint}` : 'climate',
-        discovery_payload: {
-            state_topic: false,
-            temperature_unit: 'C',
-            min_temp: `${minTemp}`,
-            max_temp: `${maxTemp}`,
-            mode_state_topic: true,
-            mode_state_template: `{{ ${jsonProperty('system_mode')} }}`,
-            mode_command_topic: true,
-            current_temperature_topic: true,
-            current_temperature_template: `{{ ${jsonProperty('local_temperature')} }}`,
-            temp_step: tempStep,
-            action_topic: true,
-            action_template:
-                '{% set values = {\'idle\':\'off\',\'heat\':\'heating\',\'cool\':\'cooling\',\'fan only\':\'fan\'}'+
-                ` %}{{ values[${jsonProperty('running_state')}] }}`,
-        },
-    };
-
-    if (endpoint) {
-        retVal.discovery_payload.state_topic_postfix = endpoint;
-    }
-
-    // system_modes empty <=> use auto (in other case ha ui is showing all modes)
-    if (systemModes.length > 0) {
-        retVal.discovery_payload.modes = systemModes;
-    } else {
-        retVal.discovery_payload.modes = ['auto'];
-    }
-    // hold_modes empty <=> don't use presets
-    if (holdModes.length > 0) {
-        // NOTE: Preset 'none' will be added as first item on HA side `mqtt/climate.py preset_modes()`
-        const indexOfNone = holdModes.indexOf('none');
-        if (indexOfNone > -1) holdModes.splice(indexOfNone, 1);
-
-        // HA has special behaviour for the away mode
-        // https://github.com/Koenkk/zigbee2mqtt/pull/4491#issuecomment-701550476
-        const indexOfAway = holdModes.indexOf('away');
-        /* istanbul ignore else */
-        if (indexOfAway > -1) {
-            holdModes.splice(indexOfAway, 1); // HA will add "Away" to modes by itself
-            retVal.discovery_payload.away_mode_command_topic = true;
-            retVal.discovery_payload.away_mode_state_topic = true;
-            retVal.discovery_payload.away_mode_state_template =
-                `{{ ${jsonProperty('away_mode')} }}`;
-        }
-
-        if (holdModes.length > 0) { // || indexOfAway > -1) {
-            retVal.discovery_payload.hold_modes = holdModes;
-            retVal.discovery_payload.hold_command_topic = true;
-            retVal.discovery_payload.hold_state_template = `{{ ${jsonProperty('preset')} }}`;
-            retVal.discovery_payload.hold_state_topic = true;
-        }
-    }
-    // fan_modes empty <=> don't use fan modes
-    if (fanModes.length > 0) {
-        retVal.discovery_payload.fan_modes = fanModes;
-        retVal.discovery_payload.fan_mode_command_topic = true;
-        retVal.discovery_payload.fan_mode_state_template = `{{ ${jsonProperty('fan_mode')} }}`;
-        retVal.discovery_payload.fan_mode_state_topic = true;
-    }
-    // if no high and low temp used then use temperature_state_topic
-    if (!temperatureHighStateTopic && !temperatureLowStateTopic) {
-        retVal.discovery_payload.temperature_state_topic = true;
-        retVal.discovery_payload.temperature_state_template = `{{ ${jsonProperty(temperatureStateProperty)} }}`;
-        retVal.discovery_payload.temperature_command_topic = temperatureStateProperty;
-    }
-    // use low target temperature
-    if (temperatureLowStateTopic) {
-        retVal.discovery_payload.temperature_low_state_topic = temperatureLowStateTopic;
-        retVal.discovery_payload.temperature_low_state_template = `{{ ${jsonProperty('occupied_heating_setpoint')} }}`;
-        retVal.discovery_payload.temperature_low_command_topic = 'occupied_heating_setpoint';
-    }
-    // use high target temperature
-    if (temperatureHighStateTopic) {
-        retVal.discovery_payload.temperature_high_state_topic = temperatureHighStateTopic;
-        retVal.discovery_payload.temperature_high_state_template = `{{ ${jsonProperty('occupied_cooling_setpoint')} }}`;
-        retVal.discovery_payload.temperature_high_command_topic = 'occupied_cooling_setpoint';
-    }
-    return retVal;
-};
-
-
-// Map Home Assistant configurations to devices.
-const manualMaping = {
-    '1TST-EU': [climate(), cfg.sensor_battery],
-    'AV2010/32': [climate(7, 30, 'occupied_heating_setpoint', 0.5), cfg.sensor_battery],
-    'SPZB0001': [climate(5, 30, 'occupied_heating_setpoint', 0.5), cfg.sensor_battery],
-    'ST218': [
-        climate(5, 30, 'occupied_heating_setpoint', 0.5),
-        cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout,
-    ],
-    'TH1123ZB': [
-        climate(7, 30, 'occupied_heating_setpoint', 0.5), cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout, cfg.sensor_power, cfg.sensor_energy,
-    ],
-    'TH1124ZB': [
-        climate(7, 30, 'occupied_heating_setpoint', 0.5), cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout, cfg.sensor_power, cfg.sensor_energy,
-    ],
-    'TH1300ZB': [
-        climate(7, 30, 'occupied_heating_setpoint', 0.5), cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout,
-    ],
-    'TH1400ZB': [climate()],
-    'TH1500ZB': [climate()],
-    'Zen-01-W': [climate(10, 30, 'occupied_heating_setpoint', 0.5)],
-    'UK7004240': [climate(), cfg.sensor_battery],
-    'WV704R0A0902': [climate()],
-    'STZB402': [
-        climate(5, 30, 'occupied_heating_setpoint', 0.5),
-        cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout,
-    ],
-    'SMT402': [
-        climate(5, 30, 'occupied_heating_setpoint', 0.5),
-        cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout,
-    ],
-    'SMT402AD': [
-        climate(5, 30, 'occupied_heating_setpoint', 0.5),
-        cfg.sensor_local_temperature,
-        cfg.lock_keypad_lockout,
-    ],
-    'SLR1b': [climate()],
-    'GS361A-H04': [
-        cfg.lock_child_lock,
-        cfg.switch_window_detection,
-        cfg.switch_valve_detection,
-        climate(5, 30, 'current_heating_setpoint', 0.5, ['off', 'auto', 'heat', 'manual']),
-        cfg.sensor_battery,
-    ],
-    '3157100': [climate(10, 30, 'occupied_heating_setpoint', 1, ['off', 'heat', 'cool'],
-        ['auto', 'on'], [], true, true), cfg.sensor_battery],
-    'RC-2000WH': [climate(10, 30, 'occupied_heating_setpoint', 1, ['off', 'auto', 'heat', 'cool'],
-        ['auto', 'on', 'smart'], [], true, true)],
-    'TS0601_thermostat': [
-        cfg.lock_child_lock, cfg.switch_window_detection, cfg.switch_valve_detection, cfg.sensor_battery,
-        climate(5, 30, 'current_heating_setpoint', 0.5, [], [],
-            ['schedule', 'manual', 'away', 'boost', 'complex', 'comfort', 'eco']),
-    ],
-    'HT-08': [
-        cfg.lock_child_lock,
-        climate(5, 35, 'current_heating_setpoint', 0.5,
-            ['off', 'heat', 'auto'], [], ['none', 'away']),
-    ],
-    'HT-10': [
-        cfg.lock_child_lock, cfg.binary_sensor_battery_low,
-        climate(5, 35, 'current_heating_setpoint', 0.5,
-            ['off', 'heat', 'auto'], [], ['none', 'away']),
-    ],
-    '07703L': [
-        cfg.lock_child_lock, cfg.binary_sensor_battery_low,
-        climate(5, 35, 'current_heating_setpoint', 0.5,
-            ['off', 'heat', 'auto'], [], ['none', 'away']),
-    ],
-    'ZK03840': [climate()],
-    'U86KWF-ZPSJ': [climate(5, 30, 'current_heating_setpoint', 0.5)],
-    'D3-DPWK-TY': [climate(5, 30, 'current_heating_setpoint', 0.5)],
-    'BHT-002-GCLZB': [
-        cfg.lock_child_lock, climate(5, 30, 'current_heating_setpoint', 1, ['off', 'heat'], [], ['hold', 'program']),
-    ],
-    'SLR2': [
-        climate(7, 30, 'occupied_heating_setpoint', 1, ['off', 'auto', 'heat'], [], [], false, false, 'heat'),
-        climate(7, 30, 'occupied_heating_setpoint', 1, ['off', 'auto', 'heat'], [], [], false, false, 'cool'),
-    ],
-    'SEA801-Zigbee': [
-        cfg.binary_sensor_battery_low,
-        climate(5, 30, 'current_heating_setpoint', 0.5, ['off', 'heat'], [], ['manual', 'auto']),
-    ],
-    'SEA802-Zigbee': [
-        cfg.binary_sensor_battery_low,
-        climate(5, 30, 'current_heating_setpoint', 0.5, ['off', 'heat'], [], ['manual', 'auto']),
-    ],
-    'HY08WE': [climate(5, 30, 'current_heating_setpoint', 0.5)],
 };
 
 const defaultStatusTopic = 'homeassistant/status';
@@ -385,7 +30,6 @@ class HomeAssistant extends Extension {
         this.mapping = {};
         this.discoveredTriggers = {};
         this.legacyApi = settings.get().advanced.legacy_api;
-        this.newApi = settings.get().experimental.new_api;
 
         if (!settings.get().advanced.cache_state) {
             logger.warn('In order for HomeAssistant integration to work properly set `cache_state: true');
@@ -410,20 +54,28 @@ class HomeAssistant extends Extension {
     populateMapping() {
         for (const def of zigbeeHerdsmanConverters.definitions) {
             if (def.hasOwnProperty('exposes')) {
-                assert(!manualMaping.hasOwnProperty(def.model), `'${def.model}' has manual mapping and exposes`);
                 this.mapping[def.model] = [];
 
                 if (['WXKG01LM', 'HS1EB/HS1EB-E', 'ICZB-KPD14S', 'TERNCY-SD01', 'TERNCY-PP01', 'ICZB-KPD18S',
                     'E1766', 'ZWallRemote0', 'ptvo.switch', '2AJZ4KPKEY', 'ZGRC-KEY-013', 'HGZB-02S', 'HGZB-045',
-                    'HGZB-1S', 'AV2010/34', 'IM6001-BTP01', 'WXKG11LM', 'WXKG03LM', 'WXKG02LM', 'QBKG04LM', 'QBKG03LM',
-                    'QBKG11LM', 'QBKG21LM', 'QBKG22LM', 'WXKG12LM', 'QBKG12LM', 'E1743'].includes(def.model)) {
+                    'HGZB-1S', 'AV2010/34', 'IM6001-BTP01', 'WXKG11LM', 'WXKG03LM', 'WXKG02LM_rev1', 'WXKG02LM_rev2',
+                    'QBKG04LM', 'QBKG03LM', 'QBKG11LM', 'QBKG21LM', 'QBKG22LM', 'WXKG12LM', 'QBKG12LM',
+                    'E1743'].includes(def.model)) {
                     // deprecated
-                    this.mapping[def.model].push(cfg.sensor_click);
+                    this.mapping[def.model].push(sensorClick);
                 }
 
                 if (['ICTC-G-1'].includes(def.model)) {
                     // deprecated
-                    this.mapping[def.model].push(cfg.sensor_brightness);
+                    this.mapping[def.model].push({
+                        type: 'sensor',
+                        object_id: 'brightness',
+                        discovery_payload: {
+                            unit_of_measurement: 'brightness',
+                            icon: 'mdi:brightness-5',
+                            value_template: '{{ value_json.brightness }}',
+                        },
+                    });
                 }
 
                 for (const expose of def.exposes) {
@@ -452,29 +104,151 @@ class HomeAssistant extends Extension {
                             discoveryEntry.discovery_payload.effect_list = effect.values;
                         }
                     } else if (expose.type === 'switch') {
+                        const state = expose.features.find((f) => f.name === 'state');
                         discoveryEntry = {
                             type: 'switch',
                             object_id: expose.endpoint ? `switch_${expose.endpoint}` : 'switch',
                             discovery_payload: {
                                 payload_off: 'OFF',
                                 payload_on: 'ON',
-                                value_template: `{{ value_json.state${expose.endpoint ? `_${expose.endpoint}` : ''} }}`,
+                                value_template: `{{ value_json.${state.property} }}`,
                                 command_topic: true,
                                 command_topic_prefix: expose.endpoint ? expose.endpoint : undefined,
                             },
                         };
+
+                        if (state.property === 'valve_detection' || state.property === 'window_detection') {
+                            discoveryEntry.discovery_payload.command_topic_postfix = state.property;
+                            discoveryEntry.discovery_payload.state_off = 'OFF';
+                            discoveryEntry.discovery_payload.state_on = 'ON';
+                            discoveryEntry.discovery_payload.state_topic = true;
+                            discoveryEntry.object_id = state.property;
+
+                            if (state.property === 'window_detection') {
+                                discoveryEntry.discovery_payload.icon = 'mdi:window-open-variant';
+                            }
+                        }
+                    } else if (expose.type === 'climate') {
+                        const setpointProperties = ['occupied_heating_setpoint', 'current_heating_setpoint'];
+                        const setpoint = expose.features.find((f) => setpointProperties.includes(f.name));
+                        assert(setpoint, 'No setpoint found');
+                        const temperature = expose.features.find((f) => f.name === 'local_temperature');
+                        assert(temperature, 'No temperature found');
+
+                        discoveryEntry = {
+                            type: 'climate',
+                            object_id: expose.endpoint ? `climate_${expose.endpoint}` : 'climate',
+                            discovery_payload: {
+                                // Static
+                                state_topic: false,
+                                temperature_unit: 'C',
+                                // Setpoint
+                                temp_step: setpoint.value_step,
+                                min_temp: setpoint.value_min.toString(),
+                                max_temp: setpoint.value_max.toString(),
+                                // Temperature
+                                current_temperature_topic: true,
+                                current_temperature_template: `{{ value_json.${temperature.property} }}`,
+                            },
+                        };
+
+                        const mode = expose.features.find((f) => f.name === 'system_mode');
+                        if (mode) {
+                            discoveryEntry.discovery_payload.mode_state_topic = true;
+                            discoveryEntry.discovery_payload.mode_state_template = `{{ value_json.${mode.property} }}`;
+                            discoveryEntry.discovery_payload.modes = mode.values;
+                            discoveryEntry.discovery_payload.mode_command_topic = true;
+                        }
+
+                        const state = expose.features.find((f) => f.name === 'running_state');
+                        if (state) {
+                            discoveryEntry.discovery_payload.action_topic = true;
+                            discoveryEntry.discovery_payload.action_template = `{% set values = ` +
+                                    `{'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'}` +
+                                    ` %}{{ values[value_json.${state.property}] }}`;
+                        }
+
+                        const coolingSetpoint = expose.features.find((f) => f.name === 'occupied_cooling_setpoint');
+                        if (coolingSetpoint) {
+                            discoveryEntry.discovery_payload.temperature_low_command_topic = setpoint.name;
+                            discoveryEntry.discovery_payload.temperature_low_state_template =
+                                `{{ value_json.${setpoint.property} }}`;
+                            discoveryEntry.discovery_payload.temperature_low_state_topic = true;
+                            discoveryEntry.discovery_payload.temperature_high_command_topic = coolingSetpoint.name;
+                            discoveryEntry.discovery_payload.temperature_high_state_template =
+                                `{{ value_json.${coolingSetpoint.property} }}`;
+                            discoveryEntry.discovery_payload.temperature_high_state_topic = true;
+                        } else {
+                            discoveryEntry.discovery_payload.temperature_command_topic = setpoint.name;
+                            discoveryEntry.discovery_payload.temperature_state_template =
+                                `{{ value_json.${setpoint.property} }}`;
+                            discoveryEntry.discovery_payload.temperature_state_topic = true;
+                        }
+
+                        const fanMode = expose.features.find((f) => f.name === 'fan_mode');
+                        if (fanMode) {
+                            discoveryEntry.discovery_payload.fan_modes = fanMode.values;
+                            discoveryEntry.discovery_payload.fan_mode_command_topic = true;
+                            discoveryEntry.discovery_payload.fan_mode_state_template =
+                                `{{ value_json.${fanMode.property} }}`;
+                            discoveryEntry.discovery_payload.fan_mode_state_topic = true;
+                        }
+
+                        const preset = expose.features.find((f) => f.name === 'preset');
+                        if (preset) {
+                            discoveryEntry.discovery_payload.hold_modes = preset.values;
+                            discoveryEntry.discovery_payload.hold_command_topic = true;
+                            discoveryEntry.discovery_payload.hold_state_template =
+                                `{{ value_json.${preset.property} }}`;
+                            discoveryEntry.discovery_payload.hold_state_topic = true;
+                        }
+
+                        const awayMode = expose.features.find((f) => f.name === 'away_mode');
+                        if (awayMode) {
+                            discoveryEntry.discovery_payload.away_mode_command_topic = true;
+                            discoveryEntry.discovery_payload.away_mode_state_topic = true;
+                            discoveryEntry.discovery_payload.away_mode_state_template =
+                                `{{ value_json.${awayMode.property} }}`;
+                        }
+
+                        if (expose.endpoint) {
+                            discoveryEntry.discovery_payload.state_topic_postfix = expose.endpoint;
+                        }
                     } else if (expose.type === 'lock') {
                         assert(!expose.endpoint, `Endpoint not supported for lock type`);
+                        const state = expose.features.find((f) => f.name === 'state');
+                        assert(state, 'No state found');
                         discoveryEntry = {
                             type: 'lock',
                             object_id: 'lock',
                             discovery_payload: {
                                 command_topic: true,
-                                value_template: '{{ value_json.state }}',
-                                state_locked: 'LOCK',
-                                state_unlocked: 'UNLOCK',
+                                value_template: `{{ value_json.${state.property} }}`,
                             },
                         };
+
+                        if (state.property === 'keypad_lockout') {
+                            // deprecated: keypad_lockout is messy, but changing is breaking
+                            discoveryEntry.discovery_payload.payload_lock = state.value_on;
+                            discoveryEntry.discovery_payload.payload_unlock = state.value_off;
+                            discoveryEntry.discovery_payload.state_topic = true;
+                            discoveryEntry.object_id = 'keypad_lock';
+                        } else if (state.property === 'child_lock') {
+                            // deprecated: child_lock is messy, but changing is breaking
+                            discoveryEntry.discovery_payload.payload_lock = state.value_on;
+                            discoveryEntry.discovery_payload.payload_unlock = state.value_off;
+                            discoveryEntry.discovery_payload.state_locked = 'LOCKED';
+                            discoveryEntry.discovery_payload.state_unlocked = 'UNLOCKED';
+                            discoveryEntry.discovery_payload.state_topic = true;
+                            discoveryEntry.object_id = 'child_lock';
+                        } else {
+                            discoveryEntry.discovery_payload.state_locked = state.value_on;
+                            discoveryEntry.discovery_payload.state_unlocked = state.value_off;
+                        }
+
+                        if (state.property !== 'state') {
+                            discoveryEntry.discovery_payload.command_topic_postfix = state.property;
+                        }
                     } else if (expose.type === 'cover') {
                         assert(!expose.endpoint, `Endpoint not supported for cover type`);
                         const hasPosition = expose.features.find((e) => e.name === 'position');
@@ -539,6 +313,7 @@ class HomeAssistant extends Extension {
                             smoke: {device_class: 'smoke'},
                             gas: {device_class: 'gas'},
                             carbon_monoxide: {device_class: 'safety'},
+                            presence: {device_class: 'presence'},
                         };
 
                         assert(!expose.endpoint, `Endpoint not supported for binary type`);
@@ -561,6 +336,7 @@ class HomeAssistant extends Extension {
                             illuminance_lux: {device_class: 'illuminance'},
                             illuminance: {device_class: 'illuminance'},
                             soil_moisture: {icon: 'mdi:water-percent'},
+                            position: {icon: 'mdi:valve'},
                             pressure: {device_class: 'pressure'},
                             power: {icon: 'mdi:flash'},
                             linkquality: {icon: 'mdi:signal'},
@@ -582,6 +358,7 @@ class HomeAssistant extends Extension {
                             requested_brightness_percent: {icon: 'mdi:brightness-5'},
                             eco2: {icon: 'mdi:molecule-co2'},
                             co2: {icon: 'mdi:molecule-co2'},
+                            local_temperature: {device_class: 'temperature'},
                         };
 
                         assert(!expose.endpoint, `Endpoint not supported for numeric type`);
@@ -595,8 +372,9 @@ class HomeAssistant extends Extension {
                                 ...lookup[expose.name],
                             },
                         };
-                    } else if (expose.type === 'enum' || expose.type === 'text') {
-                        if (expose.access === 'r' || expose.access === 'rw') {
+                    } else if (expose.type === 'enum' || expose.type === 'text' || expose.type === 'composite') {
+                        const ACCESS_STATE = 1;
+                        if (expose.access & ACCESS_STATE) {
                             const lookup = {
                                 action: {icon: 'mdi:gesture-double-tap'},
                             };
@@ -618,8 +396,6 @@ class HomeAssistant extends Extension {
                         this.mapping[def.model].push(discoveryEntry);
                     }
                 }
-            } else if (manualMaping.hasOwnProperty(def.model)) {
-                this.mapping[def.model] = manualMaping[def.model];
             } else {
                 logger.warn(`Supported device '${def.model}' has no Home Assistant mapping`);
             }
@@ -637,7 +413,7 @@ class HomeAssistant extends Extension {
         delete this.discovered[resolvedEntity.device.ieeeAddr];
         for (const config of this.getConfigs(resolvedEntity)) {
             const topic = this.getDiscoveryTopic(config, resolvedEntity.device);
-            this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic);
+            this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
         }
     }
 
@@ -711,7 +487,7 @@ class HomeAssistant extends Extension {
             },
             {
                 property: 'update',
-                condition: supportsOTA && this.newApi,
+                condition: supportsOTA,
                 value: {state: 'idle'},
             },
             {
@@ -739,7 +515,7 @@ class HomeAssistant extends Extension {
         if (homeAssisantRename) {
             for (const config of this.getConfigs(resolvedEntity)) {
                 const topic = this.getDiscoveryTopic(config, device);
-                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic);
+                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
             }
         }
 
@@ -770,23 +546,33 @@ class HomeAssistant extends Extension {
         if (!resolvedEntity || !resolvedEntity.definition || !this.mapping[resolvedEntity.definition.model]) return [];
 
         let configs = this.mapping[resolvedEntity.definition.model].slice();
-        if (!resolvedEntity.definition.hasOwnProperty('exposes')) {
-            // Exposes already has linkquality.
-            configs.push(cfg.sensor_linkquality);
-        }
-
         if (resolvedEntity.definition.hasOwnProperty('ota')) {
-            if (this.legacyApi) {
-                configs.push(cfg.binary_sensor_update_available);
-            }
+            const updateStateSensor = {
+                type: 'sensor',
+                object_id: 'update_state',
+                discovery_payload: {
+                    icon: 'mdi:update',
+                    value_template: `{{ value_json['update']['state'] }}`,
+                },
+            };
 
-            if (this.newApi) {
-                configs.push(cfg.sensor_update_state);
+            configs.push(updateStateSensor);
+            if (this.legacyApi) {
+                const updateAvailableSensor = {
+                    type: 'binary_sensor',
+                    object_id: 'update_available',
+                    discovery_payload: {
+                        payload_on: true,
+                        payload_off: false,
+                        value_template: '{{ value_json.update_available}}',
+                    },
+                };
+                configs.push(updateAvailableSensor);
             }
         }
 
         if (resolvedEntity.settings.hasOwnProperty('legacy') && !resolvedEntity.settings.legacy) {
-            configs = configs.filter((c) => c !== cfg.sensor_click);
+            configs = configs.filter((c) => c !== sensorClick);
         }
 
         if (!settings.get().advanced.homeassistant_legacy_triggers) {
@@ -796,7 +582,7 @@ class HomeAssistant extends Extension {
         // deep clone of the config objects
         configs = JSON.parse(JSON.stringify(configs));
 
-        if (resolvedEntity.settings.hasOwnProperty('homeassistant')) {
+        if (resolvedEntity.settings.homeassistant) {
             configs.forEach((config) => {
                 const configOverride = resolvedEntity.settings.homeassistant[config.object_id];
                 if (configOverride) {
@@ -847,8 +633,13 @@ class HomeAssistant extends Extension {
             payload.json_attributes_topic = stateTopic;
 
             // Set (unique) name, separate by space if friendlyName contains space.
-            const nameSeparator = friendlyName.includes(' ') ? ' ' : '_';
-            payload.name = `${friendlyName}${nameSeparator}${config.object_id}`;
+            const nameSeparator = friendlyName.includes('_') ? '_' : ' ';
+            payload.name = friendlyName;
+            if (config.object_id.startsWith(config.type) && config.object_id.includes('_')) {
+                payload.name += `${nameSeparator}${config.object_id.split(/_(.+)/)[1]}`;
+            } else if (!config.object_id.startsWith(config.type)) {
+                payload.name += `${nameSeparator}${config.object_id.replace(/_/g, nameSeparator)}`;
+            }
 
             // Set unique_id
             payload.unique_id = `${resolvedEntity.settings.ID}_${config.object_id}_${settings.get().mqtt.base_topic}`;
@@ -857,10 +648,10 @@ class HomeAssistant extends Extension {
             payload.device = this.getDevicePayload(resolvedEntity);
 
             // Availability payload
-            payload.availability = [
-                {topic: `${settings.get().mqtt.base_topic}/bridge/state`},
-                {topic: `${settings.get().mqtt.base_topic}/${friendlyName}/availability`},
-            ];
+            payload.availability = [{topic: `${settings.get().mqtt.base_topic}/bridge/state`}];
+            if (settings.get().advanced.availability_timeout) {
+                payload.availability.push({topic: `${settings.get().mqtt.base_topic}/${friendlyName}/availability`});
+            }
 
             if (payload.command_topic) {
                 payload.command_topic = `${settings.get().mqtt.base_topic}/${friendlyName}/`;
@@ -986,7 +777,7 @@ class HomeAssistant extends Extension {
             }
 
             const topic = this.getDiscoveryTopic(config, device);
-            this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic);
+            this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic, false, false);
         });
 
         this.discovered[device.ieeeAddr] = true;
@@ -1034,11 +825,14 @@ class HomeAssistant extends Extension {
                 const objectID = discoveryMatch[3];
                 clear = !this.getConfigs(resolvedEntity).find((c) => c.type === type && c.object_id === objectID);
             }
+            // Device was flagged to be excluded from homeassistant discovery
+            clear = clear || (resolvedEntity.settings.hasOwnProperty('homeassistant') &&
+                                !resolvedEntity.settings.homeassistant);
 
             if (clear) {
                 logger.debug(`Clearing Home Assistant config '${topic}'`);
                 topic = topic.substring(this.discoveryTopic.length + 1);
-                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic);
+                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
             }
         } else if ((topic === this.statusTopic || topic === defaultStatusTopic) && message.toLowerCase() === 'online') {
             const timer = setTimeout(async () => {
@@ -1085,8 +879,15 @@ class HomeAssistant extends Extension {
             return;
         }
 
-        const config = cfg[`trigger_${key}`];
-        config.object_id = `${key}_${value}`;
+        const config = {
+            type: 'device_automation',
+            object_id: `${key}_${value}`,
+            discovery_payload: {
+                automation_type: 'trigger',
+                type: key,
+            },
+        };
+
         const topic = this.getDiscoveryTopic(config, device);
         const payload = {
             ...config.discovery_payload,
@@ -1096,7 +897,7 @@ class HomeAssistant extends Extension {
             device: this.getDevicePayload(entity),
         };
 
-        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic);
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic, false, false);
         this.discoveredTriggers[device.ieeeAddr].add(discoveredKey);
     }
 

lib/extension/legacy/bridgeLegacy.js

@@ -177,7 +177,9 @@ class BridgeLegacy extends Extension {
         });
 
         if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, stringify(devices), {});
+            this.mqtt.publish(
+                `bridge/config/devices`, stringify(devices), {}, settings.get().mqtt.base_topic, false, false,
+            );
         } else {
             this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
         }
@@ -229,7 +231,7 @@ class BridgeLegacy extends Extension {
             settings.changeFriendlyName(from, to);
             logger.info(`Successfully renamed - ${from} to ${to} `);
             const entity = this.zigbee.resolveEntity(to);
-            const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device, from, to};
             eventData.homeAssisantRename = false;
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
 
@@ -351,8 +353,6 @@ class BridgeLegacy extends Extension {
     }
 
     async onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
         await this.publish();
     }
 

lib/extension/legacy/deviceGroupMembership.js

@@ -6,13 +6,6 @@ const Extension = require('../extension');
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
 class DeviceGroupMembership extends Extension {
-    onMQTTConnected() {
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
-            this.mqtt.subscribe(topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         const match = topic.match(topicRegex);
         if (!match) {

lib/extension/networkMap.js

@@ -29,18 +29,6 @@ class NetworkMap extends Extension {
         };
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(this.legacyTopic);
-            this.mqtt.subscribe(this.legacyTopicRoutes);
-        }
-
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         /* istanbul ignore else */
         if (this.legacyApi) {
@@ -54,7 +42,7 @@ class NetworkMap extends Extension {
             }
         }
 
-        if (settings.get().experimental.new_api && topic === this.topic) {
+        if (topic === this.topic) {
             try {
                 message = utils.parseJSON(message, message);
                 const type = typeof message === 'object' ? message.type : message;
@@ -256,7 +244,11 @@ class NetworkMap extends Extension {
                 model: resolvedEntity.definition.model,
                 vendor: resolvedEntity.definition.vendor,
                 description: resolvedEntity.definition.description,
-                supports: resolvedEntity.definition.supports,
+                supports: Array.from(new Set((resolvedEntity.definition.exposes || []).map((e) => {
+                    return e.hasOwnProperty('name') ?
+                        e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
             } : null;
 
             networkMap.nodes.push({

lib/extension/otaUpdate.js

@@ -3,6 +3,7 @@ const logger = require('../util/logger');
 const stringify = require('json-stable-stringify-without-jsonify');
 const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const tradfriOTA = require('zigbee-herdsman-converters/lib/ota/tradfri');
 const topicRegex =
     new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
 
@@ -15,21 +16,13 @@ class OTAUpdate extends Extension {
         this.inProgress = new Set();
         this.lastChecked = {};
         this.legacyApi = settings.get().advanced.legacy_api;
+
+        if (settings.get().advanced.ikea_ota_use_test_url) {
+            tradfriOTA.useTestURL();
+        }
     }
 
     onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
-        }
-
         for (const device of this.zigbee.getClients()) {
             // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
             // remove them.
@@ -102,20 +95,15 @@ class OTAUpdate extends Extension {
     }
 
     getEntityPublishPayload(state, progress=null, remaining=null) {
-        const payload = {};
+        const payload = {update: {state}};
+        if (progress !== null) payload.update.progress = progress;
+        if (remaining !== null) payload.update.remaining = Math.round(remaining);
 
         /* istanbul ignore else */
         if (this.legacyApi) {
             payload.update_available = state === 'available';
         }
 
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            payload.update = {state};
-            if (progress !== null) payload.update.progress = progress;
-            if (remaining !== null) payload.update.remaining = Math.round(remaining);
-        }
-
         return payload;
     }
 

lib/extension/publish.js

@@ -27,18 +27,6 @@ const groupConverters = [
 ];
 
 class EntityPublish extends Extension {
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < 20; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
     parseTopic(topic) {
         const match = topic.match(topicRegex);
         if (!match) {
@@ -178,14 +166,13 @@ class EntityPublish extends Extension {
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const underscoreIndex = key.lastIndexOf('_');
-                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getEndpointNames().includes(possibleEndpointName)) {
-                    endpointName = possibleEndpointName;
-                    key = key.substring(0, underscoreIndex);
+                const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
+                if (endpointNameMatch) {
+                    endpointName = endpointNameMatch;
+                    key = key.replace(`_${endpointNameMatch}`, '');
+
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
-
                     if (!actualTarget) {
                         logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
                         continue;
@@ -212,7 +199,7 @@ class EntityPublish extends Extension {
             const meta = {
                 endpoint_name: endpointName,
                 options,
-                message: json,
+                message: {...json},
                 logger,
                 device,
                 state: entityState,
@@ -220,6 +207,17 @@ class EntityPublish extends Extension {
                 mapped: definition,
             };
 
+            // Strip endpoint name from meta.message properties.
+            if (meta.endpoint_name) {
+                for (const [key, value] of Object.entries(meta.message)) {
+                    if (key.endsWith(meta.endpoint_name)) {
+                        delete meta.message[key];
+                        const keyWithoutEndpoint = key.substring(0, key.length - meta.endpoint_name.length - 1);
+                        meta.message[keyWithoutEndpoint] = value;
+                    }
+                }
+            }
+
             try {
                 if (topic.type === 'set' && converter.convertSet) {
                     logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);

lib/extension/receive.js

@@ -74,16 +74,13 @@ class Receive extends Extension {
             return false;
         }
 
-        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
-            logger.debug('Ignoring message from coordinator');
-            return false;
-        }
-
         if (!resolvedEntity.definition) {
             if (data.device.interviewing) {
                 logger.debug(`Skipping message, definition is undefined and still interviewing`);
             } else {
-                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.modelID}' ` +
+                    `and manufacturer name '${data.device.manufacturerName}'`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
             }
 

lib/extension/report.js

@@ -81,8 +81,13 @@ const pollOnMessage = [
         },
         // Read the following attributes
         read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
-        // When the bound devices/members of group have the following manufacturerID
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
     },
     {
         key: 2,
@@ -98,7 +103,12 @@ const pollOnMessage = [
             ],
         },
         read: {cluster: 'genOnOff', attributes: ['onOff']},
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+        manufacturerIDs: [
+            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
     },
 ];
 
@@ -164,6 +174,8 @@ class Report extends Extension {
                 delete device.meta.reporting;
                 this.eventBus.emit('reportingDisabled', {device});
             }
+
+            this.eventBus.emit(`devicesChanged`);
         } catch (error) {
             logger.error(
                 `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
@@ -185,8 +197,14 @@ class Report extends Extension {
         // This message is typically send when a device comes online after being powered off
         // Ikea TRADFRI tend to forget their reporting after powered off.
         // Re-setup reporting.
+        // Only resetup reporting if configuredReportings was not populated yet,
+        // else reconfigure is done in zigbee-herdsman-converters ikea.js/bulbOnEvent
+        // configuredReportings are saved since Zigbee2MQTT 1.17.0
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
+        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device) &&
+            device.endpoints.filter((e) => e.configuredReportings.length === 0).length === device.endpoints.length) {
+            return true;
+        }
 
         // These do not support reproting.
         // https://github.com/Koenkk/zigbee-herdsman/issues/110
@@ -264,7 +282,7 @@ class Report extends Extension {
 
             for (const endpoint of toPoll) {
                 for (const poll of polls) {
-                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
                         continue;
                     }
 

lib/mqtt.js

@@ -8,6 +8,7 @@ class MQTT extends events.EventEmitter {
     constructor() {
         super();
         this.onMessage = this.onMessage.bind(this);
+        this.publishedTopics = new Set();
     }
 
     async connect() {
@@ -18,7 +19,7 @@ class MQTT extends events.EventEmitter {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
                 payload: 'offline',
-                retain: true,
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
             },
         };
 
@@ -44,8 +45,11 @@ class MQTT extends events.EventEmitter {
         }
 
         if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
         }
 
         if (mqttSettings.client_id) {
@@ -69,8 +73,11 @@ class MQTT extends events.EventEmitter {
         return new Promise((resolve) => {
             this.client = mqtt.connect(mqttSettings.server, options);
 
+            const self = this;
             this.client.on('connect', () => {
                 logger.info('Connected to MQTT server');
+                self.subscribe(`${settings.get().mqtt.base_topic}/#`);
+                self.publish('bridge/state', 'online', {retain: true, qos: 0});
                 resolve();
             });
 
@@ -91,17 +98,24 @@ class MQTT extends events.EventEmitter {
     }
 
     onMessage(topic, message) {
-        this.emit('message', {topic, message: message + ''});
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            this.emit('message', {topic, message: message + ''});
+        }
     }
 
     isConnected() {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -114,8 +128,13 @@ class MQTT extends events.EventEmitter {
             logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
         }
 
+        const actualOptions = {...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
+
         return new Promise((resolve) => {
-            this.client.publish(topic, payload, options, () => {
+            this.client.publish(topic, payload, actualOptions, () => {
                 this.emit('publishedMessage', {topic, payload, options});
                 resolve();
             });

lib/state.js

@@ -3,7 +3,6 @@ const data = require('./util/data');
 const settings = require('./util/settings');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
-const stringify = require('json-stable-stringify-without-jsonify');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
@@ -57,7 +56,7 @@ class State {
     save() {
         if (settings.get().advanced.cache_state_persistent) {
             logger.debug(`Saving state to file ${this.file}`);
-            const json = stringify(this.state, null, 4);
+            const json = JSON.stringify(this.state, null, 4);
             try {
                 fs.writeFileSync(this.file, json, 'utf8');
             } catch (e) {

lib/util/settings.js

@@ -17,6 +17,11 @@ const defaults = {
     permit_join: false,
     mqtt: {
         include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
     },
     serial: {
         disable_led: false,
@@ -45,7 +50,6 @@ const defaults = {
     experimental: {
         // json or attribute or attribute_and_json
         output: 'json',
-        new_api: false,
     },
     advanced: {
         legacy_api: true,
@@ -60,6 +64,7 @@ const defaults = {
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         adapter_concurrent: null,
+        adapter_delay: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
@@ -153,6 +158,7 @@ const schema = {
                 reject_unauthorized: {type: 'boolean'},
                 include_device_information: {type: 'boolean'},
                 version: {type: 'number'},
+                force_disable_retain: {type: 'boolean'},
             },
             required: ['base_topic', 'server'],
         },
@@ -208,6 +214,7 @@ const schema = {
                 homeassistant_status_topic: {type: 'string'},
                 timestamp_format: {type: 'string'},
                 adapter_concurrent: {type: 'number'},
+                adapter_delay: {type: 'number'},
             },
         },
         map_options: {
@@ -317,15 +324,27 @@ function write() {
         }
     }
 
-    if (typeof actual.devices === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
-        toWrite.devices = actual.devices;
-    }
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type) => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            const content = objectAssignDeep.noMutate(settings[type]);
 
-    if (typeof actual.groups === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
-        toWrite.groups = actual.groups;
-    }
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f, i) => i !== 0)
+                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c) => Object.keys(c))
+                    .forEach((k) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
 
     yaml.writeIfChanged(file, toWrite);
 
@@ -420,16 +439,20 @@ function read() {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    // Read devices/groups configuration from separate file.
-    if (typeof s.devices === 'string') {
-        const file = data.joinPath(s.devices);
-        s.devices = yaml.readIfExists(file) || {};
-    }
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type) => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
 
-    if (typeof s.groups === 'string') {
-        const file = data.joinPath(s.groups);
-        s.groups = yaml.readIfExists(file) || {};
-    }
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
 
     return s;
 }
@@ -450,15 +473,15 @@ function applyEnvironmentVariables(settings) {
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
                             setting[key] = JSON.parse(process.env[envVariableName]);
-                        }
-                        if (type.indexOf('number') >= 0) {
+                        } else if (type.indexOf('number') >= 0) {
                             setting[key] = process.env[envVariableName] * 1;
-                        }
-                        if (type.indexOf('boolean') >= 0) {
+                        } else if (type.indexOf('boolean') >= 0) {
                             setting[key] = process.env[envVariableName].toLowerCase() === 'true';
-                        }
-                        if (type.indexOf('string') >= 0) {
-                            setting[key] = process.env[envVariableName];
+                        } else {
+                            /* istanbul ignore else */
+                            if (type.indexOf('string') >= 0) {
+                                setting[key] = process.env[envVariableName];
+                            }
                         }
                     }
                 }

lib/util/utils.js

@@ -1,6 +1,8 @@
 const equals = require('fast-deep-equal/es6');
 const humanizeDuration = require('humanize-duration');
 const data = require('./data');
+const vm = require('vm');
+const fs = require('fs');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -38,7 +40,7 @@ const endpointNames = [
     'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
     'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-    'heat', 'cool',
+    'heat', 'cool', 'water',
 ];
 
 function capitalize(s) {
@@ -69,6 +71,14 @@ async function getZigbee2mqttVersion() {
     });
 }
 
+async function getDependencyVersion(depend) {
+    return new Promise((resolve, reject) => {
+        const packageJSON = require('../../node_modules/'+depend+'/package.json');
+        const version = packageJSON.version;
+        resolve({version});
+    });
+}
+
 function formatDate(date, type) {
     let result;
 
@@ -138,13 +148,20 @@ function* getExternalConvertersDefinitions(settings) {
     const externalConverters = settings.get().external_converters;
 
     for (const moduleName of externalConverters) {
-        let converterModule = moduleName;
+        let converter;
 
         if (moduleName.endsWith('.js')) {
-            converterModule = data.joinPath(moduleName.split('.')[0]);
+            const sandbox = {
+                require,
+                module: {},
+            };
+            const converterCode = fs.readFileSync(data.joinPath(moduleName), {encoding: 'utf8'});
+            vm.runInNewContext(converterCode, sandbox);
+            converter = sandbox.module.exports;
+        } else {
+            converter = require(moduleName);
         }
 
-        const converter = require(converterModule);
         if (Array.isArray(converter)) {
             for (const item of converter) {
                 yield item;
@@ -157,6 +174,7 @@ function* getExternalConvertersDefinitions(settings) {
 
 function toSnakeCase(value) {
     if (typeof value === 'object') {
+        value = {...value};
         for (const key of Object.keys(value)) {
             const keySnakeCase = toSnakeCase(key);
             if (key !== keySnakeCase) {
@@ -184,6 +202,7 @@ module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
+    getDependencyVersion,
     objectHasProperties,
     toSnakeCase,
     getObjectProperty,

lib/util/yaml.js

@@ -14,8 +14,8 @@ function read(file) {
     }
 }
 
-function readIfExists(file) {
-    return fs.existsSync(file) ? read(file) : null;
+function readIfExists(file, default_=null) {
+    return fs.existsSync(file) ? read(file) : default_;
 }
 
 function writeIfChanged(file, content) {

lib/zigbee.js

@@ -19,7 +19,8 @@ class Zigbee extends events.EventEmitter {
     }
 
     async start() {
-        logger.info(`Starting zigbee-herdsman...`);
+        const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
+        logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
         const herdsmanSettings = {
             network: {
                 panID: settings.get().advanced.pan_id,
@@ -38,6 +39,7 @@ class Zigbee extends events.EventEmitter {
             },
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
+                delay: settings.get().advanced.adapter_delay,
             },
         };
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.16.0",
+  "version": "1.17.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^14"
+    "node": "^10 || ^12 || ^14 || ^15"
   },
   "keywords": [
     "xiaomi",
@@ -38,13 +38,12 @@
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.1.2",
     "git-last-commit": "^1.0.0",
-    "http-proxy": "^1.18.1",
     "humanize-duration": "^3.23.1",
     "js-yaml": "^3.14.0",
     "json-stable-stringify-without-jsonify": "=1.0.1",
     "mkdir-recursive": "^0.4.0",
     "moment": "^2.27.0",
-    "mqtt": "^4.2.1",
+    "mqtt": "4.2.1",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
     "semver": "^7.3.2",
@@ -52,9 +51,9 @@
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
     "ws": "^7.3.1",
-    "zigbee-herdsman": "0.13.26",
-    "zigbee-herdsman-converters": "12.0.224",
-    "zigbee2mqtt-frontend": "0.2.68"
+    "zigbee-herdsman": "0.13.46",
+    "zigbee-herdsman-converters": "14.0.3",
+    "zigbee2mqtt-frontend": "0.2.129"
   },
   "devDependencies": {
     "eslint": "*",
@@ -75,5 +74,8 @@
     "collectCoverageFrom": [
       "lib/**"
     ]
+  },
+  "bin": {
+    "zigbee2mqtt": "./cli.js"
   }
 }

scripts/zStackEraseAllNvMem.js

@@ -0,0 +1,89 @@
+const {ZnpCommandStatus, NvSystemIds} = require('zigbee-herdsman/dist/adapter/z-stack/constants/common');
+const {ZnpVersion} = require('zigbee-herdsman/dist/adapter/z-stack/adapter/tstype');
+const {Subsystem} = require('zigbee-herdsman/dist/adapter/z-stack/unpi/constants');
+const {Znp} = require('zigbee-herdsman/dist/adapter/z-stack/znp');
+
+
+class ZStackNvMemEraser {
+    constructor(device) {
+        this.znp = new Znp(device, 115200, false);
+    }
+
+    async start() {
+        await this.znp.open();
+        const attempts = 3;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                await this.znp.request(Subsystem.SYS, 'ping', {capabilities: 1});
+                break;
+            } catch (e) {
+                if (attempts - 1 === i) {
+                    throw new Error(`Failed to connect to the adapter (${e})`);
+                }
+            }
+        }
+        // Old firmware did not support version, assume it's Z-Stack 1.2 for now.
+        try {
+            this.version = (await this.znp.request(Subsystem.SYS, 'version', {})).payload;
+        } catch (e) {
+            console.log(`Failed to get zStack version, assuming 1.2`);
+            this.version = {'transportrev': 2, 'product': 0, 'majorrel': 2,
+                'minorrel': 0, 'maintrel': 0, 'revision': ''};
+        }
+
+        console.log(`Detected znp version '${ZnpVersion[this.version.product]}' (${JSON.stringify(this.version)})`);
+
+        await this.clearAllNvMemItems();
+
+        process.exit(0);
+    }
+
+    async clearAllNvMemItems() {
+        let maxNvMemId;
+        switch (this.version.product) {
+        case ZnpVersion.zStack12: maxNvMemId = 0x0302; break;
+        case ZnpVersion.zStack30x: maxNvMemId = 0x033F; break;
+        case ZnpVersion.zStack3x0: maxNvMemId = 0x032F; break;
+        }
+
+        let deletedCount = 0;
+        console.log(`Clearing all NVMEM items, from 0 to ${maxNvMemId}`);
+        for (let id=0; id<=maxNvMemId; id++) {
+            let len;
+            const needOsal = !(this.version.product == ZnpVersion.zStack3x0 && id <= 7);
+            if (needOsal) {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'osalNvLength', {id: id});
+                len = lengthRes.payload['length'];
+            } else {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'nvLength',
+                    {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0});
+                len = lengthRes.payload['len'];
+            }
+            if (len != 0) {
+                console.log(`NVMEM item #${id} - deleting, size: ${len}`);
+                if (needOsal) {
+                    await this.znp.request(Subsystem.SYS, 'osalNvDelete',
+                        {id: id, len: len},
+                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                } else {
+                    await this.znp.request(Subsystem.SYS, 'nvDelete',
+                        {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0},
+                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                }
+                deletedCount++;
+            }
+        }
+        console.log(`Clearing all NVMEM items finished, deleted ${deletedCount} items`);
+    }
+}
+const processArgs = process.argv.slice(2);
+if (processArgs.length != 1) {
+    console.log('ZStack NVMEM eraser.');
+    console.log('Usage:');
+    console.log('   node zStackEraseAllNvMem.js <SERIAL_PORT>');
+    process.exit(1);
+}
+
+const eraser = new ZStackNvMemEraser(processArgs[0]);
+
+eraser.start();

test/assets/mock-external-converter.js

@@ -1,5 +1,11 @@
 const mockDevice = {
-    mock: true
+    mock: true,
+    zigbeeModel: ['external_converter_device'],
+    vendor: 'external',
+    model: ['external_converter_device'],
+    definition: {
+        foo: 'bar'
+    }
 };
 
 module.exports = mockDevice;

test/availability.test.js

@@ -1,5 +1,6 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
@@ -28,7 +29,6 @@ describe('Availability', () => {
         data.writeEmptyState();
         jest.useFakeTimers();
         settings.set(['advanced', 'availability_timeout'], 10);
-        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         mocksClear.forEach((m) => m.mockClear());
         await controller.start();
@@ -343,7 +343,7 @@ describe('Availability', () => {
     });
 
     it('Should publish availability when end device joins', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const payload = {device};
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.deviceJoined(payload);
@@ -409,4 +409,34 @@ describe('Availability', () => {
             {retain: true, qos: 0}, expect.any(Function)
         );
     });
+
+    it('Should clear retained availability topic when device is renamed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "bulb_color", "to": "bulb_color_new_name"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should clear retained availability topic when device does not exist', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/not_existing_hahaha/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/not_existing_hahaha/availability', null, {retain: true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/0x000b57fffec6a5b3/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/0x000b57fffec6a5b3/availability', null, {retain: true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bulb_color/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
 });

test/bind.test.js

@@ -25,7 +25,6 @@ describe('Bind', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
-        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -33,12 +32,6 @@ describe('Bind', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
-    });
-
     it('Should bind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);

test/controller.test.js

@@ -31,7 +31,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -256,6 +256,17 @@ describe('Controller', () => {
         expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'dummytopic' with data 'dummymessage'")
     });
 
+    it('Skip MQTT messages on topic we published to', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'zigbee2mqtt/skip-this-topic' with data 'skipped'")
+        logger.debug.mockClear();
+        await controller.mqtt.publish('skip-this-topic', '', {});
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledTimes(0);
+    });
+
     it('On zigbee event message', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
@@ -598,4 +609,24 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
     });
 
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
+
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
 });

test/externalConverters.test.js

@@ -67,7 +67,13 @@ describe('Loads external converters', () => {
         await flushPromises();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
-            mock: true
+            mock: true,
+            zigbeeModel: ['external_converter_device'],
+            vendor: 'external',
+            model: ['external_converter_device'],
+            definition: {
+                foo: 'bar'
+            }
         });
     });
 

test/frontend.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
 const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mockHTTP = {
@@ -18,20 +19,17 @@ const mockHTTP = {
     events: {},
 };
 
-const mockHTTPProxy = {
-    implementation: {
-        web: jest.fn(),
-        ws: jest.fn(),
-    },
-    variables: {},
-    events: {},
+const mockWSocket = {
+    close: jest.fn(),
 };
 
 const mockWS = {
     implementation: {
         clients: [],
         on: (event, handler) => {mockWS.events[event] = handler},
-        handleUpgrade: jest.fn(),
+        handleUpgrade: jest.fn().mockImplementation((request, socket, head, cb) => {
+            cb(mockWSocket)
+        }),
         emit: jest.fn(),
     },
     variables: {},
@@ -51,13 +49,6 @@ jest.mock('http', () => ({
     }),
 }));
 
-jest.mock('http-proxy', () => ({
-    createProxyServer: jest.fn().mockImplementation((initParameter) => {
-        mockHTTPProxy.variables.initParameter = initParameter;
-        return mockHTTPProxy.implementation;
-    }),
-}));
-
 jest.mock("serve-static", () =>
     jest.fn().mockImplementation((path) => {
         mockNodeStatic.variables.path = path
@@ -84,16 +75,20 @@ describe('Frontend', () => {
         data.writeDefaultConfiguration();
         data.writeDefaultState();
         settings._reRead();
-        settings.set(['experimental'], {new_api: true});
-        settings.set(['frontend'], {port: 8081});
+        settings.set(['frontend'], {port: 8081, host: "127.0.0.1"});
         settings.set(['homeassistant'], true);
+        zigbeeHerdsman.devices.bulb.linkquality = 10;
+    });
+
+    afterEach(async() => {
+        delete zigbeeHerdsman.devices.bulb.linkquality;
     });
 
     it('Start/stop', async () => {
         controller = new Controller();
         await controller.start();
         expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
-        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081);
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
 
         const mockWSClient = {
             implementation: {
@@ -122,7 +117,6 @@ describe('Frontend', () => {
         };
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
-        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(48);
 
         expect(JSON.parse(mockWSClient.implementation.send.mock.calls[0])).toStrictEqual({topic: 'bridge/state', payload: 'online'});
         expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}});
@@ -176,32 +170,44 @@ describe('Frontend', () => {
         mockWS.implementation.handleUpgrade.mock.calls[0][3](99);
         expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', 99, {"url": "http://localhost:8080/api"});
 
-        mockWS.implementation.handleUpgrade.mockClear();
-        mockHTTP.events.upgrade({url: 'http://localhost:8080/unkown'}, mockSocket, 3);
-        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(0);
-        expect(mockSocket.destroy).toHaveBeenCalledTimes(1);
-
         mockHTTP.variables.onRequest(1, 2);
         expect(mockNodeStatic.implementation).toHaveBeenCalledTimes(1);
         expect(mockNodeStatic.implementation).toHaveBeenCalledWith(1, 2, expect.any(Function));
     });
 
-    it('Development server', async () => {
-        settings.set(['frontend'], {development_server: 'localhost:3001'});
+    it('Static server', async () => {
         controller = new Controller();
         await controller.start();
-        expect(mockHTTPProxy.variables.initParameter).toStrictEqual({ws: true});
-        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8080);
 
-        mockHTTP.variables.onRequest(1, 2);
-        expect(mockHTTPProxy.implementation.web).toHaveBeenCalledTimes(1);
-        expect(mockHTTPProxy.implementation.web).toHaveBeenCalledWith(1, 2, {"target": "http://localhost:3001"});
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+    });
+
+    it('Authentification', async () => {
+        const authToken = 'sample-secure-token'
+        settings.set(['frontend'], {auth_token: authToken});
+        controller = new Controller();
+        await controller.start();
 
         const mockSocket = {destroy: jest.fn()};
-        mockHTTPProxy.implementation.ws.mockClear();
-        mockHTTP.events.upgrade({url: 'http://localhost:8080/sockjs-node'}, mockSocket, 3);
-        expect(mockHTTPProxy.implementation.ws).toHaveBeenCalledTimes(1);
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: '/api'}, mockSocket, mockWSocket);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
         expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
-        expect(mockHTTPProxy.implementation.ws).toHaveBeenCalledWith({"url": "http://localhost:8080/sockjs-node"}, mockSocket, 3, {"target": "ws://localhost:3001"});
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "/api"}, mockSocket, mockWSocket, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledWith(4401, "Unauthorized");
+
+        mockWSocket.close.mockClear();
+        mockWS.implementation.emit.mockClear();
+
+        const url = `/api?token=${authToken}`;
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: url}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({url}, mockSocket, 3, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledTimes(0);
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](mockWSocket);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', mockWSocket, {url});
+
     });
 });

test/group.test.js

@@ -20,7 +20,6 @@ describe('Groups', () => {
 
     beforeEach(() => {
         data.writeEmptyState();
-        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
@@ -127,12 +126,6 @@ describe('Groups', () => {
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}), {"retain": false, qos: 0}, expect.any(Function));
-
-        // Test if subscribed to topics with slashes
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
     });
 
     it('Legacy api: Add to group via MQTT with postfix', async () => {

test/homeassistant.test.js

@@ -10,6 +10,8 @@ const fs = require('fs');
 const path = require('path');
 const HomeAssistant = require('../lib/extension/homeassistant');
 
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
 describe('HomeAssistant extension', () => {
     beforeEach(async () => {
         this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
@@ -22,19 +24,6 @@ describe('HomeAssistant extension', () => {
         settings.set(['homeassistant'], true);
     });
 
-    it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
-        const missing = [];
-        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
-
-        require('zigbee-herdsman-converters').definitions.forEach((d) => {
-            if (!d.hasOwnProperty('exposes') && !ha._getMapping()[d.model]) {
-                missing.push(d.model);
-            }
-        });
-
-        expect(missing).toHaveLength(0);
-    });
-
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
@@ -77,7 +66,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -102,7 +91,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -127,7 +116,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -152,7 +141,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -177,7 +166,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -186,6 +175,114 @@ describe('HomeAssistant extension', () => {
             { retain: true, qos: 0 },
             expect.any(Function),
         );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "command_topic":"zigbee2mqtt/wall_switch_double/left/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45542"
+                ],
+                "manufacturer":"Xiaomi",
+                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
+                "name":"wall_switch_double",
+                "sw_version":this.version
+            },
+            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
+            "name":"wall_switch_double_left",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/wall_switch_double",
+            "unique_id":"0x0017880104e45542_switch_left_zigbee2mqtt",
+            "value_template":"{{ value_json.state_left }}"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/0x0017880104e45542/switch_left/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "command_topic":"zigbee2mqtt/wall_switch_double/right/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45542"
+                ],
+                "manufacturer":"Xiaomi",
+                "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
+                "name":"wall_switch_double",
+                "sw_version":this.version
+            },
+            "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
+            "name":"wall_switch_double_right",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/wall_switch_double",
+            "unique_id":"0x0017880104e45542_switch_right_zigbee2mqtt",
+            "value_template":"{{ value_json.state_right }}"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/0x0017880104e45542/switch_right/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[
+                {
+                    "topic":"zigbee2mqtt/bridge/state"
+                }
+            ],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_temp":true,
+            "command_topic":"zigbee2mqtt/bulb/set",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x000b57fffec6a5b2"
+                ],
+                "manufacturer":"IKEA",
+                "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
+                "name":"bulb",
+                "sw_version":this.version,
+            },
+            "effect":true,
+            "effect_list":[
+                "blink",
+                "breathe",
+                "okay",
+                "channel_change",
+                "finish_effect",
+                "stop_effect"
+            ],
+            "hs":false,
+            "json_attributes_topic":"zigbee2mqtt/bulb",
+            "name":"bulb",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/bulb",
+            "unique_id":"0x000b57fffec6a5b2_light_zigbee2mqtt",
+            "xy":false
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/0x000b57fffec6a5b2/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
     });
 
     it('Should discover devices with precision', async () => {
@@ -218,7 +315,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -243,7 +340,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -268,7 +365,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -324,7 +421,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'custom model',
                 'manufacturer': 'From Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             'expire_after': 90,
             'icon': 'mdi:test',
         };
@@ -350,7 +447,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'custom model',
                 'manufacturer': 'Not from Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             'expire_after': 30,
             'icon': 'mdi:test',
         };
@@ -386,7 +483,7 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
 
         payload = {
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/my_switch/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             "command_topic": "zigbee2mqtt/my_switch/set",
             "device": {
               "identifiers": [
@@ -456,7 +553,7 @@ describe('HomeAssistant extension', () => {
                "smart"
             ],
             "json_attributes_topic":"zigbee2mqtt/fan",
-            "name":"fan_fan",
+            "name":"fan",
             "unique_id":"0x0017880104e45548_fan_zigbee2mqtt",
             "device":{
                "identifiers":[
@@ -467,7 +564,7 @@ describe('HomeAssistant extension', () => {
                "model":"Universal wink enabled white ceiling fan premier remote control (99432)",
                "manufacturer":"Hampton Bay"
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/fan/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
          };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -488,7 +585,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             "action_template":"{% set values = {'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'} %}{{ values[value_json.running_state] }}",
             "action_topic":"zigbee2mqtt/TS0601_thermostat",
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/TS0601_thermostat/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
             "away_mode_state_template":"{{ value_json.away_mode }}",
             "away_mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
@@ -515,15 +612,15 @@ describe('HomeAssistant extension', () => {
             "hold_state_template":"{{ value_json.preset }}",
             "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "json_attributes_topic":"zigbee2mqtt/TS0601_thermostat",
-            "max_temp":"30",
+            "max_temp":"35",
             "min_temp":"5",
             "mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/system_mode",
             "mode_state_template":"{{ value_json.system_mode }}",
             "mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
             "modes":[
-               "auto"
+               "heat", "auto", "off"
             ],
-            "name":"TS0601_thermostat_climate",
+            "name":"TS0601_thermostat",
             "temp_step":0.5,
             "temperature_command_topic":"zigbee2mqtt/TS0601_thermostat/set/current_heating_setpoint",
             "temperature_state_template":"{{ value_json.current_heating_setpoint }}",
@@ -554,7 +651,7 @@ describe('HomeAssistant extension', () => {
             set_position_template: '{ "position": {{ position }} }',
             value_template: '{{ value_json.position }}',
             json_attributes_topic: 'zigbee2mqtt/smart vent',
-            name: 'smart vent cover',
+            name: 'smart vent',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
             device:
             {
@@ -564,7 +661,7 @@ describe('HomeAssistant extension', () => {
                 model: 'Smart vent (SV01)',
                 manufacturer: 'Keen Home'
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/smart vent/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -598,7 +695,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -663,7 +760,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -684,8 +781,6 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        // 1 publish is from device_removed
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
     it('Should send all status when home assistant comes online (default topic)', async () => {
@@ -702,13 +797,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update":{"state":"idle"},"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"brightness":255,"update_available":false}),
+            stringify({"brightness":255,"update_available":false,"update":{"state":"idle"}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -728,13 +823,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false,"update":{"state":"idle"}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"brightness":255,"update_available":false}),
+            stringify({"brightness":255,"update_available":false,"update":{"state":"idle"}}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -853,7 +948,6 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should refresh discovery when device is renamed', async () => {
-        settings.set(['experimental', 'new_api'], true);
         controller = new Controller(false);
         await controller.start();
         await flushPromises();
@@ -878,7 +972,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor_renamed/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -919,7 +1013,6 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
-        settings.set(['experimental', 'new_api'], true);
         controller = new Controller(false);
         await controller.start();
         await flushPromises();
@@ -949,7 +1042,7 @@ describe('HomeAssistant extension', () => {
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/weather_sensor_renamed/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -970,7 +1063,7 @@ describe('HomeAssistant extension', () => {
             "value_template":"{{ value_json.update_available}}",
             "state_topic":"zigbee2mqtt/bulb",
             "json_attributes_topic":"zigbee2mqtt/bulb",
-            "name":"bulb_update_available",
+            "name":"bulb update available",
             "unique_id":"0x000b57fffec6a5b2_update_available_zigbee2mqtt",
             "device":{
                 "identifiers":[
@@ -981,7 +1074,7 @@ describe('HomeAssistant extension', () => {
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "manufacturer":"IKEA"
             },
-            'availability': [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/bulb/availability'}],
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -1290,7 +1383,7 @@ describe('HomeAssistant extension', () => {
                 command_topic: true,
             },
         };
-        expect(ha._getMapping()['external_converters_device']).toStrictEqual([homeassistantSwitch]);
+        expect(ha._getMapping()['external_converters_device']).toEqual([homeassistantSwitch]);
     });
 
     it('Should clear outdated configs', async () => {
@@ -1300,26 +1393,26 @@ describe('HomeAssistant extension', () => {
 
         // Non-existing device -> clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/0x123/availability'}]}));
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x123/temperature/config', null, {qos: 0, retain: true}, expect.any(Function));
 
         // Existing device -> don't clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}]}));
+        await MQTT.events.message('homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
 
         // Non-existing device of different instance -> don't clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt_different/bridge/state'}, {topic: 'zigbee2mqtt_different/0x123/availability'}]}));
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt_different/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
 
         // Existing device but non-existing config -> don't clear
         MQTT.publish.mockClear();
-        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}, {topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}]}));
+        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', null, {qos: 0, retain: true}, expect.any(Function));
@@ -1341,5 +1434,22 @@ describe('HomeAssistant extension', () => {
         await MQTT.events.message('homeassistant/device_automation/0x000b57fffec6a5b2_not_existing/action_button_3_single/config', stringify({topic: 'zigbee2mqtt_different/0x000b57fffec6a5b2_not_existing/availability'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Device was flagged to be excluded from homeassistant discovery
+        await controller.stop();
+        await flushPromises();
+        settings.set(['devices', '0x000b57fffec6a5b2', 'homeassistant'], null);
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', null, {qos: 0, retain: true}, expect.any(Function));
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', stringify({topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', null, {qos: 0, retain: true}, expect.any(Function));
     });
 });

test/networkMap.test.js

@@ -2,15 +2,17 @@ const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const stringify = require('json-stable-stringify-without-jsonify');
-
-const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
+const fs = require('fs');
+const path = require('path');
+const {coordinator, bulb, bulb_color, WXKG02LM_rev1, CC2530_ROUTER, unsupported_router, external_converter_device} = zigbeeHerdsman.devices;
 
 zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(bulb_color.ieeeAddr);
-zigbeeHerdsman.returnDevices.push(WXKG02LM.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(WXKG02LM_rev1.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(CC2530_ROUTER.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(unsupported_router.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(external_converter_device.ieeeAddr)
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -27,7 +29,8 @@ describe('Networkmap', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
-        settings.set(['experimental', 'new_api'], true);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        settings.set(['external_converters'], ['mock-external-converter.js']);
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
@@ -39,22 +42,24 @@ describe('Networkmap', () => {
         await flushPromises();
         const device = zigbeeHerdsman.devices.bulb_color;
         device.lastSeen = 1000;
+        external_converter_device.lastSeen = 1000;
     });
 
     function mock() {
         /**
          * Topology
-         *
+         *             | -> external_device
          *             | -> bulb_color -> unsupported_router (offline)
          * coordinator |      ^     ^
          *             |      |     | (not valid)
          *             | -> bulb    |
-         *                    |  -> CC2530_ROUTER -> WXKG02LM
+         *                    |  -> CC2530_ROUTER -> WXKG02LM_rev1
          *
          */
         coordinator.lqi = () => {return {neighbors: [
             {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 2, depth: 1, linkquality: 120},
-            {ieeeAddr: bulb.ieeeAddr, networkAddress: bulb.networkAddress, relationship: 2, depth: 1, linkquality: 92}
+            {ieeeAddr: bulb.ieeeAddr, networkAddress: bulb.networkAddress, relationship: 2, depth: 1, linkquality: 92},
+            {ieeeAddr: external_converter_device.ieeeAddr, networkAddress: external_converter_device.networkAddress, relationship: 2, depth: 1, linkquality: 92}
         ]}};
         coordinator.routingTable = () => {return {table: [
             {destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress},
@@ -70,7 +75,7 @@ describe('Networkmap', () => {
         bulb_color.routingTable = () => {return {table: []}};
 
         CC2530_ROUTER.lqi = () => {return {neighbors: [
-            {ieeeAddr: '0x0000000000000000', networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
+            {ieeeAddr: '0x0000000000000000', networkAddress: WXKG02LM_rev1.networkAddress, relationship: 1, depth: 2, linkquality: 130},
             {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 4, depth: 2, linkquality: 130},
         ]}};
         CC2530_ROUTER.routingTable = () => {return {table: []}};
@@ -87,26 +92,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            "nodes":[
-                {"definition":null,"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                {"definition":{"description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model": "LED1545G12","supports": "on/off, brightness, color temperature","vendor": "IKEA"},"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                {"definition":{"description": "Hue Go","model": "7146060PH","supports": "on/off, brightness, color temperature, color xy, power-on behavior","vendor": "Philips"},"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                {"definition":{"description": "Aqara double key wireless wall switch","model": "WXKG02LM","supports": "left, right, both click (and double, long click for left, right and both depending on model)","vendor": "Xiaomi"},"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                {"definition":null,"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                {"definition":{"description": "[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model": "CC2530.ROUTER","supports": "state, description, type, rssi","vendor": "Custom devices (DiY)"},"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-            ],
-            "links":[
-                {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-            ]
-        };
-
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"model":["external_converter_device"],"supports":"","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -152,11 +138,13 @@ describe('Networkmap', () => {
               "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|9 seconds ago}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
               "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
               "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
+              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (1114)|external undefined (external_converter_device)|9 seconds ago}"];
+              "0x0017880104e45511" -> "0x00124b00120144ae" [penwidth=1, weight=0, color="#994444", label="92"]
             }`;
 
         const expectedLines = expected.split('\n');
@@ -184,6 +172,16 @@ describe('Networkmap', () => {
         const expected = `' paste into: https://www.planttext.com/
 
         @startuml
+        card 0x0017880104e45511 [
+        0x0017880104e45511
+        ---
+        0x0017880104e45511 (1114)
+        ---
+        external undefined (external_converter_device)
+        ---
+        9 seconds ago
+        ]
+
         card 0x0017880104e45525 [
         0x0017880104e45525
         ---
@@ -219,7 +217,7 @@ describe('Networkmap', () => {
         ---
         0x0017880104e45521 (6538)
         ---
-        Xiaomi Aqara double key wireless wall switch (WXKG02LM)
+        Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)
         ---
         9 seconds ago
         ]
@@ -244,6 +242,7 @@ describe('Networkmap', () => {
 
         0x000b57fffec6a5b3 --> 0x00124b00120144ae: 120
         0x000b57fffec6a5b2 --> 0x00124b00120144ae: 92
+        0x0017880104e45511 --> 0x00124b00120144ae: 92
         0x000b57fffec6a5b3 --> 0x000b57fffec6a5b2: 110
         0x0017880104e45559 --> 0x000b57fffec6a5b2: 100
         0x0017880104e45521 --> 0x0017880104e45559: 130
@@ -267,33 +266,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            status: 'ok',
-            data: {
-                type: 'raw',
-                routes: true,
-                value: {
-                    "nodes":[
-                        {"definition":null,"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                        {"definition":{"description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model": "LED1545G12","supports": "on/off, brightness, color temperature","vendor": "IKEA"},"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                        {"definition":{"description": "Hue Go","model": "7146060PH","supports": "on/off, brightness, color temperature, color xy, power-on behavior","vendor": "Philips"},"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                        {"definition":{"description": "Aqara double key wireless wall switch","model": "WXKG02LM","supports": "left, right, both click (and double, long click for left, right and both depending on model)","vendor": "Xiaomi"},"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                        {"definition":null,"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                        {"definition":{"description": "[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model": "CC2530.ROUTER","supports": "state, description, type, rssi","vendor": "Custom devices (DiY)"},"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-                    ],
-                    "links":[
-                        {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                        {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                        {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                        {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                        {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-                    ]
-                }
-            }
-        };
-
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"model":["external_converter_device"],"supports":"","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
         expect(actual).toStrictEqual(expected);
     });

test/otaUpdate.test.js

@@ -19,7 +19,7 @@ describe('OTA update', () => {
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings._reRead();
-        settings.set(['experimental', 'new_api'], true);
+        settings.set(['advanced', 'ikea_ota_use_test_url'], true);
         data.writeEmptyState();
         controller = new Controller();
         await controller.start();
@@ -27,13 +27,6 @@ describe('OTA update', () => {
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/update');
-    });
-
     it('Should OTA update a device', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];

test/publish.test.js

@@ -195,6 +195,21 @@ describe('Publish', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
+    it('Should publish messages to zigbee devices to non default-ep with brightness_[EP]', async () => {
+        const device = zigbeeHerdsman.devices.QS_Zigbee_D02_TRIAC_2C_LN;
+        const endpoint = device.getEndpoint(2);
+        await MQTT.events.message('zigbee2mqtt/0x0017882194e45543/set', stringify({state_l2: 'ON', brightness_l2: 50}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/QS-Zigbee-D02-TRIAC-2C-LN',
+            stringify({brightness_l2: 50, state_l2: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
     it('Should publish messages to zigbee devices with color xy', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -339,7 +354,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/thermostat_group/set', stringify({child_lock: 'LOCK'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
-        expect(group.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 263, fn: 0, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 7, length_hi: 0, length_lo: 1, status: 0, transid: 0}, {disableDefaultResponse: true});
     });
 
     it('Should publish messages to groups with on and brightness', async () => {
@@ -388,10 +403,10 @@ describe('Publish', () => {
 
     it('Should create and publish to group which is in configuration.yaml but not in zigbee-herdsman', async () => {
         delete zigbeeHerdsman.groups.group_2;
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(6);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(7);
         await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'ON'}));
         await flushPromises();
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(7);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(8);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
     });
@@ -729,6 +744,35 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
     });
 
+    it('Should turn bulb on with correct brightness when device is turned off twice and brightness is reported', async () => {
+        // Test case for: https://github.com/Koenkk/zigbee2mqtt/issues/5413
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({"state": "ON", "brightness": 200}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({"state": "OFF", "transition": 0}));
+        await flushPromises();
+        await zigbeeHerdsman.events.message({data: {currentLevel: 1}, cluster: 'genLevelCtrl', device, endpoint, type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({"state": "OFF", "transition": 0}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({"state": "ON", "transition": 0}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1, 'zigbee2mqtt/bulb_color', stringify({state: 'ON', brightness: 200}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2, 'zigbee2mqtt/bulb_color', stringify({state: 'OFF', brightness: 200}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenNthCalledWith(3, 'zigbee2mqtt/bulb_color', stringify({state: 'OFF', brightness: 1}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenNthCalledWith(4, 'zigbee2mqtt/bulb_color', stringify({state: 'OFF', brightness: 1}), {retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenNthCalledWith(5, 'zigbee2mqtt/bulb_color', stringify({state: 'ON', brightness: 200}), {retain: false, qos: 0}, expect.any(Function));
+
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command).toHaveBeenNthCalledWith(1, "genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(2, "genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(3, "genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(4, "genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 0}, {});
+    });
+
     it('Should turn bulb on with full brightness when transition is used and no brightness is known', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/3799
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -1203,7 +1247,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 3, length_hi: 0, length_lo: 1, status: 0, transid: 1}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});

test/receive.test.js

@@ -35,7 +35,7 @@ describe('Receive', () => {
     });
 
     it('Should handle a zigbee message which uses ep (left)', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -47,7 +47,7 @@ describe('Receive', () => {
     });
 
     it('Should handle a zigbee message which uses ep (right)', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(2), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -259,7 +259,7 @@ describe('Receive', () => {
     });
 
     it('Should handle a zigbee message with voltage 3010', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 3010}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -271,7 +271,7 @@ describe('Receive', () => {
     });
 
     it('Should handle a zigbee message with voltage 2850', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 2850}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -283,7 +283,7 @@ describe('Receive', () => {
     });
 
     it('Should handle a zigbee message with voltage 2650', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 2650}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -295,7 +295,7 @@ describe('Receive', () => {
     });
 
     it('Should handle a zigbee message with voltage 2000', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 2000}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -328,7 +328,7 @@ describe('Receive', () => {
     });
 
     it('Should publish last_seen epoch', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'epoch');
         const data = {onOff: 1};
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
@@ -341,7 +341,7 @@ describe('Receive', () => {
     });
 
     it('Should publish last_seen ISO_8601', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'ISO_8601');
         const data = {onOff: 1};
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
@@ -354,7 +354,7 @@ describe('Receive', () => {
     });
 
     it('Should publish last_seen ISO_8601_local', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'ISO_8601_local');
         const data = {onOff: 1};
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
@@ -418,7 +418,7 @@ describe('Receive', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID' and manufacturer name 'notSupportedMfg'`);
         expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });

test/report.test.js

@@ -195,7 +195,7 @@ describe('Report', () => {
     });
 
     it('Should configure reporting when deviceAnnounce message from IKEA device', async () => {
-        const device = zigbeeHerdsman.devices.bulb;
+        const device = zigbeeHerdsman.devices.bulb_2;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         const payload = {device};
@@ -267,7 +267,7 @@ describe('Report', () => {
         expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
 
         // Should only call Hue bulb, not e.g. tradfri
-        expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
     });
 
     it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
@@ -287,6 +287,8 @@ describe('Report', () => {
         const device = zigbeeHerdsman.devices.bulb;
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
+        const configuredReportings = endpoint.configuredReportings;
+        endpoint.configuredReportings = [];
         delete device.meta.reporting;
         mockClear(device);
         endpoint.getClusterAttributeValue = jest.fn();
@@ -308,5 +310,6 @@ describe('Report', () => {
         expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
         expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        endpoint.configuredReportings = configuredReportings;
     });
 });

test/settings.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings.js');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
+const devicesFile2 = data.joinPath('devices2.yaml');
 const groupsFile = data.joinPath('groups.yaml');
 const secretFile = data.joinPath('secret.yaml');
 const yaml = require('js-yaml');
@@ -29,7 +30,7 @@ describe('Settings', () => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
     const clearEnvironmentVariables = () => {
-        Object.keys(process.env).forEach((key) => { 
+        Object.keys(process.env).forEach((key) => {
             if(key.indexOf('ZIGBEE2MQTT_CONFIG_') >= 0) {
                 delete process.env[key];
             }
@@ -153,6 +154,7 @@ describe('Settings', () => {
 
         const expected = {
             include_device_information: false,
+            force_disable_retain: false,
             password: "mysecretpassword",
             server: "my.mqtt.server",
             user: "mysecretusername",
@@ -196,6 +198,32 @@ describe('Settings', () => {
         expect(device).toStrictEqual(expected);
     });
 
+    it('Should read devices form 2 separate files', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+        expect(settings.getDevice('0x12345678').friendly_name).toStrictEqual('0x12345678');
+        expect(settings.getDevice('0x87654321').friendly_name).toStrictEqual('0x87654321');
+    });
+
     it('Should add devices to a separate file', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',
@@ -228,6 +256,47 @@ describe('Settings', () => {
         expect(read(devicesFile)).toStrictEqual(expected);
     });
 
+    it('Should add devices for first file when using 2 separates file', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual({devices: ['devices.yaml', 'devices2.yaml']});
+
+        const expected = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expected);
+        expect(read(devicesFile2)).toStrictEqual(contentDevices2);
+    });
+
     it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',

test/stub/data.js

@@ -162,6 +162,9 @@ function writeDefaultConfiguration() {
             '0x0017882104a44559': {
                 friendly_name: 'TS0601_thermostat',
             },
+            '0x0017882194e45543': {
+                friendly_name: 'QS-Zigbee-D02-TRIAC-2C-LN',
+            },
         },
         groups: {
             '1': {

test/stub/zigbeeHerdsman.js

@@ -29,7 +29,7 @@ const clusters = {
 }
 
 class Endpoint {
-    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}) {
+    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}, configuredReportings=[]) {
         this.deviceIeeeAddress = deviceIeeeAddress;
         this.clusterValues = clusterValues;
         this.ID = ID;
@@ -45,6 +45,7 @@ class Endpoint {
         this.configureReporting = jest.fn();
         this.meta = {};
         this.binds = binds;
+        this.configuredReportings = configuredReportings;
         this.getInputClusters = () => inputClusters.map((c) => {
             return {ID: c, name: getKeyByValue(clusters, c)};
         }).filter((c) => c.name);
@@ -121,22 +122,22 @@ const returnDevices = [];
 
 const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
 const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020", false, 'Philips', '2019.09', '5.127.1.26581');
-const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
+const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 
 const devices = {
     'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
-    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
+    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}}, [{cluster: {name: 'genOnOff'}, attribute: {name: 'onOff'}, minimumReportInterval: 1, maximumReportInterval: 10, reportableChange: 20}])], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
     'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: new Group(1, []), cluster: {ID: 6, name: 'genOnOff'}}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
-    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID", false, "notSupportedMfg"),
     'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
     'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
     'notInSettings': new Device('EndDevice', '0x0017880104e45519', 6537, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
     'WXKG11LM': new Device('EndDevice', '0x0017880104e45520', 6537,4151, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
-    'WXKG02LM': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
+    'WXKG02LM_rev1': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
     'WSDCGQ11LM': new Device('EndDevice', '0x0017880104e45522', 6539,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.weather"),
     'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
     'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug"),
@@ -168,6 +169,8 @@ const devices = {
     '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
     'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
     'TS0601_thermostat': TS0601_thermostat,
+    'external_converter_device': new Device('EndDevice', '0x0017880104e45511', 1114, 'external', [], false, null, 'external_converter_device' ),
+    'QS_Zigbee_D02_TRIAC_2C_LN':new Device('Router', '0x0017882194e45543', 6549,4151, [new Endpoint(1, [0], [], '0x0017882194e45543'), new Endpoint(2, [0, 6], [], '0x0017882194e45543')], true, "Mains (single phase)", 'TS110F', false, '_TYZB01_v8gtiaed'),
 }
 
 const groups = {
@@ -177,6 +180,7 @@ const groups = {
     'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
     'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
     'group_with_switch': new Group(14, [devices.ZNCZ02LM.endpoints[0]]),
+    'default_bind_group': new Group(901, []),
 }
 
 const mock = {

test/utils.test.js

@@ -1,5 +1,6 @@
 const utils = require('../lib/util/utils.js');
 const version = require('../package.json').version;
+const versionHerdsman = require('../node_modules/zigbee-herdsman/package.json').version;
 
 describe('Utils', () => {
     describe('Is xiaomi device', () => {
@@ -41,6 +42,11 @@ describe('Utils', () => {
         expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
     })
 
+    it('Check dependency version', async () => {
+        var dependency = 'zigbee-herdsman';
+        expect(await utils.getDependencyVersion(dependency)).toStrictEqual({"version": versionHerdsman});
+    })
+    
     it('To local iso string', async () => {
         var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
         var getTimezoneOffset = Date.prototype.getTimezoneOffset;