1.5.0

* Support for PP-WHT-US new sensors

* Update homeassistant.js

.eslintrc.json

@@ -12,7 +12,8 @@
     "rules": {
         "require-jsdoc": "off",
         "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }]
+        "max-len": ["error", { "code": 120 }],
+        "no-prototype-builtins": "off"
     },
     "plugins": [
         "jest"

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [koenkk]
+custom: https://www.paypal.me/koenkk

.gitignore

@@ -63,4 +63,6 @@ data/config.json
 data/log*.txt
 data/state.json
 data/log
-data-backup/
+data-backup/
+data/coordinator_backup.json
+data/.storage

docker/run.sh

@@ -13,8 +13,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-if [ ! -z "$DEBUG" ]; then
-    npm start
-else
-    DEBUG="$DEBUG" npm start
-fi
+exec npm start

index.js

@@ -13,6 +13,11 @@ controller.start();
 process.on('SIGINT', handleQuit);
 process.on('SIGTERM', handleQuit);
 
+let stopping = false;
+
 function handleQuit() {
-    controller.stop(() => process.exit());
+    if (!stopping) {
+        stopping = true;
+        controller.stop(() => process.exit());
+    }
 }

lib/controller.js

@@ -22,15 +22,13 @@ const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
 const ExtensionDeviceBind = require('./extension/deviceBind');
 const ExtensionDeviceReport = require('./extension/deviceReport');
 const ExtensionLivolo = require('./extension/livolo');
+const ExtensionResponder = require('./extension/responder');
 
 class Controller {
     constructor() {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
-
-        if (settings.get().advanced.cache_state) {
-            this.state = new State();
-        }
+        this.state = new State();
 
         // Bind methods
         this.onMQTTConnected = this.onMQTTConnected.bind(this);
@@ -49,7 +47,7 @@ class Controller {
             new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState),
             new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState),
             new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-
+            new ExtensionResponder(this.zigbee, this.mqtt, this.state, this.publishEntityState),
         ];
 
         if (settings.get().advanced.report) {
@@ -159,9 +157,11 @@ class Controller {
     }
 
     start() {
-        if (this.state) {
-            this.state.start();
-        }
+        logger.info(`Logging to directory: '${logger.directory}'`);
+
+        logger.cleanup();
+
+        this.state.start();
 
         this.startupLogVersion(() => {
             this.zigbee.start(this.onZigbeeMessage, (error) => {
@@ -181,9 +181,7 @@ class Controller {
         this.extensions.filter((e) => e.stop).forEach((e) => e.stop());
 
         // Wrap-up
-        if (this.state) {
-            this.state.stop();
-        }
+        this.state.stop();
 
         this.mqtt.disconnect();
         this.zigbee.stop(callback);
@@ -210,7 +208,7 @@ class Controller {
         }
 
         if (device.type) {
-            type = device.type;
+            type = utils.correctDeviceType(device);
         }
 
         return `${friendlyName} (${device.ieeeAddr}): ${friendlyDevice.model} - ` +
@@ -219,7 +217,7 @@ class Controller {
 
     sendAllCachedStates() {
         this.zigbee.getAllClients().forEach((device) => {
-            if (this.state && this.state.exists(device.ieeeAddr)) {
+            if (this.state.exists(device.ieeeAddr)) {
                 this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
             }
         });
@@ -230,14 +228,15 @@ class Controller {
         const appSettings = settings.get();
         let messagePayload = {...payload};
 
-        if (this.state) {
-            // Add cached state to payload
-            if (this.state.exists(entityID)) {
-                messagePayload = objectAssignDeep.noMutate(this.state.get(entityID), payload);
-            }
+        const currentState = this.state.exists(entityID) ? this.state.get(entityID) : {};
+        const newState = objectAssignDeep.noMutate(currentState, payload);
 
-            // Update state cache with new state.
-            this.state.set(entityID, messagePayload);
+        // Update state cache with new state.
+        this.state.set(entityID, newState);
+
+        if (settings.get().advanced.cache_state) {
+            // Add cached state to payload
+            messagePayload = newState;
         }
 
         const entitySettings = entity.type === 'device' ? settings.getDevice(entityID) : settings.getGroup(entityID);
@@ -250,22 +249,41 @@ class Controller {
             messagePayload.device = this.getDeviceInfoForMqtt(entityID);
         }
 
-        if (settings.get().experimental.output === 'json') {
-            this.mqtt.publish(entity.friendlyName, JSON.stringify(messagePayload), options);
-        } else if (settings.get().experimental.output === 'attribute') {
-            Object.keys(messagePayload).forEach((key) => {
-                if (typeof messagePayload[key] == 'object') {
-                    Object.keys(messagePayload[key]).forEach((subKey) => {
-                        this.mqtt.publish(`${entity.friendlyName}/${key}-${subKey}`,
-                            JSON.stringify(messagePayload[key][subKey]), options);
-                    });
-                } else {
-                    this.mqtt.publish(`${entity.friendlyName}/${key}`, JSON.stringify(messagePayload[key]), options);
-                }
-            });
+        if (Object.entries(messagePayload).length) {
+            if (settings.get().experimental.output === 'json') {
+                this.mqtt.publish(entity.friendlyName, JSON.stringify(messagePayload), options);
+            } else if (settings.get().experimental.output === 'attribute') {
+                this.iteratePayloadForAttrOutput(entity.friendlyName+'/', messagePayload, options);
+            }
         }
     }
 
+    iteratePayloadForAttrOutput(topicRoot, payload, options) {
+        Object.keys(payload).forEach((key) => {
+            let subPayload = payload[key];
+            let message;
+
+            // Special cases
+            if (key === 'color' &&
+                subPayload.r !== undefined &&
+                subPayload.g !== undefined &&
+                subPayload.b !== undefined) {
+                subPayload = [subPayload.r, subPayload.g, subPayload.b];
+            }
+
+            // Check Array first, since it is also an Object
+            if (Array.isArray(subPayload)) {
+                message = subPayload.map((x) => `${x}`).join(',');
+            } else if (typeof subPayload === 'object') {
+                return this.iteratePayloadForAttrOutput(topicRoot+key+'-', subPayload, options);
+            } else {
+                message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
+            }
+
+            this.mqtt.publish(`${topicRoot}${key}`, message, options);
+        });
+    }
+
     getDeviceInfoForMqtt(ieeeAddr) {
         const device = this.zigbee.getDevice(ieeeAddr);
         const {

lib/extension/bridgeConfig.js

@@ -27,6 +27,7 @@ class BridgeConfig {
         this.deviceOptions = this.deviceOptions.bind(this);
         this.addGroup = this.addGroup.bind(this);
         this.removeGroup = this.removeGroup.bind(this);
+        this.whitelist= this.whitelist.bind(this);
 
         // Set supported options
         this.supportedOptions = {
@@ -44,9 +45,19 @@ class BridgeConfig {
             'device_options': this.deviceOptions,
             'add_group': this.addGroup,
             'remove_group': this.removeGroup,
+            'whitelist': this.whitelist,
         };
     }
 
+    whitelist(topic, message) {
+        message = message.toString();
+        const IDByFriendlyName = settings.getIeeeAddrByFriendlyName(message);
+        const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
+        const whitelisted = settings.whitelistDevice(deviceID);
+        whitelisted ? logger.info(`Whitelisted '${deviceID}'`) : logger.error(`Failed to whitelist '${deviceID}'`);
+        this.mqtt.log('device_whitelisted', deviceID);
+    }
+
     deviceOptions(topic, message) {
         let json = null;
         try {
@@ -218,9 +229,7 @@ class BridgeConfig {
             settings.removeDevice(deviceID);
 
             // Remove from state
-            if (this.state) {
-                this.state.remove(deviceID);
-            }
+            this.state.remove(deviceID);
 
             logger.info(`Successfully ${ban ? 'banned' : 'removed'} ${deviceID}`);
             this.mqtt.log(ban ? 'device_banned' : 'device_removed', message);
@@ -267,7 +276,7 @@ class BridgeConfig {
             const payload = {
                 version: info.version,
                 commit: info.commitHash,
-                coordinator_firmware: this.zigbee.getFirmwareVersion(),
+                coordinator: this.zigbee.getFirmwareVersion(),
                 log_level: logger.transports.console.level,
                 permit_join: this.zigbee.getPermitJoin(),
             };

lib/extension/deviceAvailability.js

@@ -30,15 +30,21 @@ class DeviceAvailability {
     }
 
     isPingable(device) {
+        logger.trace(`Checking if ${device.ieeeAddr} is pingable`);
+
         if (this.blacklist.includes(device.ieeeAddr)) {
+            logger.trace(`${device.ieeeAddr} is not pingable because of blacklist`);
             return false;
         }
 
         if (pingableDevices.find((d) => d.zigbeeModel.includes(device.modelId))) {
+            logger.trace(`${device.ieeeAddr} is pingable because in pingable devices`);
             return true;
         }
 
-        return utils.isRouter(device) && !utils.isBatteryPowered(device);
+        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
+        logger.trace(`${device.ieeeAddr} is pingable (${result}) not router or battery powered`);
+        return result;
     }
 
     getAllPingableDevices() {
@@ -84,13 +90,13 @@ class DeviceAvailability {
         }, mechanism);
     }
 
-    setTimer(ieeeAddr) {
-        if (this.timers[ieeeAddr]) {
-            clearTimeout(this.timers[ieeeAddr]);
+    setTimer(device) {
+        if (this.timers[device.ieeeAddr]) {
+            clearTimeout(this.timers[device.ieeeAddr]);
         }
 
-        this.timers[ieeeAddr] = setTimeout(() => {
-            this.handleInterval(ieeeAddr);
+        this.timers[device.ieeeAddr] = setTimeout(() => {
+            this.handleInterval(device);
         }, utils.secondsToMilliseconds(this.availability_timeout));
     }
 
@@ -114,12 +120,14 @@ class DeviceAvailability {
             });
 
             converters.forEach((converter) => {
-                const converted = converter.convert(null, null, null, 'get');
-                if (converted) {
-                    this.zigbee.publish(
-                        ieeeAddr, 'device', converted.cid, converted.cmd, converted.cmdType,
-                        converted.zclData, converted.cfg, null, () => {}
-                    );
+                const convertedResults = converter.convert(null, null, null, 'get');
+                if (convertedResults && convertedResults.length) {
+                    convertedResults.forEach((converted) => {
+                        this.zigbee.publish(
+                            ieeeAddr, 'device', converted.cid, converted.cmd, converted.cmdType,
+                            converted.zclData, converted.cfg, null, () => {}
+                        );
+                    });
                 }
             });
         }
@@ -139,18 +147,34 @@ class DeviceAvailability {
     }
 
     onZigbeeMessage(message, device, mappedDevice) {
-        // When a zigbee message from a device is received we know the device is still alive.
-        // => reset the timer.
         if (device && this.isPingable(this.zigbee.getDevice(device.ieeeAddr))) {
-            // When a message is received and the device is marked as offline, mark it online.
-            if (this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr]) {
-                this.publishAvailability(device.ieeeAddr, true);
-            } else if (!this.state.hasOwnProperty(device.ieeeAddr)) {
+            // When a zigbee message from a device is received we know the device is still alive.
+            // => reset the timer.
+            this.setTimer(device);
+
+            const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
+            const offline = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr];
+
+            if (!online && !offline) {
                 // A new device has been connected
                 this.publishAvailability(device.ieeeAddr, true);
+            } else if (offline) {
+                // When a message is received and the device is marked as offline, mark it online.
+                this.publishAvailability(device.ieeeAddr, true);
+            } else if (online) {
+                /**
+                 * In case the device is powered off AND on within the availability timeout,
+                 * zigbee2qmtt does not detect the device as offline (device is still marked online).
+                 * When a device is turned on again the state could be out of sync.
+                 * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
+                 * endDeviceAnnce is typically send when a device comes online.
+                 *
+                 * This isn't needed for TRADFRI devices as they already send the state themself.
+                 */
+                if (message.type === 'endDeviceAnnce' && !utils.isIkeaTradfriDevice(device)) {
+                    this.onReconnect(device.ieeeAddr);
+                }
             }
-
-            this.setTimer(device);
         }
     }
 }

lib/extension/deviceBind.js

@@ -96,18 +96,21 @@ class DeviceBind {
 
         // Bind
         clusters.forEach((cluster) => {
-            logger.debug(`${topic.type}ing cluster '${cluster}' from ${sourceEntity.ID}' to '${targetEntity.ID}'`);
+            logger.debug(
+                `${topic.type}ing cluster '${cluster}' from '${sourceEntity.ID}' ` +
+                `(${source.epId}) to '${targetEntity.ID}'`
+            );
 
             this.zigbee[topic.type](source, cluster, target, (error) => {
                 if (error) {
                     logger.error(
-                        `Failed to ${topic.type} cluster '${cluster}' from ${sourceEntity.ID}' to ` +
+                        `Failed to ${topic.type} cluster '${cluster}' from '${sourceEntity.ID}' to ` +
                         `'${targetEntity.ID}' (${error})`
                     );
                 } else {
                     logger.info(
                         `Successfully ${topic.type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `${sourceEntity.ID}' to '${targetEntity.ID}'`
+                        `'${sourceEntity.ID}' to '${targetEntity.ID}'`
                     );
 
                     this.mqtt.log(

lib/extension/devicePublish.js

@@ -4,8 +4,8 @@ const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/.+/(set|get)$`);
 const postfixes = utils.getPostfixes();
+const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
 
 const maxDepth = 20;
 
@@ -13,6 +13,7 @@ const groupConverters = [
     zigbeeShepherdConverters.toZigbeeConverters.light_onoff_brightness,
     zigbeeShepherdConverters.toZigbeeConverters.light_colortemp,
     zigbeeShepherdConverters.toZigbeeConverters.light_color,
+    zigbeeShepherdConverters.toZigbeeConverters.light_alert,
     zigbeeShepherdConverters.toZigbeeConverters.ignore_transition,
 ];
 
@@ -26,41 +27,29 @@ class DevicePublish {
 
     onMQTTConnected() {
         // Subscribe to topics.
+        const baseTopic = settings.get().mqtt.base_topic;
         for (let step = 1; step < maxDepth; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/${'+/'.repeat(step)}`;
+            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
             this.mqtt.subscribe(`${topic}set`);
+            this.mqtt.subscribe(`${topic}set/+`);
             this.mqtt.subscribe(`${topic}get`);
+            this.mqtt.subscribe(`${topic}get/+`);
         }
     }
 
     parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
+        const match = topic.match(topicRegex);
+        if (!match) {
             return null;
         }
 
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/`, '');
-        if (topic.match(/bridge\/config/)) {
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't repalce base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge\/config/)) {
             return null;
         }
-        // Parse type from topic
-        const type = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
 
-        // Remove type from topic
-        topic = topic.replace(`/${type}`, '');
-
-        // Check if we have to deal with a postfix.
-        let postfix = '';
-        if (postfixes.find((p) => topic.endsWith(`/${p}`))) {
-            postfix = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-            // Remove postfix from topic
-            topic = topic.replace(`/${postfix}`, '');
-        }
-
-        const ID = topic;
-
-        return {type, ID, postfix};
+        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
     }
 
     handlePublishError(entity, message, error) {
@@ -81,6 +70,10 @@ class DevicePublish {
                 delete msg.state;
             }
 
+            if (settings.get().advanced.last_seen !== 'disable') {
+                msg.last_seen = utils.formatDate(Date.now(), settings.get().advanced.last_seen);
+            }
+
             this.publishEntityState(entity.ID, msg);
         }
     }
@@ -99,11 +92,12 @@ class DevicePublish {
         let endpoint = null;
         let converters = null;
         let device = null;
+        let options = {};
 
         if (entity.type === 'device') {
             device = this.zigbee.getDevice(entity.ID);
             if (!device) {
-                logger.error(`Failed to find device with ieeAddr: '${entity.ID}'`);
+                logger.error(`Failed to find device with ieeeAddr: '${entity.ID}'`);
                 this.mqtt.log('entity_not_found', entity.ID);
                 return;
             }
@@ -126,17 +120,22 @@ class DevicePublish {
             }
 
             converters = model.toZigbee;
+            options = model.options || {};
         } else if (entity.type === 'group') {
             converters = groupConverters;
         }
 
         // Convert the MQTT message to a Zigbee message.
-        let json = null;
-        try {
-            json = JSON.parse(message);
-        } catch (e) {
-            // Cannot be parsed to JSON, assume state message.
-            json = {state: message.toString()};
+        let json = {};
+        if (topic.hasOwnProperty('attribute') && topic.attribute) {
+            json[topic.attribute] = message.toString();
+        } else {
+            try {
+                json = JSON.parse(message);
+            } catch (e) {
+                // Cannot be parsed to JSON, assume state message.
+                json = {state: message.toString()};
+            }
         }
 
         /**
@@ -144,7 +143,7 @@ class DevicePublish {
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        if (settings.get().homeassistant && this.state) {
+        if (settings.get().homeassistant) {
             const deviceState = this.state.get(entity.ID);
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
@@ -176,49 +175,53 @@ class DevicePublish {
             }
 
             // Converter didn't return a result, skip
-            const converted = converter.convert(key, json[key], json, topic.type, topic.postfix);
-            if (!converted) {
+            const convertedResults = converter.convert(key, json[key], json, topic.type, topic.postfix, options);
+            if (!convertedResults || !convertedResults.length) {
                 return;
             }
 
-            this.zigbee.publish(
-                entity.ID,
-                entity.type,
-                converted.cid,
-                converted.cmd,
-                converted.cmdType,
-                converted.zclData,
-                converted.cfg,
-                endpoint,
-                (error, rsp) => {
-                    if (!error) {
-                        this.handlePublished(entity, topic, converter, converted, key, json[key]);
-                    } else {
-                        this.handlePublishError(entity, message, error);
+            convertedResults.forEach((converted) => {
+                this.zigbee.publish(
+                    entity.ID,
+                    entity.type,
+                    converted.cid,
+                    converted.cmd,
+                    converted.cmdType,
+                    converted.zclData,
+                    converted.cfg,
+                    endpoint,
+                    (error, rsp) => {
+                        if (!error) {
+                            this.handlePublished(entity, topic, converter, converted, key, json[key]);
+                        } else {
+                            this.handlePublishError(entity, message, error);
+                        }
                     }
+                );
+
+                // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+                // So here we re-read the value after a specified timeout, this timeout could for example be the
+                // transition time of a color change or for forcing a state read for devices that don't
+                // automatically report a new state when set.
+                // When reporting is requested for a device (report: true in device-specific settings) we won't
+                // ever issue a read here, as we assume the device will properly report changes.
+                // Only do this when the retrieve_state option is enabled for this device.
+                const deviceSettings = settings.getDevice(entity.ID);
+                if (topic.type === 'set' && entity.type === 'device' &&
+                    converted.hasOwnProperty('readAfterWriteTime') && deviceSettings && deviceSettings.retrieve_state) {
+                    const getConvertedResults = converter.convert(key, json[key], json, 'get', '', options);
+                    getConvertedResults.forEach((getConverted) => {
+                        setTimeout(() => {
+                            this.zigbee.publish(
+                                entity.ID, entity.type, getConverted.cid, getConverted.cmd, getConverted.cmdType,
+                                getConverted.zclData, getConverted.cfg, endpoint, () => {}
+                            );
+                        }, converted.readAfterWriteTime);
+                    });
                 }
-            );
 
-            // It's possible for devices to get out of sync when writing an attribute that's not reportable.
-            // So here we re-read the value after a specified timeout, this timeout could for example be the
-            // transition time of a color change or for forcing a state read for devices that don't
-            // automatically report a new state when set.
-            // When reporting is requested for a device (report: true in device-specific settings) we won't
-            // ever issue a read here, as we assume the device will properly report changes.
-            // Only do this when the retrieve_state option is enabled for this device.
-            const deviceSettings = settings.getDevice(entity.ID);
-            if (topic.type === 'set' && entity.type === 'device' && converted.hasOwnProperty('readAfterWriteTime') &&
-                deviceSettings && deviceSettings.retrieve_state) {
-                const getConverted = converter.convert(key, json[key], json, 'get');
-                setTimeout(() => {
-                    this.zigbee.publish(
-                        entity.ID, entity.type, getConverted.cid, getConverted.cmd, getConverted.cmdType,
-                        getConverted.zclData, getConverted.cfg, endpoint, () => {}
-                    );
-                }, converted.readAfterWriteTime);
-            }
-
-            usedConverters.push(converter);
+                usedConverters.push(converter);
+            });
         });
 
         return true;

lib/extension/deviceReceive.js

@@ -158,16 +158,8 @@ class DeviceReceive {
 
             // Add last seen timestamp
             const now = Date.now();
-            switch (settings.get().advanced.last_seen) {
-            case 'ISO_8601':
-                payload.last_seen = new Date(now).toISOString();
-                break;
-            case 'ISO_8601_local':
-                payload.last_seen = utils.toLocalISOString(new Date(now));
-                break;
-            case 'epoch':
-                payload.last_seen = now;
-                break;
+            if (settings.get().advanced.last_seen !== 'disable') {
+                payload.last_seen = utils.formatDate(now, settings.get().advanced.last_seen);
             }
 
             if (settings.get().advanced.elapsed) {
@@ -183,6 +175,21 @@ class DeviceReceive {
                 this.publishDebounce(device.ieeeAddr, payload, settingsDevice.debounce);
             } else {
                 this.publishEntityState(device.ieeeAddr, payload);
+
+                if (settings.get().homeassistant) {
+                    /**
+                     * Publish an empty value for click and action payload, in this way Home Assistant
+                     * can use Home Assistant entities in automations.
+                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
+                     */
+                    Object.keys(payload).forEach((key) => {
+                        if (['action', 'click'].includes(key)) {
+                            const counterPayload = {};
+                            counterPayload[key] = '';
+                            this.publishEntityState(device.ieeeAddr, counterPayload);
+                        }
+                    });
+                }
             }
         };
 

lib/extension/deviceReport.js

@@ -1,6 +1,7 @@
 const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
 const logger = require('../util/logger');
 const CC2530Router = zigbeeShepherdConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
+const utils = require('../util/utils');
 
 const candidates = {
     'genOnOff': {
@@ -82,14 +83,20 @@ class DeviceReport {
         });
     }
 
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => {
-            const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
+    shouldSetupReporting(device) {
+        return utils.isRouter(device) && !utils.isBatteryPowered(device);
+    }
 
-            if (mappedDevice) {
-                this.setupReporting(mappedDevice, device);
-            }
-        });
+    onZigbeeStarted() {
+        this.zigbee.getAllClients()
+            .filter((d) => this.shouldSetupReporting(d))
+            .forEach((device) => {
+                const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
+
+                if (mappedDevice) {
+                    this.setupReporting(mappedDevice, device);
+                }
+            });
     }
 
     onZigbeeMessage(message, device, mappedDevice) {
@@ -98,7 +105,8 @@ class DeviceReport {
         // Ikea TRADFRI tend to forget their reporting after powered off.
         // Re-setup reporting.
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (device && mappedDevice && ['endDeviceAnnce', 'devIncoming'].includes(message.type)) {
+        if (device && mappedDevice && ['endDeviceAnnce', 'devIncoming'].includes(message.type) &&
+            this.shouldSetupReporting(device)) {
             this.setupReporting(mappedDevice, device);
         }
     }

lib/extension/groups.js

@@ -1,7 +1,12 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
+const data = require('../util/data');
+const utils = require('../util/utils');
+const fs = require('fs');
+const diff = require('deep-diff');
 
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/.+/(remove|add|remove_all)$`);
+const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(remove|add|remove_all)$`);
 
 class Groups {
     constructor(zigbee, mqtt, state, publishEntityState) {
@@ -9,16 +14,112 @@ class Groups {
         this.mqtt = mqtt;
         this.state = state;
         this.publishEntityState = publishEntityState;
+        this.onStateChange = this.onStateChange.bind(this);
+
+        this.groupsCacheFile = data.joinPathStorage('groups_cache.json');
+        this.groupsCache = this.readGroupsCache();
+    }
+
+    readGroupsCache() {
+        return fs.existsSync(this.groupsCacheFile) ? JSON.parse(fs.readFileSync(this.groupsCacheFile, 'utf8')) : {};
+    }
+
+    writeGroupsCache() {
+        fs.writeFileSync(this.groupsCacheFile, JSON.stringify(this.groupsCache), 'utf8');
     }
 
     onMQTTConnected() {
         this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove`);
         this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/add`);
         this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove_all`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
+    }
+
+    apply(from, to) {
+        const sortGroups = (obj) => Object.keys(obj).forEach((key) => obj[key] = obj[key].sort());
+
+        sortGroups(from);
+        sortGroups(to);
+
+        const differences = diff(from, to);
+        if (differences) {
+            differences.forEach((diff) => {
+                const groupID = diff.path[0];
+
+                if (diff.kind === 'N') {
+                    diff.rhs.forEach((ID) => this.updateDeviceGroup(ID, 'add', groupID));
+                } else if (diff.kind === 'A') {
+                    if (diff.item.lhs) {
+                        this.updateDeviceGroup(diff.item.lhs, 'remove', groupID);
+                    } else {
+                        this.updateDeviceGroup(diff.item.rhs, 'add', groupID);
+                    }
+                } else if (diff.kind === 'D') {
+                    diff.lhs.forEach((ID) => this.updateDeviceGroup(ID, 'remove', groupID));
+                } else if (diff.kind === 'E') {
+                    this.updateDeviceGroup(diff.rhs, 'add', groupID);
+                    this.updateDeviceGroup(diff.lhs, 'remove', groupID);
+                }
+            });
+        }
+    }
+
+    parseID(ID) {
+        let entityID = ID;
+        let endpointID = null;
+        const postfix = utils.getPostfixes().find((p) => entityID.endsWith(`/${p}`));
+        if (postfix) {
+            // Found a postfix, retrieve the endpoint which correspodns to the postfix
+            entityID = entityID.substring(0, entityID.length - (postfix.length + 1));
+            const endpoint = utils.getEndpointByEntityID(this.zigbee, entityID, postfix);
+
+            if (!endpoint) {
+                return;
+            }
+
+            endpointID = endpoint.epId;
+        }
+
+        return {endpointID, entityID};
+    }
+
+    getGroupsOfDevice(entityID) {
+        return Object.keys(settings.getGroups()).filter((groupID) => {
+            return settings.getGroup(groupID).devices.includes(entityID);
+        });
+    }
+
+    onStateChange(ieeeAddr, from, to) {
+        const properties = ['state', 'brightness', 'color_temp', 'color'];
+        const payload = {};
+
+        properties.forEach((prop) => {
+            if (to.hasOwnProperty(prop) && (!from || from[prop] != to[prop])) {
+                payload[prop] = to[prop];
+            }
+        });
+
+        if (Object.keys(payload)) {
+            const groups = this.getGroupsOfDevice(ieeeAddr);
+            groups.forEach((groupID) => {
+                this.publishEntityState(groupID, payload);
+            });
+        }
+    }
+
+    onZigbeeStarted() {
+        this.state.registerOnStateChangeListener(this.onStateChange);
+
+        const settingsGroups = {};
+        Object.keys(settings.getGroups()).forEach((groupID) => {
+            settingsGroups[groupID] = settings.getGroup(groupID).devices;
+        });
+
+        this.apply(this.groupsCache, settingsGroups);
     }
 
     parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
+        if (!topic.match(topicRegex) && !topic.match(topicRegexRemoveAll)) {
             return null;
         }
 
@@ -31,7 +132,77 @@ class Groups {
         // Remove type from topic
         topic = topic.replace(`/${type}`, '');
 
-        return {friendly_name: topic, type};
+        return {friendly_name: type === 'remove_all' ? null : topic, type};
+    }
+
+    updateDeviceGroup(ID, cmd, groupID) {
+        let payload = null;
+        const orignalCmd = cmd;
+        if (cmd === 'add') {
+            payload = {groupid: groupID, groupname: ''};
+            cmd = 'add';
+        } else if (cmd === 'remove') {
+            payload = {groupid: groupID};
+            cmd = 'remove';
+        } else if (cmd === 'remove_all') {
+            payload = {};
+            cmd = 'removeAll';
+        }
+
+        const {entityID, endpointID} = this.parseID(ID);
+        const ieeeAddr = settings.resolveEntity(entityID).ID;
+
+        const cb = (error, rsp) => {
+            if (error) {
+                logger.error(`Failed to ${cmd} ${ieeeAddr} from ${groupID}`);
+            } else {
+                logger.info(`Successfully ${cmd} ${ieeeAddr} to ${groupID}`);
+
+                // Log to MQTT
+                this.mqtt.log({
+                    device: settings.getDevice(ieeeAddr).friendly_name,
+                    group: groupID,
+                    action: orignalCmd,
+                });
+
+                // Update group cache
+                if (cmd === 'add') {
+                    if (!this.groupsCache[groupID]) {
+                        this.groupsCache[groupID] = [];
+                    }
+
+                    if (!this.groupsCache[groupID].includes(ieeeAddr)) {
+                        this.groupsCache[groupID].push(ieeeAddr);
+                    }
+                } else if (cmd === 'remove') {
+                    if (this.groupsCache[groupID]) {
+                        this.groupsCache[groupID] = this.groupsCache[groupID].filter((device) => device != ieeeAddr);
+                    }
+                } else if (cmd === 'removeAll') {
+                    Object.keys(this.groupsCache).forEach((groupID_) => {
+                        this.groupsCache[groupID_] = this.groupsCache[groupID_].filter((device) => device != ieeeAddr);
+                    });
+                }
+
+                this.writeGroupsCache();
+
+                // Update settings
+                if (cmd === 'add') {
+                    settings.addDeviceToGroup(groupID, ieeeAddr);
+                } else if (cmd === 'remove') {
+                    settings.removeDeviceFromGroup(groupID, ieeeAddr);
+                } else if (cmd === 'removeAll') {
+                    Object.keys(settings.get().groups).forEach((groupID_) => {
+                        settings.removeDeviceFromGroup(groupID_, ieeeAddr);
+                    });
+                }
+            }
+        };
+
+        this.zigbee.publish(
+            ieeeAddr, 'device', 'genGroups', cmd, 'functional',
+            payload, null, endpointID, cb,
+        );
     }
 
     onMQTTMessage(topic, message) {
@@ -42,59 +213,19 @@ class Groups {
         }
 
         // Find ID of this group.
-        const groupID = settings.getGroupIDByFriendlyName(topic.friendly_name);
-        if (!groupID) {
-            logger.error(`Group with friendly_name '${topic.friendly_name}' doesn't exist`);
-            return;
-        }
+        let groupID = null;
+        if (topic.type !== 'remove_all') {
+            groupID = settings.getGroupIDByFriendlyName(topic.friendly_name);
+            if (!groupID) {
+                logger.error(`Group with friendly_name '${topic.friendly_name}' doesn't exist`);
+                return;
+            }
 
-        if (groupID === 99) {
-            logger.error('Group 99 is reserved, please use a different groupID');
-            return;
-        }
-
-        // Map message to ieeeAddr and check if device exist.
-        message = message.toString();
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(message) || message;
-        if (!this.zigbee.getDevice(ieeeAddr)) {
-            logger.error(`Failed to find device '${message}'`);
-            return;
+            groupID = groupID.toString();
         }
 
         // Send command to the device.
-        let payload = null;
-        let cmd = null;
-        if (topic.type === 'add') {
-            payload = {groupid: groupID, groupname: ''};
-            cmd = 'add';
-        } else if (topic.type === 'remove') {
-            payload = {groupid: groupID};
-            cmd = 'remove';
-        } else if (topic.type === 'remove_all') {
-            payload = {};
-            cmd = 'removeAll';
-        }
-
-        const callback = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to ${topic.type} ${ieeeAddr} from ${topic.friendly_name}`);
-            } else {
-                logger.info(`Successfully ${topic.type} ${ieeeAddr} to ${topic.friendly_name}`);
-
-                // Log to MQTT
-                const log = {device: message};
-                if (['remove', 'add'].includes(topic.type)) {
-                    log.group = topic.friendly_name;
-                }
-
-                this.mqtt.log(log);
-            }
-        };
-
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', cmd, 'functional',
-            payload, null, null, callback,
-        );
+        this.updateDeviceGroup(message.toString(), topic.type, groupID);
 
         return true;
     }

lib/extension/homeassistant.js

@@ -95,6 +95,26 @@ const configurations = {
             device_class: 'battery',
         },
     },
+    'binary_sensor_lock': {
+        type: 'binary_sensor',
+        object_id: 'lock',
+        discovery_payload: {
+            payload_on: 'UNLOCK',
+            payload_off: 'LOCK',
+            value_template: '{{ value_json.state}}',
+            device_class: 'lock',
+        },
+    },
+    'binary_sensor_lock_reverse': {
+        type: 'binary_sensor',
+        object_id: 'lock_reverse',
+        discovery_payload: {
+            payload_on: 'UNLOCK',
+            payload_off: 'LOCK',
+            value_template: '{{ value_json.reverse}}',
+            device_class: 'lock',
+        },
+    },
 
     // Sensor
     'sensor_illuminance': {
@@ -139,25 +159,41 @@ const configurations = {
         discovery_payload: {
             icon: 'mdi:toggle-switch',
             value_template: '{{ value_json.click }}',
-            expire_after: 1,
         },
     },
     'sensor_power': {
         type: 'sensor',
         object_id: 'power',
         discovery_payload: {
-            unit_of_measurement: 'Watt',
-            icon: 'mdi:flash',
+            unit_of_measurement: 'W',
+            icon: 'mdi:factory',
             value_template: '{{ value_json.power }}',
         },
     },
+    'sensor_current': {
+        type: 'sensor',
+        object_id: 'current',
+        discovery_payload: {
+            unit_of_measurement: 'A',
+            icon: 'mdi:power-plug',
+            value_template: '{{ value_json.current }}',
+        },
+    },
+    'sensor_voltage': {
+        type: 'sensor',
+        object_id: 'voltage',
+        discovery_payload: {
+            unit_of_measurement: 'V',
+            icon: 'mdi:flash',
+            value_template: '{{ value_json.voltage }}',
+        },
+    },
     'sensor_action': {
         type: 'sensor',
         object_id: 'action',
         discovery_payload: {
             icon: 'mdi:gesture-double-tap',
             value_template: '{{ value_json.action }}',
-            expire_after: 1,
         },
     },
     'sensor_brightness': {
@@ -296,6 +332,42 @@ const configurations = {
             value_template: '{{ value_json.state }}',
         },
     },
+
+    // Thermostat/HVAC
+    'thermostat': {
+        type: 'climate',
+        object_id: 'climate',
+        discovery_payload: {
+            state_topic: false,
+            min_temp: 7,
+            max_temp: 30,
+            modes: ['off', 'auto', 'heat'],
+            mode_state_topic: true,
+            mode_state_template: '{{ value_json.system_mode }}',
+            mode_command_topic: true,
+            current_temperature_topic: true,
+            current_temperature_template: '{{ value_json.local_temperature }}',
+            temperature_state_topic: true,
+            temperature_state_template: '{{ value_json.occupied_heating_setpoint }}',
+            temperature_command_topic: true,
+        },
+    },
+
+    // Fan
+    'fan': {
+        type: 'fan',
+        object_id: 'fan',
+        discovery_payload: {
+            state_topic: true,
+            state_value_template: '{{ value_json.fan_state }}',
+            command_topic: true,
+            command_topic_postfix: 'fan_state',
+            speed_state_topic: true,
+            speed_command_topic: true,
+            speed_value_template: '{{ value_json.fan_mode }}',
+            speeds: ['off', 'low', 'medium', 'high', 'on', 'auto', 'smart'],
+        },
+    },
 };
 
 const switchWithPostfix = (postfix) => {
@@ -317,9 +389,10 @@ const mapping = {
     'WXKG01LM': [configurations.sensor_click, configurations.sensor_battery],
     'WXKG11LM': [configurations.sensor_click, configurations.sensor_battery],
     'WXKG12LM': [configurations.sensor_click, configurations.sensor_battery],
-    'WXKG03LM': [configurations.sensor_click, configurations.sensor_battery],
+    // BREAKING_IMPROVEMENT: only use sensor_click for WXKG03LM (action hold -> click hold)
+    'WXKG03LM': [configurations.sensor_click, configurations.sensor_battery, configurations.sensor_action],
     'WXKG02LM': [configurations.sensor_click, configurations.sensor_battery],
-    'QBKG04LM': [configurations.switch, configurations.sensor_click],
+    'QBKG04LM': [configurations.switch, configurations.sensor_click, configurations.sensor_action],
     'QBKG03LM': [switchWithPostfix('left'), switchWithPostfix('right'), configurations.sensor_click],
     'WSDCGQ01LM': [configurations.sensor_temperature, configurations.sensor_humidity, configurations.sensor_battery],
     'WSDCGQ11LM': [
@@ -345,9 +418,11 @@ const mapping = {
     'LED1536G5': [configurations.light_brightness_colortemp],
     '7299760PH': [configurations.light_brightness_colorxy],
     '7146060PH': [configurations.light_brightness_colortemp_colorxy],
+    '046677476816': [configurations.light_brightness],
     'F7C033': [configurations.light_brightness],
     'JTYJ-GD-01LM/BW': [configurations.binary_sensor_smoke, configurations.sensor_battery],
     'PLUG EDP RE:DY': [configurations.switch, configurations.sensor_power],
+    'SWITCH EDP RE:DY': [configurations.switch],
     'CC2530.ROUTER': [configurations.binary_sensor_router],
     'AA70155': [configurations.light_brightness_colortemp],
     '4058075816718': [configurations.light_brightness_colortemp_colorxy],
@@ -380,6 +455,7 @@ const mapping = {
     'RS 125': [configurations.light_brightness],
     'RS 225': [configurations.light_brightness],
     'RB 145': [configurations.light_brightness],
+    'RB 245': [configurations.light_brightness],
     'PL 110': [configurations.light_brightness],
     'ST 110': [configurations.light_brightness],
     'UC 110': [configurations.light_brightness],
@@ -437,6 +513,7 @@ const mapping = {
         configurations.binary_sensor_gas,
         configurations.sensor_gas_density,
     ],
+    '50043': [configurations.switch],
     '50045': [configurations.light_brightness],
     'AV2010/22': [configurations.binary_sensor_occupancy, configurations.sensor_battery],
     '3210-L': [configurations.switch],
@@ -449,7 +526,7 @@ const mapping = {
     '3261030P7': [configurations.light_brightness_colortemp],
     '3216431P5': [configurations.light_brightness_colortemp],
     'DJT11LM': [configurations.sensor_action, configurations.sensor_battery],
-    'E1603': [configurations.switch],
+    'E1603/E1702': [configurations.switch],
     '7199960PH': [configurations.light_brightness_colorxy],
     '74696': [configurations.light_brightness],
     'AB35996': [configurations.light_brightness_colortemp_colorxy],
@@ -480,8 +557,9 @@ const mapping = {
     'SP 120': [configurations.switch, configurations.sensor_power],
     'RB 248 T': [configurations.light_brightness_colortemp],
     'HS3SA': [configurations.binary_sensor_smoke, configurations.binary_sensor_battery_low],
-    'HS1DS': [configurations.binary_sensor_contact],
-    'HS1WL': [configurations.binary_sensor_water_leak],
+    'HS1DS/HS3DS': [configurations.binary_sensor_contact],
+    'HS1WL/HS3WL': [configurations.binary_sensor_water_leak],
+    'HS1-WL-E': [configurations.binary_sensor_water_leak],
     '421786': [configurations.light_brightness],
     'ICZB-IW11D': [configurations.light_brightness],
     'HGZB-01A': [configurations.light_brightness],
@@ -501,7 +579,7 @@ const mapping = {
     'GL-B-008Z': [configurations.light_brightness_colortemp_colorxy],
     'AV2010/25': [configurations.switch, configurations.sensor_power],
     'E12-N14': [configurations.light_brightness],
-    '1TST-EU': [],
+    '1TST-EU': [configurations.thermostat, configurations.sensor_battery],
     'RB 178 T': [configurations.light_brightness_colortemp],
     '45856GE': [configurations.switch],
     'GL-D-003Z': [configurations.light_brightness_colortemp_colorxy],
@@ -515,7 +593,7 @@ const mapping = {
     'E1524': [configurations.sensor_action],
     'GL-C-006/GL-C-009': [configurations.light_brightness_colortemp],
     '100.424.11': [configurations.light_brightness_colortemp],
-    'AC0251100NJ': [configurations.sensor_click, configurations.sensor_brightness, configurations.sensor_battery],
+    'AC0251100NJ': [configurations.sensor_action, configurations.sensor_battery],
     '71831': [configurations.light_brightness_colortemp],
     '404000/404005/404012': [configurations.light_brightness_colortemp_colorxy],
     '404006/404008/404004': [configurations.light_brightness_colortemp],
@@ -549,12 +627,17 @@ const mapping = {
         configurations.sensor_illuminance, configurations.sensor_battery,
     ],
     'HGZB-042': [switchWithPostfix('top'), switchWithPostfix('bottom')],
+    'HGZB-42': [switchWithPostfix('top'), switchWithPostfix('bottom')],
     'GL-FL-004TZ': [configurations.light_brightness_colortemp_colorxy],
     'IM6001-OTP05': [configurations.switch],
     'SV01': [
         configurations.cover_position, configurations.sensor_temperature, configurations.sensor_pressure,
         configurations.sensor_battery,
     ],
+    'SV02': [
+        configurations.cover_position, configurations.sensor_temperature, configurations.sensor_pressure,
+        configurations.sensor_battery,
+    ],
     '316GLEDRF': [configurations.light_brightness],
     'LVS-ZB500D': [configurations.light_brightness],
     'ST218': [],
@@ -575,6 +658,14 @@ const mapping = {
         configurations.binary_sensor_occupancy, configurations.sensor_temperature,
         configurations.sensor_battery, configurations.binary_sensor_battery_low,
     ],
+    'NCZ-3041-HA': [
+        configurations.binary_sensor_occupancy, configurations.sensor_temperature,
+        configurations.sensor_battery, configurations.binary_sensor_battery_low,
+    ],
+    'NCZ-3045-HA': [
+        configurations.binary_sensor_occupancy, configurations.sensor_temperature,
+        configurations.sensor_battery, configurations.binary_sensor_battery_low,
+    ],
     'STS-IRM-250': [
         configurations.binary_sensor_occupancy, configurations.sensor_temperature,
         configurations.sensor_battery, configurations.binary_sensor_battery_low,
@@ -589,7 +680,10 @@ const mapping = {
     ],
     'IM6001-BTP01': [configurations.sensor_click, configurations.sensor_temperature],
     'AV2010/34': [configurations.sensor_click],
-    'PP-WHT-US': [configurations.switch, configurations.sensor_power],
+    'PP-WHT-US': [
+        configurations.switch, configurations.sensor_power,
+        configurations.sensor_current, configurations.sensor_voltage,
+    ],
     'CR701-YZ': [
         configurations.binary_sensor_battery_low, configurations.binary_sensor_carbon_monoxide,
         configurations.binary_sensor_gas,
@@ -620,13 +714,14 @@ const mapping = {
     ],
     'U86K31ND6': [switchWithPostfix('left'), switchWithPostfix('right'), switchWithPostfix('center')],
     'HLD812-Z-SC': [configurations.light_brightness],
+    'HLC610-Z': [configurations.light_brightness],
     'BY 285 C': [configurations.light_brightness_colortemp_colorxy],
     'HS1RC-M': [configurations.sensor_action, configurations.sensor_battery],
     'SWO-WDS1PA': [configurations.binary_sensor_contact],
     'LLKZMK11LM': [switchWithPostfix('l1'), switchWithPostfix('l2')],
     'LVS-SM10ZW': [configurations.binary_sensor_contact, configurations.binary_sensor_battery_low],
     'HS2SK': [configurations.switch, configurations.sensor_power],
-    '45853GE': [configurations.switch],
+    '45853GE': [configurations.switch, configurations.sensor_power],
     '50064': [configurations.light_brightness_colortemp],
     '9290011998B': [configurations.light_brightness_colortemp],
     '4096730U7': [configurations.light_brightness_colortemp],
@@ -635,6 +730,76 @@ const mapping = {
         configurations.sensor_temperature, configurations.sensor_battery, configurations.binary_sensor_water_leak,
     ],
     'N2G-SP': [configurations.sensor_power, configurations.switch],
+    'AC0363900NJ': [configurations.light_brightness_colortemp_colorxy],
+    'LXZB-02A': [configurations.light_brightness],
+    'GL-S-004Z': [configurations.light_brightness_colortemp],
+    'SCM-5ZBS': [configurations.cover_position],
+    'YRD226HA2619': [configurations.sensor_battery, configurations.lock],
+    'YMF40': [configurations.lock],
+    'V3-BTZB': [configurations.lock],
+    '3RSS008Z': [configurations.switch, configurations.sensor_battery],
+    '99432': [configurations.fan, configurations.light_brightness],
+    '511.10': [configurations.light_brightness],
+    'IM6001-MPP01': [
+        configurations.sensor_temperature, configurations.binary_sensor_contact, configurations.sensor_battery,
+    ],
+    'HLC821-Z-SC': [configurations.light_brightness],
+    'RS 228 T': [configurations.light_brightness_colortemp],
+    '67200BL': [configurations.switch],
+    '2430-100': [configurations.sensor_action],
+    '100.425.90': [configurations.switch],
+    '74580': [configurations.light_brightness],
+    'HS1CA-E': [
+        configurations.binary_sensor_carbon_monoxide, configurations.binary_sensor_battery_low,
+        configurations.sensor_battery,
+    ],
+    'MCT-340 E': [configurations.binary_sensor_contact],
+    'D1542': [configurations.light_brightness_colortemp],
+    'ZGRC-KEY-013': [configurations.sensor_click],
+    'ZigUP': [configurations.switch],
+    'YRD256HA20BP': [configurations.sensor_battery, configurations.lock],
+    'SZ-ESW01-AU': [configurations.sensor_power, configurations.switch],
+    'PSM-29ZBSR': [configurations.switch],
+    'ZM350STW1TCF': [configurations.light_brightness_colortemp],
+    'M350STW1': [configurations.light_brightness],
+    'A806S-Q1R': [configurations.light_brightness],
+    'XY12S-15': [configurations.light_brightness_colortemp_colorxy],
+    'B07KG5KF5R': [configurations.light_brightness_colortemp],
+    'SCM-S1': [configurations.cover_position],
+    'HEIMAN-M1': [configurations.binary_sensor_contact],
+    '3216131P5': [configurations.light_brightness_colortemp],
+    'ST8AU-CON': [configurations.light_brightness],
+    'HS3MS': [configurations.binary_sensor_occupancy],
+    'DIYRUZ_R4_5': [
+        switchWithPostfix('bottom_left'), switchWithPostfix('bottom_right'), switchWithPostfix('center'),
+        switchWithPostfix('top_left'), switchWithPostfix('top_right'),
+    ],
+    'NCZ-3011-HA': [
+        configurations.binary_sensor_occupancy, configurations.sensor_humidity, configurations.sensor_temperature,
+    ],
+    'MEAZON_BIZY_PLUG': [configurations.sensor_power, configurations.switch, configurations.sensor_temperature],
+    'MEAZON_DINRAIL': [configurations.sensor_power, configurations.switch, configurations.sensor_temperature],
+    'HS1CA-M': [configurations.binary_sensor_carbon_monoxide, configurations.binary_sensor_battery_low],
+    '7099860PH': [configurations.light_brightness_colorxy],
+    'HV-GSCXZB269': [configurations.light_brightness_colortemp],
+    '3216231P5': [configurations.light_brightness_colortemp],
+    'AC03647': [configurations.light_brightness_colortemp_colorxy],
+    '12031': [configurations.cover_position],
+    '421792': [configurations.light_brightness_colortemp_colorxy],
+    'HGZB-06A': [configurations.light_brightness_colortemp_colorxy],
+    'LED1733G7': [configurations.light_brightness_colortemp],
+    '9290011370B': [configurations.light_brightness],
+    'RB 250 C': [configurations.light_brightness_colortemp_colorxy],
+    '8718696170625': [configurations.light_brightness],
+    'GL-G-001Z': [configurations.light_brightness_colortemp_colorxy],
+    'HV-GSCXZB279_HV-GSCXZB229': [configurations.light_brightness_colortemp],
+    'HS2WD-E': [configurations.sensor_battery],
+    'ZNMS12LM': [
+        configurations.sensor_action, configurations.binary_sensor_lock, configurations.binary_sensor_lock_reverse,
+    ],
+    '12050': [configurations.switch, configurations.sensor_power],
+    'ROB_200-004-0': [configurations.light_brightness],
+    'RH3040': [configurations.sensor_battery, configurations.binary_sensor_occupancy],
 };
 
 Object.keys(mapping).forEach((key) => {
@@ -664,10 +829,11 @@ class HomeAssistant {
         }
 
         this.discoveryTopic = settings.get().advanced.homeassistant_discovery_topic;
+        this.statusTopic = settings.get().advanced.homeassistant_status_topic;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe('hass/status');
+        this.mqtt.subscribe(this.statusTopic);
 
         // MQTT discovery of all paired devices on startup.
         this.zigbee.getAllClients().forEach((device) => {
@@ -709,7 +875,7 @@ class HomeAssistant {
 
             // Set json_attributes_topic for types which support this
             // https://github.com/Koenkk/zigbee2mqtt/issues/840
-            if (['binary_sensor', 'sensor', 'lock'].includes(config.type)) {
+            if (['binary_sensor', 'sensor', 'lock', 'climate', 'fan'].includes(config.type)) {
                 payload.json_attributes_topic = payload.state_topic;
             }
 
@@ -721,7 +887,7 @@ class HomeAssistant {
 
             // Attributes for device registry
             payload.device = {
-                identifiers: `zigbee2mqtt_${entityID}`,
+                identifiers: [`zigbee2mqtt_${entityID}`],
                 name: entity.friendlyName,
                 sw_version: `Zigbee2mqtt ${this.zigbee2mqttVersion}`,
                 model: `${mappedModel.description} (${mappedModel.model})`,
@@ -756,12 +922,45 @@ class HomeAssistant {
                 }
 
                 payload.command_topic += 'set';
+
+                if (payload.command_topic_postfix) {
+                    payload.command_topic += `/${payload.command_topic_postfix}`;
+                    delete payload.command_topic_postfix;
+                }
             }
 
             if (payload.set_position_topic && payload.command_topic) {
                 payload.set_position_topic = payload.command_topic;
             }
 
+            if (payload.mode_state_topic) {
+                payload.mode_state_topic = stateTopic;
+            }
+
+            if (payload.mode_command_topic) {
+                payload.mode_command_topic = `${stateTopic}/set/system_mode`;
+            }
+
+            if (payload.current_temperature_topic) {
+                payload.current_temperature_topic = stateTopic;
+            }
+
+            if (payload.temperature_state_topic) {
+                payload.temperature_state_topic = stateTopic;
+            }
+
+            if (payload.speed_state_topic) {
+                payload.speed_state_topic = stateTopic;
+            }
+
+            if (payload.temperature_command_topic) {
+                payload.temperature_command_topic = `${stateTopic}/set/occupied_heating_setpoint`;
+            }
+
+            if (payload.speed_command_topic) {
+                payload.speed_command_topic = `${stateTopic}/set/fan_mode`;
+            }
+
             // Override configuration with user settings.
             if (device.hasOwnProperty('homeassistant')) {
                 const add = (obj) => {
@@ -790,7 +989,7 @@ class HomeAssistant {
     }
 
     onMQTTMessage(topic, message) {
-        if (!topic === 'hass/status') {
+        if (!topic === this.statusTopic) {
             return false;
         }
 
@@ -798,7 +997,7 @@ class HomeAssistant {
             const timer = setTimeout(() => {
                 // Publish all device states.
                 this.zigbee.getAllClients().forEach((device) => {
-                    if (this.state && this.state.exists(device.ieeeAddr)) {
+                    if (this.state.exists(device.ieeeAddr)) {
                         this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
                     }
                 });

lib/extension/networkMap.js

@@ -1,4 +1,5 @@
 const settings = require('../util/settings');
+const utils = require('../util/utils');
 const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
 
 class NetworkMap {
@@ -6,10 +7,15 @@ class NetworkMap {
         this.zigbee = zigbee;
         this.mqtt = mqtt;
         this.state = state;
+        this.lastSeenMap = {};
 
         // Subscribe to topic.
         this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
 
+        // Bind
+        this.raw = this.raw.bind(this);
+        this.graphviz = this.graphviz.bind(this);
+
         // Set supported formats
         this.supportedFormats = {
             'raw': this.raw,
@@ -17,6 +23,12 @@ class NetworkMap {
         };
     }
 
+    onZigbeeMessage(message, device, mappedDevice) {
+        if (device) {
+            this.lastSeenMap[device.ieeeAddr] = Date.now();
+        }
+    }
+
     onMQTTConnected() {
         this.mqtt.subscribe(this.topic);
     }
@@ -41,19 +53,26 @@ class NetworkMap {
     }
 
     graphviz(zigbee, topology) {
+        const colors = settings.get().map_options.graphviz.colors;
+
         let text = 'digraph G {\nnode[shape=record];\n';
         let devStyle = '';
 
-        zigbee.getDevices().forEach((device) => {
+        topology.nodes.forEach((device) => {
             const labels = [];
-            const friendlyDevice = settings.getDevice(device.ieeeAddr);
-            const friendlyName = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
 
             // Add friendly name
-            labels.push(friendlyName);
+            labels.push(`${device.friendlyName}`);
 
-            // Add the device type
-            labels.push(device.type);
+            // Add the device short network address and scan note (if any)
+            let scanNote = '';
+            if (device.scanfailed.includes('lqi')) {
+                scanNote += ' no lqi';
+            }
+            if (device.scanfailed.includes('rtg')) {
+                scanNote += ' no routes';
+            }
+            labels.push(`0x${device.nwkAddr.toString(16)} ${scanNote}`);
 
             // Add the device model
             const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
@@ -65,30 +84,53 @@ class NetworkMap {
                 labels.push(zigbeeModel ? zigbeeModel : 'No model information available');
             }
 
-            // Add the device status (online/offline)
-            labels.push(device.status);
+            // Add the device status (online/offline) and last_seen timestamp
+            let lastSeen = 'unknown';
+            if (device.type == 'Coordinator') {
+                lastSeen = utils.formatDate(Date.now(), settings.get().advanced.last_seen,
+                    utils.toLocalISOString(new Date(Date.now())));
+            } else {
+                if (this.lastSeenMap[device.ieeeAddr]) {
+                    const lastSeenAgo =
+                        `${new Date(Date.now() - this.lastSeenMap[device.ieeeAddr]).toISOString().substr(11, 8)}s ago`;
+                    lastSeen = utils.formatDate(this.lastSeenMap[device.ieeeAddr],
+                        settings.get().advanced.last_seen, lastSeenAgo);
+                }
+            }
+            labels.push(`${device.status} (${lastSeen})`);
 
             // Shape the record according to device type
             if (device.type == 'Coordinator') {
-                devStyle = 'style="bold"';
+                devStyle = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
+                    `fontcolor="${colors.font.coordinator}"`;
             } else if (device.type == 'Router') {
-                devStyle = 'style="rounded"';
+                devStyle = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
+                    `fontcolor="${colors.font.router}"`;
             } else {
-                devStyle = 'style="rounded, dashed"';
+                devStyle = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", `
+                    + `fontcolor="${colors.font.enddevice}"`;
             }
 
             // Add the device with its labels to the graph as a node.
             text += `  "${device.ieeeAddr}" [`+devStyle+`, label="{${labels.join('|')}}"];\n`;
 
             /**
-             * Add an edge between the device and its parent to the graph
+             * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.filter((e) => e.ieeeAddr === device.ieeeAddr).forEach((e) => {
-                const lineStyle = (e.lqi==0) ? `style="dashed", ` : ``;
-                text += `  "${device.ieeeAddr}" -> "${e.parent}" [`+lineStyle+`label="${e.lqi}"]\n`;
-            });
+            topology.links.filter((e) => (e.sourceIeeeAddr === device.ieeeAddr) || (e.SourceNwkAddr === device.nwkAddr))
+                .forEach((e) => {
+                    const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, '
+                        : (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
+                    const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", `
+                        : `weight=1, color="${colors.line.active}", `;
+                    const textRoutes = e.routes.map((r) => `0x${r.toString(16)}`);
+                    const lineLabels = (!e.routes.length) ? `label="${e.lqi}"`
+                        : `label="${e.lqi}\\n[${textRoutes.join(']\\n[')}]"`;
+                    text += `  "${device.ieeeAddr}" -> "${e.targetIeeeAddr}"`;
+                    text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
+                });
         });
 
         text += '}';

lib/extension/responder.js

@@ -0,0 +1,75 @@
+const zclId = require('zigbee-herdsman/dist/zcl-id');
+
+const OneJanuary2000 = new Date('January 01, 2000 00:00:00').getTime();
+
+/**
+ * This extensions is reponsible for respondig to device requests.
+ */
+class ExtensionResponder {
+    constructor(zigbee, mqtt, state, publishEntityState) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+
+        this.onZclFoundation = this.onZclFoundation.bind(this);
+        this.configured = [];
+    }
+
+    setupOnZclFoundation(device) {
+        if (device && device.epList && !this.configured.includes(device.ieeeAddr)) {
+            this.configured.push(device.ieeeAddr);
+
+            device.epList.forEach((epID) => {
+                const ep = this.zigbee.getEndpoint(device.ieeeAddr, epID);
+                ep.onZclFoundation = this.onZclFoundation;
+            });
+        }
+    }
+
+    onZigbeeStarted() {
+        this.zigbee.getAllClients().forEach((device) => this.setupOnZclFoundation(device));
+    }
+
+    onZigbeeMessage(message, device, mappedDevice) {
+        this.setupOnZclFoundation(device);
+    }
+
+    readResponse(message, endpoint) {
+        const clusterID = message.clusterid;
+        const cluster = zclId.cluster(clusterID).key;
+        const attributes = message.zclMsg.payload.map((p) => zclId.attr(message.clusterid, p.attrId));
+        const response = [];
+
+        attributes.forEach((attribute) => {
+            if (cluster === 'genTime' && attribute.key === 'time') {
+                const time = Math.round(((new Date()).getTime() - OneJanuary2000) / 1000);
+                response.push(this.createReadResponseRec(clusterID, attribute.value, time));
+            }
+        });
+
+        this.zigbee.publish(
+            endpoint.device.ieeeAddr, 'device', cluster, 'readRsp', 'foundation', response,
+            {direction: 1, seqNum: message.zclMsg.seqNum, disDefaultRsp: 1}, endpoint.epId,
+        );
+    }
+
+    createReadResponseRec(cId, attrId, value) {
+        return {
+            attrId: attrId,
+            status: 0,
+            attrData: value,
+            dataType: zclId.attrType(cId, attrId).value,
+        };
+    }
+
+    onZclFoundation(message, endpoint) {
+        const cmd = message.zclMsg.cmdId;
+
+        if (cmd === 'read') {
+            this.readResponse(message, endpoint);
+        }
+    }
+}
+
+module.exports = ExtensionResponder;

lib/mqtt.js

@@ -1,6 +1,7 @@
 const mqtt = require('mqtt');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
+const fs = require('fs');
 
 class MQTT {
     constructor() {
@@ -21,6 +22,32 @@ class MQTT {
             },
         };
 
+        if (mqttSettings.ca) {
+            logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
+            const ca = fs.readFileSync(mqttSettings.ca);
+
+            if (ca) {
+                options.ca = ca;
+            } else {
+                logger.error(`Error loading CA certificate for MQTT SSL/TLS configuration.`);
+            }
+        }
+
+        if (mqttSettings.key && mqttSettings.cert) {
+            logger.debug(`MQTT SSL/TLS: Path to client key = ${mqttSettings.key}`);
+            logger.debug(`MQTT SSL/TLS: Path to client certificate = ${mqttSettings.cert}`);
+
+            const key = fs.readFileSync(mqttSettings.key);
+            const cert = fs.readFileSync(mqttSettings.cert);
+
+            if (key && cert) {
+                options.key = key;
+                options.cert = cert;
+            } else {
+                logger.error(`Error loading key and/or certificate for MQTT SSL/TLS client authentication.`);
+            }
+        }
+
         if (mqttSettings.user && mqttSettings.password) {
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;

lib/state.js

@@ -10,6 +10,7 @@ const dontCacheProperties = [
     'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
     'step_size', 'transition_time', 'action_color_temperature', 'action_color',
     'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed',
 ];
 
 class State {
@@ -17,6 +18,7 @@ class State {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
+        this.stateChangeListeners = [];
 
         this.handleSettingsChanged = this.handleSettingsChanged.bind(this);
     }
@@ -38,6 +40,10 @@ class State {
         this.checkLastSeen();
     }
 
+    registerOnStateChangeListener(listener) {
+        this.stateChangeListeners.push(listener);
+    }
+
     checkLastSeen() {
         if (settings.get().advanced.last_seen === 'disable') {
             Object.values(this.state).forEach((s) => {
@@ -90,14 +96,17 @@ class State {
     }
 
     set(ieeeAddr, state) {
-        const s = objectAssignDeep.noMutate(state);
+        const toState = objectAssignDeep.noMutate(state);
         dontCacheProperties.forEach((property) => {
-            if (s.hasOwnProperty(property)) {
-                delete s[property];
+            if (toState.hasOwnProperty(property)) {
+                delete toState[property];
             }
         });
 
-        this.state[ieeeAddr] = s;
+        const fromState = this.state[ieeeAddr];
+        this.stateChangeListeners.forEach((listener) => listener(ieeeAddr, fromState, toState));
+
+        this.state[ieeeAddr] = toState;
     }
 
     remove(ieeeAddr) {

lib/util/data.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const fs = require('fs');
 
 let dataPath = null;
 
@@ -13,8 +14,18 @@ function load() {
 
 load();
 
+function joinPathStorage(file) {
+    const storagePath = path.join(dataPath, '.storage');
+    if (!fs.existsSync(storagePath)) {
+        fs.mkdirSync(storagePath);
+    }
+
+    return path.join(storagePath, file);
+}
+
 module.exports = {
     joinPath: (file) => path.join(dataPath, file),
+    joinPathStorage: (file) => joinPathStorage(file),
     getPath: () => dataPath,
 
     // For test only.

lib/util/logger.js

@@ -16,8 +16,27 @@ const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', t
 // Make sure that log directoy exsists
 fx.mkdirSync(directory);
 
+// Custom level
+const levels = {
+    levels: {
+        error: 0,
+        warn: 1,
+        info: 2,
+        debug: 3,
+        trace: 4,
+    },
+    colors: {
+        error: 'red',
+        warn: 'yellow',
+        info: 'green',
+        debug: 'blue',
+        trace: 'magenta',
+    },
+};
+
 // Create logger
 const logger = new winston.Logger({
+    levels: levels.levels,
     transports: [
         new winston.transports.File({
             filename: path.join(directory, 'log.txt'),
@@ -37,25 +56,31 @@ const logger = new winston.Logger({
     ],
 });
 
-logger.info(`Logging to directory: '${directory}'`);
+// Add colors
+winston.addColors(levels.colors);
 
 logger.transports.console.level = level;
 
 // Cleanup any old log directory.
-if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
-    const rootDirectory = path.join(directory, '..');
+function cleanup() {
+    if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
+        const rootDirectory = path.join(directory, '..');
 
-    let directories = fs.readdirSync(rootDirectory).map((d) => {
-        d = path.join(rootDirectory, d);
-        return {path: d, birth: fs.statSync(d).birthtimeMs};
-    });
+        let directories = fs.readdirSync(rootDirectory).map((d) => {
+            d = path.join(rootDirectory, d);
+            return {path: d, birth: fs.statSync(d).birthtimeMs};
+        });
 
-    directories.sort((a, b) => b.birth - a.birth);
-    directories = directories.slice(10, directories.length);
-    directories.forEach((dir) => {
-        logger.debug(`Removing old log directory '${dir.path}'`);
-        rimraf.sync(dir.path);
-    });
+        directories.sort((a, b) => b.birth - a.birth);
+        directories = directories.slice(10, directories.length);
+        directories.forEach((dir) => {
+            logger.debug(`Removing old log directory '${dir.path}'`);
+            rimraf.sync(dir.path);
+        });
+    }
 }
 
+logger.cleanup = cleanup;
+logger.directory = directory;
+
 module.exports = logger;

lib/util/settings.js

@@ -6,6 +6,7 @@ const fs = require('./fs');
 const onChangeHandlers = [];
 
 const defaults = {
+    whitelist: [],
     ban: [],
     permit_join: false,
     mqtt: {
@@ -13,6 +14,30 @@ const defaults = {
     },
     groups: {},
     device_options: {},
+    map_options: {
+        graphviz: {
+            colors: {
+                fill: {
+                    enddevice: '#fff8ce',
+                    coordinator: '#e04e5d',
+                    router: '#4ea3e0',
+                },
+                font: {
+                    coordinator: '#ffffff',
+                    router: '#ffffff',
+                    enddevice: '#000000',
+                },
+                line: {
+                    active: '#009900',
+                    inactive: '#994444',
+                },
+            },
+        },
+    },
+    queue: {
+        delay: 250,
+        simultaneously: 5,
+    },
     experimental: {
         livolo: false,
         // json or attribute
@@ -71,6 +96,11 @@ const defaults = {
          * Home Assistant discovery topic
          */
         homeassistant_discovery_topic: 'homeassistant',
+
+        /**
+         * Home Assistant status topic
+         */
+        homeassistant_status_topic: 'hass/status',
     },
 };
 
@@ -153,9 +183,17 @@ const getDevices = () => getSettings().devices || [];
 
 const getDevice = (ieeeAddr) => getDevices()[ieeeAddr];
 
-const getGroups = () => getSettings().groups || [];
+const getGroups = () => getSettings().groups || {};
 
-const getGroup = (ID) => getGroups()[ID];
+const getGroup = (ID) => {
+    const group = getGroups()[ID];
+
+    if (group && !group.hasOwnProperty('devices')) {
+        group.devices = [];
+    }
+
+    return group;
+};
 
 
 function addDevice(ieeeAddr) {
@@ -168,6 +206,17 @@ function addDevice(ieeeAddr) {
     writeRead();
 }
 
+function whitelistDevice(ieeeAddr) {
+    const settings = getSettings();
+    if (!settings.whitelist) {
+        settings.whitelist = [];
+    }
+
+    settings.whitelist.push(ieeeAddr);
+
+    writeRead();
+}
+
 function banDevice(ieeeAddr) {
     const settings = getSettings();
     if (!settings.ban) {
@@ -198,12 +247,45 @@ function addGroup(groupName) {
         ID = (Number.parseInt(ID) + 1).toString();
     }
 
+    if (Object.values(settings.groups).find((g) => g.friendly_name === groupName)) {
+        return false; // Duplicate group names are not allowed
+    }
+
     settings.groups[ID] = {friendly_name: groupName};
     writeRead();
 
     return true;
 }
 
+function addDeviceToGroup(ID, ieeeAddr) {
+    const settings = getSettings();
+    const group = settings.groups[ID];
+    if (!group.devices) {
+        group.devices = [];
+    }
+
+    if (!group.devices.includes(ieeeAddr)) {
+        group.devices.push(ieeeAddr);
+        writeRead();
+        return true;
+    } else {
+        return false;
+    }
+}
+
+function removeDeviceFromGroup(ID, ieeeAddr) {
+    const settings = getSettings();
+    const group = settings.groups[ID];
+
+    if (group.devices && group.devices.includes(ieeeAddr)) {
+        group.devices = group.devices.filter((d) => d != ieeeAddr);
+        writeRead();
+        return true;
+    } else {
+        return false;
+    }
+}
+
 function removeGroup(name) {
     const settings = getSettings();
     if (!settings.groups) return;
@@ -303,11 +385,14 @@ module.exports = {
     getGroup,
     getGroups,
     getDevices,
+    whitelistDevice: (ieeeAddr) => whitelistDevice(ieeeAddr),
     banDevice: (ieeeAddr) => banDevice(ieeeAddr),
     addDevice: (ieeeAddr) => addDevice(ieeeAddr),
     removeDevice: (ieeeAddr) => removeDevice(ieeeAddr),
     addGroup: (name) => addGroup(name),
     removeGroup: (name) => removeGroup(name),
+    addDeviceToGroup: (ID, ieeeAddr) => addDeviceToGroup(ID, ieeeAddr),
+    removeDeviceFromGroup: (ID, ieeeAddr) => removeDeviceFromGroup(ID, ieeeAddr),
 
     getIeeeAddrByFriendlyName: (friendlyName) => getIeeeAddrByFriendlyName(friendlyName),
     getGroupIDByFriendlyName: (friendlyName) => getGroupIDByFriendlyName(friendlyName),

lib/util/utils.js

@@ -28,10 +28,21 @@ function toLocalISOString(dDate) {
 }
 
 const postfixes = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right',
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'l1', 'l2',
     'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom',
 ];
 
+function flatten(arr) {
+    return arr.reduce((flat, toFlatten) => {
+        return flat.concat(Array.isArray(toFlatten) ? flatten(toFlatten) : toFlatten);
+    }, []);
+}
+
+const forceEndDevice = flatten(
+    ['QBKG03LM', 'QBKG04LM']
+        .map((model) => zigbeeShepherdConverters.devices.find((d) => d.model === model))
+        .map((mappedModel) => mappedModel.zigbeeModel));
+
 function getEndpointByEntityID(zigbee, entityID, epName) {
     const device = zigbee.getDevice(entityID);
     if (!device) {
@@ -57,6 +68,9 @@ function getEndpointByEntityID(zigbee, entityID, epName) {
             logger.error(`Device ${mappedDevice.model} doesn't have ep named '${epName}'`);
             return;
         }
+    } else if (mappedDevice.hasOwnProperty('ep')) {
+        const eps = mappedDevice.ep(device);
+        epID = eps[''] || null;
     }
 
     const endpoint = zigbee.getEndpoint(entityID, epID);
@@ -89,16 +103,50 @@ function getZigbee2mqttVersion(callback) {
     });
 }
 
+function formatDate(date, type, _default) {
+    let result;
+
+    switch (type) {
+    case 'ISO_8601':
+        result = new Date(date).toISOString();
+        break;
+    case 'ISO_8601_local':
+        result = toLocalISOString(new Date(date));
+        break;
+    case 'epoch':
+        result = date;
+        break;
+    default:
+        result = _default;
+        break;
+    }
+
+    return result;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
-    isXiaomiDevice: (device) => device.modelId !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufId),
+    isXiaomiDevice: (device) => {
+        return device.modelId !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufId) &&
+            (!device.manufName || !device.manufName.startsWith('Trust'));
+    },
     isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufId),
-    isRouter: (device) => device.type === 'Router',
+    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelId),
     isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     isNumeric: (string) => /^\d+$/.test(string),
     toLocalISOString: (dDate) => toLocalISOString(dDate),
     getPostfixes: () => postfixes,
+    formatDate: (date, type, _default=null) => formatDate(date, type, _default),
+    correctDeviceType: (device) => {
+        if (device) {
+            if (forceEndDevice.includes(device.modelId)) {
+                return 'EndDevice';
+            }
+        }
+
+        return device.type;
+    },
     getEndpointByEntityID,
     getZigbee2mqttVersion,
 };

lib/util/zigbeeQueue.js

@@ -1,6 +1,9 @@
-const maxSimultaneouslyRunning = 5;
-const delay = 250;
+const settings = require('./settings');
+
+const maxSimultaneouslyRunning = settings.get().queue.simultaneously;
+const delay = settings.get().queue.delay;
 const error17Retries = 3;
+const logger = require('./logger');
 
 class ZigbeeQueue {
     constructor() {
@@ -8,6 +11,7 @@ class ZigbeeQueue {
         this.active = [];
         this.timer = null;
         this.running = false;
+        this.ID = 0;
     }
 
     start() {
@@ -16,7 +20,13 @@ class ZigbeeQueue {
     }
 
     push(entityID, func) {
-        this.queue.push({entityID, func, attempts: 0});
+        this.ID++;
+        this.queue.push({entityID, func, attempts: 0, ID: this.ID});
+        this.log(`Added new job with ID ${this.ID} for ${entityID}`);
+    }
+
+    log(message) {
+        logger.trace(`zigbeeQueue: ${message}`);
     }
 
     stopTimer() {
@@ -40,6 +50,8 @@ class ZigbeeQueue {
     }
 
     handleJobComplete(job, error) {
+        this.log(`Completed job with ID ${job.ID} for ${job.entityID}${error ? ` with error ${error}` : ''}`);
+
         if (error && error.message === 'rsp error: 17' && job.attempts < error17Retries) {
             // Error 17 means that the buffer of the ZNP was full,
             // retry this for a maximum of 3 times.
@@ -60,6 +72,7 @@ class ZigbeeQueue {
 
         if (next) {
             this.active.push(next);
+            this.log(`Executing job with ID ${next.ID} for ${next.entityID}`);
             next.func((error) => this.handleJobComplete(next, error));
         }
     }

lib/zapp/cie.js

@@ -3,49 +3,13 @@
  * Based on: https://github.com/zigbeer/zapp-cie
  */
 
-const Ziee = require('ziee');
-const Zive = require('zive');
+const Ziee = require('zigbee-herdsman/dist/ziee');
+const Zive = require('zigbee-herdsman/dist/zive');
 const cieClusters = new Ziee();
+// Direction-Value 1: input, 2: output, 3: input/output
 
-cieClusters.init('genBasic', 'dir', {value: 1}); // Server Side(Input)
-cieClusters.init('ssIasAce', 'dir', {value: 1}); // Server Side(Input)
+// IasZone Cluster-Init Direction
 cieClusters.init('ssIasZone', 'dir', {value: 2}); // Client Side(Output)
-cieClusters.init('ssIasWd', 'dir', {value: 2}); // Client Side(Output)
-cieClusters.init('genIdentify', 'dir', {value: 3}); // Server and Client Side(Input/Output)
-
-// Init Attributes Access Control
-cieClusters.init('genBasic', 'acls', {
-    zclVersion: 'R',
-    hwVersion: 'R',
-    manufacturerName: 'R',
-    modelId: 'R',
-    dateCode: 'R',
-    powerSource: 'R',
-    locationDesc: 'RW',
-    physicalEnv: 'RW',
-    deviceEnabled: 'RW',
-});
-
-cieClusters.init('genIdentify', 'acls', {
-    identifyTime: 'RW',
-});
-
-// Init Attributes Value
-cieClusters.init('genBasic', 'attrs', {
-    zclVersion: 1,
-    hwVersion: 1,
-    manufacturerName: 'sivann inc.',
-    modelId: 'hiver0001',
-    dateCode: '20170407',
-    powerSource: 1,
-    locationDesc: '    ',
-    physicalEnv: 0,
-    deviceEnabled: 1,
-});
-
-cieClusters.init('genIdentify', 'attrs', {
-    identifyTime: 0,
-});
 
 // Init Command Response Handler
 cieClusters.init('ssIasZone', 'cmdRsps', {
@@ -53,5 +17,7 @@ cieClusters.init('ssIasZone', 'cmdRsps', {
     statusChangeNotification: (zapp, argObj, cb) => {},
 });
 
+// Create Zive with profId: 0x0104 (Decimal: 260 / HA), devId: 0x0400 (Decimal: 1024 / iasControlIndicatingEquipment)
+// Check the Values by https://github.com/zigbeer/zcl-id/wiki#5-table-of-identifiers
 const cieApp = new Zive({profId: 0x0104, devId: 0x0400, discCmds: []}, cieClusters);
 module.exports = cieApp;

lib/zigbee.js

@@ -1,4 +1,4 @@
-const ZShepherd = require('zigbee-shepherd');
+const ZShepherd = require('zigbee-herdsman');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
 const data = require('./util/data');
@@ -6,7 +6,7 @@ const utils = require('./util/utils');
 const ZigbeeQueue = require('./util/zigbeeQueue');
 const cieApp = require('./zapp/cie');
 const objectAssignDeep = require('object-assign-deep');
-const zclId = require('zcl-id');
+const zclId = require('zigbee-herdsman/dist/zcl-id');
 
 const advancedSettings = settings.get().advanced;
 
@@ -22,6 +22,7 @@ const shepherdSettings = {
         precfgkey: settings.get().advanced.network_key,
     },
     dbPath: data.joinPath('database.db'),
+    coordBackupPath: data.joinPath('coordinator_backup.json'),
     sp: {
         baudRate: advancedSettings.baudrate,
         rtscts: advancedSettings.rtscts,
@@ -36,7 +37,6 @@ const defaultCfg = {
 // Don't print network key.
 const shepherdSettingsLog = objectAssignDeep.noMutate(shepherdSettings);
 shepherdSettingsLog.net.precfgkey = 'HIDDEN';
-logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettingsLog)}'`);
 
 class Zigbee {
     constructor() {
@@ -44,25 +44,28 @@ class Zigbee {
         this.onMessage = this.onMessage.bind(this);
         this.onError = this.onError.bind(this);
         this.messageHandler = null;
+        this.permitJoinTimer = null;
 
         this.queue = new ZigbeeQueue();
     }
 
     start(messageHandler, callback) {
         logger.info(`Starting zigbee-shepherd`);
+        logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettingsLog)}'`);
+
         this.messageHandler = messageHandler;
         this.shepherd = new ZShepherd(settings.get().serial.port, shepherdSettings);
 
         this.shepherd.start((error) => {
             if (error) {
-                logger.info('Error while starting zigbee-shepherd, attempting to fix... (takes 60 seconds)');
+                logger.info(`Error while starting zigbee-shepherd, attempting to fix... (takes 60 seconds) (${error})`);
                 this.shepherd.controller._znp.close((() => null));
 
                 setTimeout(() => {
                     logger.info(`Starting zigbee-shepherd`);
                     this.shepherd.start((error) => {
                         if (error) {
-                            logger.error('Error while starting zigbee-shepherd!');
+                            logger.error(`Error while starting zigbee-shepherd! (${error})`);
                             logger.error(
                                 'Press the reset button on the stick (the one closest to the USB) and start again'
                             );
@@ -91,24 +94,44 @@ class Zigbee {
         this.logStartupInfo();
 
         this.getAllClients().forEach((device) => {
-            if (settings.get().ban.includes(device.ieeeAddr)) {
-                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing..`);
-                this.removeDevice(device.ieeeAddr, false, () => {});
+            // If set whitelist devices, all other device will be ban or reject to join the network
+            if (settings.get().whitelist.size>0) {
+                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
+                    logger.warn(`Blacklist device is connected (${device.ieeeAddr}), removing...`);
+                    this.removeDevice(device.ieeeAddr, true, () => {});
+                }
+            } else {
+                if (settings.get().ban.includes(device.ieeeAddr)) {
+                    logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+                    this.removeDevice(device.ieeeAddr, false, () => {});
+                }
             }
         });
+
+        this.shepherd.backupCoordinator(() => {});
     }
 
     _acceptDevIncoming(devInfo, callback) {
         logger.debug(
             `Accept device incoming with ieeeAddr '${devInfo.ieeeAddr}' permit join is '${this.getPermitJoin()}'`
         );
-
-        if (settings.get().ban.includes(devInfo.ieeeAddr)) {
-            logger.info(`Banned device tried to connect (${devInfo.ieeeAddr})`);
-            callback(null, false);
+        // If set whitelist devices, all other device will be ban or reject to join the network
+        if (settings.get().whitelist.size>0) {
+            if (settings.get().whitelist.includes(devInfo.ieeeAddr)) {
+                logger.info(`whitelist device tried to connect (${devInfo.ieeeAddr})`);
+                callback(null, true);
+            } else {
+                logger.debug(`Not allowing device '${devInfo.ieeeAddr}' to join`);
+                callback(null, false);
+            }
         } else {
-            logger.debug(`Allowing device '${devInfo.ieeeAddr}' to join`);
-            callback(null, true);
+            if (settings.get().ban.includes(devInfo.ieeeAddr)) {
+                logger.info(`Banned device tried to connect (${devInfo.ieeeAddr})`);
+                callback(null, false);
+            } else {
+                logger.debug(`Allowing device '${devInfo.ieeeAddr}' to join`);
+                callback(null, true);
+            }
         }
     }
 
@@ -127,11 +150,18 @@ class Zigbee {
     }
 
     stop(callback) {
+        if (this.permitJoinTimer) {
+            clearInterval(this.permitJoinTimer);
+        }
+
         this.queue.stop();
 
-        this.shepherd.stop((error) => {
-            logger.info('zigbee-shepherd stopped');
-            callback(error);
+        // Backup coordinator
+        this.shepherd.backupCoordinator(() => {
+            this.shepherd.stop((error) => {
+                logger.info('zigbee-shepherd stopped');
+                callback(error);
+            });
         });
     }
 
@@ -170,6 +200,24 @@ class Zigbee {
             logger.info('Zigbee: disabling joining new devices.');
         }
 
+        // In zigbee 3.0 a network automatically closes after 254 seconds.
+        // As a workaround, we enable joining again.
+        if (this.permitJoinTimer) {
+            clearInterval(this.permitJoinTimer);
+        }
+
+        if (permit) {
+            this.permitJoinTimer = setInterval(() => {
+                this.shepherd.permitJoin(255, (error) => {
+                    if (error) {
+                        logger.error('Failed to reenable joining');
+                    } else {
+                        logger.info('Successfully reenabled joining');
+                    }
+                });
+            }, utils.secondsToMilliseconds(160));
+        }
+
         this.shepherd.permitJoin(permit ? 255 : 0, (error) => {
             if (error) {
                 logger.info(error);
@@ -194,12 +242,14 @@ class Zigbee {
             settings.banDevice(deviceID);
         }
 
+        const friendlyName = this.getDeviceFriendlyName(deviceID);
+
         this.shepherd.remove(deviceID, {reJoin: true}, (error) => {
             if (error) {
-                logger.warn(`Failed to remove '${deviceID}', trying force remove...`);
+                logger.warn(`Failed to remove '${friendlyName}', trying force remove...`);
                 this.forceRemove(deviceID, callback);
             } else {
-                logger.info(`Removed ${deviceID}`);
+                logger.info(`Removed ${friendlyName}`);
                 callback(null);
             }
         });
@@ -209,8 +259,9 @@ class Zigbee {
         const device = this.shepherd._findDevByAddr(deviceID);
 
         if (device) {
+            const friendlyName = this.getDeviceFriendlyName(deviceID);
             return this.shepherd._unregisterDev(device, (error) => {
-                logger.info(`Force removed ${deviceID}`);
+                logger.info(`Force removed ${friendlyName}`);
                 callback(error);
             });
         } else {
@@ -233,21 +284,178 @@ class Zigbee {
         return this.getDevices().find((d) => d.ieeeAddr === ieeeAddr);
     }
 
+    getDeviceFriendlyName(ieeeAddr) {
+        const device = settings.getDevice(ieeeAddr);
+        return device ? device.friendly_name || ieeeAddr : ieeeAddr;
+    }
+
     getCoordinator() {
         const device = this.getDevices().find((d) => d.type === 'Coordinator');
         return this.shepherd.find(device.ieeeAddr, 1);
     }
 
+    getScanable() {
+        return this.getDevices().filter((d) => d.type != 'EndDevice');
+    }
+
     getGroup(ID) {
         return this.shepherd.getGroup(ID);
     }
 
+    getGroupFriendlyName(ID) {
+        let friendlyName = null;
+        friendlyName = settings.getGroup(ID).friendly_name;
+        return (friendlyName ? friendlyName : ID);
+    }
+
     networkScan(callback) {
         logger.info('Starting network scan...');
-        this.shepherd.lqiScan().then((result) => {
-            logger.info('Network scan completed');
-            callback(result);
+        const lqiScanList = new Set();
+        const rtgScanList = new Set();
+        const linkMap = [];
+        const routeMap = [];
+
+        // Gather the lqi and the route info into separate lists and only collate them when done.
+        const collateMap = () => {
+            linkMap.sort();
+            logger.debug(`Link map: %j`, linkMap);
+            logger.debug(`Route map: %j`, routeMap);
+            // Merge the routes into the linkMap by matching on 'source|target' link short addresses
+            linkMap.forEach((link) => {
+                routeMap.filter((e) => e.key === link.key).forEach((e) => {
+                    link.routes.push(e.destAddr);
+                });
+                link.routes.sort(function(a, b) {
+                    return a-b;
+                });
+                delete link.key;
+            });
+            const networkMap = {nodes: [], links: linkMap};
+            this.getDevices().forEach((device) => {
+                const friendlyDevice = settings.getDevice(device.ieeeAddr);
+                const friendlyName = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
+                const deviceType = utils.correctDeviceType(device);
+                const scanfailed = [];
+                if (lqiScanList.has(device.ieeeAddr)) {
+                    scanfailed.push('lqi');
+                }
+                if (rtgScanList.has(device.ieeeAddr)) {
+                    scanfailed.push('rtg');
+                }
+                networkMap.nodes.push({ieeeAddr: device.ieeeAddr, friendlyName: friendlyName, type: deviceType,
+                    nwkAddr: device.nwkAddr, manufName: device.manufName, modelId: device.modelId,
+                    status: device.status, scanfailed: scanfailed});
+            });
+            // Clear remaining devices so they don't process when/if they eventually complete
+            lqiScanList.clear();
+            rtgScanList.clear();
+            logger.debug(`Merged map: %j`, networkMap);
+            callback(networkMap);
+        };
+
+        const processLqiResponse = (error, rsp, targetIeeeAddr, targetNwkAddr) => {
+            if (error) {
+                logger.warn(`Failed network lqi scan for device: '${targetIeeeAddr}' with error: '${error}'`);
+            } else {
+                if (lqiScanList.has(targetIeeeAddr)) {
+                    // Haven't processed this one yet
+                    if (rsp && rsp.status === 0 && rsp.neighborlqilist) {
+                        logger.debug(`lqi scan: '${targetIeeeAddr}' with '${rsp.neighborlqilistcount}' neighbors`);
+                        rsp.neighborlqilist.forEach(function(neighbor) {
+                            // only include active relationships
+                            if (neighbor.relationship <= 3) {
+                                // lqi is measured at receiver so link is from neighbor (source) to scanned router
+                                const key = neighbor.nwkAddr + '|' + targetNwkAddr;
+                                linkMap.push({
+                                    key: key, sourceIeeeAddr: neighbor.extAddr, targetIeeeAddr: targetIeeeAddr,
+                                    sourceNwkAddr: neighbor.nwkAddr, lqi: neighbor.lqi, depth: neighbor.depth,
+                                    relationship: neighbor.relationship, routes: []});
+                            }
+                        });
+                        // Remove from scan list and if both lists are done return the completed network map
+                        lqiScanList.delete(targetIeeeAddr);
+                        if (lqiScanList.size === 0 && rtgScanList === 0) {
+                            logger.info('Network scan completed');
+                            collateMap();
+                        } else {
+                            logger.debug(`Outstanding network lqi scans for devices: '${[...lqiScanList].join(' ')}'`);
+                            logger.debug(`Outstanding network rtg scans for devices: '${[...rtgScanList].join(' ')}'`);
+                        }
+                    } else {
+                        logger.warn(`Empty network lqi scan result for: '${targetIeeeAddr}'`);
+                    }
+                } else {
+                    // This target has already had timeout so don't add to result network map
+                    logger.warn(`Ignoring late network lqi scan result for: '${targetIeeeAddr}'`);
+                }
+            }
+        };
+
+        const processRtgResponse = (error, rsp, sourceIeeeAddr, sourceNwkAddr) => {
+            if (error) {
+                logger.warn(`Failed network rtg scan for device: '${sourceIeeeAddr}' with error: '${error}'`);
+            } else {
+                if (rtgScanList.has(sourceIeeeAddr)) {
+                    // Haven't processed this one yet
+                    if (rsp && rsp.status === 0 && rsp.routingtablelist) {
+                        logger.debug(`rtg scan: '${sourceIeeeAddr}' with '${rsp.routingtablelistcount}' entries`);
+                        rsp.routingtablelist.forEach(function(route) {
+                            if (route.routeStatus === 0) {
+                                const key = sourceNwkAddr + '|' + route.nextHopNwkAddr;
+                                routeMap.push({
+                                    key: key, destAddr: route.destNwkAddr});
+                            }
+                        });
+                        // Remove from scan list and if both lists are done return the completed network map
+                        rtgScanList.delete(sourceIeeeAddr);
+                        if (lqiScanList.size === 0 && rtgScanList.size === 0) {
+                            logger.info('Network scan completed');
+                            collateMap();
+                        } else {
+                            logger.debug(`Outstanding network lqi scans for devices: '${[...lqiScanList].join(' ')}'`);
+                            logger.debug(`Outstanding network rtg scans for devices: '${[...rtgScanList].join(' ')}'`);
+                        }
+                    } else {
+                        logger.warn(`Empty network rtg scan result for: '${sourceIeeeAddr}'`);
+                    }
+                } else {
+                    // This source has already had timeout so don't add to result network map
+                    logger.warn(`Ignoring late network rtg scan result for: '${sourceIeeeAddr}'`);
+                }
+            }
+        };
+
+        // Queue up an lqi scan and an rtg scan for coordinator and each router
+        this.getScanable().forEach((dev) => {
+            logger.debug(`Queing network scans for device: '${dev.ieeeAddr}'`);
+            lqiScanList.add(dev.ieeeAddr);
+            this.queue.push(dev.ieeeAddr, (queueCallback) => {
+                this.shepherd.controller.request('ZDO', 'mgmtLqiReq', {dstaddr: dev.nwkAddr, startindex: 0},
+                    (error, rsp, ieeeAddr, nwkAddr) => {
+                        processLqiResponse(error, rsp, dev.ieeeAddr, dev.nwkAddr);
+                        queueCallback(error);
+                    });
+            });
+            rtgScanList.add(dev.ieeeAddr);
+            this.queue.push(dev.ieeeAddr, (queueCallback) => {
+                this.shepherd.controller.request('ZDO', 'mgmtRtgReq', {dstaddr: dev.nwkAddr, startindex: 0},
+                    (error, rsp, ieeeAddr, nwkAddr) => {
+                        processRtgResponse(error, rsp, dev.ieeeAddr, dev.nwkAddr);
+                        queueCallback(error);
+                    });
+            });
         });
+
+        // Wait for all device scans before forcing map with whatever results are already in
+        setTimeout(() => {
+            if (lqiScanList.size === 0 && rtgScanList.size === 0) {
+                logger.info('Network scan timeout no outstanding requests');
+            } else {
+                logger.warn(`Network scan timeout, skipping outstanding lqi scans for '${[...lqiScanList].join(' ')}'`);
+                logger.warn(`Network scan timeout, skipping outstanding rtg scans for '${[...rtgScanList].join(' ')}'`);
+                collateMap();
+            }
+        }, lqiScanList.size * 1000);
     }
 
     getEndpoint(ieeeAddr, ep) {
@@ -266,10 +474,13 @@ class Zigbee {
 
     publish(entityID, entityType, cid, cmd, cmdType, zclData, cfg=defaultCfg, ep, callback) {
         let entity = null;
+        let friendlyName = null;
         if (entityType === 'device') {
             entity = this.getEndpoint(entityID, ep);
+            friendlyName = this.getDeviceFriendlyName(entityID);
         } else if (entityType === 'group') {
             entity = this.getGroup(entityID);
+            friendlyName = this.getGroupFriendlyName(entityID);
         }
 
         if (!entity) {
@@ -281,14 +492,15 @@ class Zigbee {
 
         this.queue.push(entityID, (queueCallback) => {
             logger.info(
-                `Zigbee publish to ${entityType} '${entityID}', ${cid} - ${cmd} - ` +
-                `${JSON.stringify(zclData)} - ${JSON.stringify(cfg)} - ${ep}`
+                `Zigbee publish to ${entityType} '${friendlyName}', ${cid} - ${cmd} - ` +
+                `${JSON.stringify(zclData)} - ${JSON.stringify(cfg)} - ${entity.epId}`
             );
 
             const callback_ = (error, rsp) => {
                 if (error) {
                     logger.error(
-                        `Zigbee publish to ${entityType} '${entityID}', ${cid} - ${cmd} - ${JSON.stringify(zclData)} ` +
+                        `Zigbee publish to ${entityType} '${friendlyName}', ${cid} ` +
+                        `- ${cmd} - ${JSON.stringify(zclData)} ` +
                         `- ${JSON.stringify(cfg)} - ${ep} ` +
                         `failed with error ${error}`);
                 }
@@ -311,11 +523,12 @@ class Zigbee {
     }
 
     ping(ieeeAddr, errorLogLevel='error', cb, mechanism='default') {
+        const friendlyName = this.getDeviceFriendlyName(ieeeAddr);
         const callback = (error) => {
             if (error) {
-                logger[errorLogLevel](`Failed to ping ${ieeeAddr}`);
+                logger[errorLogLevel](`Failed to ping '${friendlyName}'`);
             } else {
-                logger.debug(`Successfully pinged ${ieeeAddr}`);
+                logger.debug(`Successfully pinged '${friendlyName}'`);
             }
 
             if (cb) {
@@ -349,7 +562,8 @@ class Zigbee {
     }
 
     bind(ep, cluster, target, callback) {
-        const log = ` ${ep.device.ieeeAddr} - ${cluster}`;
+        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
+        const log = ` '${friendlyName}' - ${cluster}`;
         target = !target ? this.getCoordinator() : target;
 
         this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
@@ -368,7 +582,8 @@ class Zigbee {
     }
 
     unbind(ep, cluster, target, callback) {
-        const log = ` ${ep.device.ieeeAddr} - ${cluster}`;
+        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
+        const log = ` '${friendlyName}' - ${cluster}`;
         target = !target ? this.getCoordinator() : target;
 
         this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
@@ -396,6 +611,7 @@ class Zigbee {
      *     change  the minimum amount of change before sending a report
      */
     report(ep, cluster, attributes) {
+        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
         const cfgArr = attributes.map((attribute) => {
             const attrId = zclId.attr(cluster, attribute.attr).value;
             const dataType = zclId.attrType(cluster, attribute.attr).value;
@@ -409,7 +625,7 @@ class Zigbee {
             };
         });
 
-        const log=`for ${ep.device.ieeeAddr} - ${cluster} - ${attributes.length}`;
+        const log=`for '${friendlyName}' - ${cluster} - ${attributes.length}`;
 
         const configReport = () => {
             this.queue.push(ep.device.ieeeAddr, (queueCallback) => {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "description": "Zigbee to MQTT bridge using zigbee-shepherd",
   "main": "index.js",
   "repository": {
@@ -43,11 +43,9 @@
     "rimraf": "*",
     "semver": "*",
     "winston": "2.4.2",
-    "zcl-id": "git+https://github.com/Koenkk/zcl-id.git#4fa75c92424cd070a3bf6e1f4640b1e3ad8f802d",
-    "ziee": "*",
-    "zigbee-shepherd": "git+https://github.com/Koenkk/zigbee-shepherd.git#12c2e7f875e8ebccc3971736a4b684f86de9d79e",
-    "zigbee-shepherd-converters": "8.1.1",
-    "zive": "*"
+    "zigbee-herdsman": "0.1.16",
+    "zigbee-shepherd-converters": "10.1.4",
+    "deep-diff": "*"
   },
   "devDependencies": {
     "@types/jest": "^24.0.11",

scripts/generateNewSupportedDevices.js

@@ -0,0 +1,10 @@
+const fs = require('fs');
+const filename = process.argv[2];
+const text = fs.readFileSync(filename, 'utf8');
+const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+
+
+for (const line of text.split('\n')) {
+    const model = zigbeeShepherdConverters.devices.find((d) => d.model === line);
+    console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
+}

test/bridgeConfig.test.js

@@ -4,6 +4,7 @@ const fs = require('../lib/util/fs');
 const objectAssignDeep = require('object-assign-deep');
 const data = require('../lib/util/data');
 const configurationFile = data.joinPath('configuration.yaml');
+const utils = require('./utils');
 
 const mqtt = {
     subscribe: (topic) => {},
@@ -17,6 +18,7 @@ describe('BridgeConfig', () => {
     const files = new Map();
 
     beforeAll(() => {
+        utils.stubLogger(jest);
         jest.spyOn(fs, 'readYaml').mockImplementation((file) => {
             if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
             throw new Error(`Fake file not found: ${file}`);
@@ -137,4 +139,36 @@ describe('BridgeConfig', () => {
 
         bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/groups', 'whatever');
     });
+
+    it('Rename device with é', async () => {
+        jest.spyOn(mqtt, 'log').mockImplementation((type, message) => {
+            expect(type).toBe('device_renamed');
+        });
+
+        write(configurationFile, {
+            devices: {
+                '0x12345678': {
+                    friendly_name: 'test123',
+                    retain: false,
+                },
+            },
+        });
+
+        const payload = {
+            old: 'test123',
+            new: 'tést123',
+        };
+
+        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/rename', JSON.stringify(payload));
+        const expected = {
+            devices: {
+                '0x12345678': {
+                    friendly_name: 'tést123',
+                    retain: false,
+                },
+            },
+        };
+
+        expect(read(configurationFile)).toStrictEqual(expected);
+    });
 });

test/controller.test.js

@@ -45,6 +45,7 @@ describe('Controller', () => {
                 },
                 advanced: {
                     cache_state: false,
+                    last_seen: 'disable',
                 },
                 experimental: {
                     output: 'json',
@@ -69,8 +70,7 @@ describe('Controller', () => {
             controller.onZigbeeMessage(message);
             expect(mqttPublish).toHaveBeenCalledTimes(1);
             expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual(payload);
-        }
-        );
+        });
 
         it('Should output to json by default', () => {
             const payload = {temperature: 1, humidity: 2};
@@ -92,13 +92,92 @@ describe('Controller', () => {
                 },
             });
 
-            const payload = {temperature: 1, humidity: 2};
+            const payload = {
+                temperature: 1,
+                humidity: 2,
+                state: 'ON',
+                allowedStates: ['ON', 'OFF'],
+                color: {r: 100, g: 0, b: 102, a: 0},
+                nested: {
+                    state: 'OFF',
+                    color: {r: 1, g: 0, b: 2},
+                },
+            };
             controller.publishEntityState('0x12345678', payload);
-            expect(mqttPublish).toHaveBeenCalledTimes(2);
-            expect(mqttPublish.mock.calls[0][0]).toBe('test/temperature');
-            expect(mqttPublish.mock.calls[0][1]).toBe('1');
-            expect(mqttPublish.mock.calls[1][0]).toBe('test/humidity');
-            expect(mqttPublish.mock.calls[1][1]).toBe('2');
+            expect(mqttPublish.mock.calls.map((x) => [x[0], x[1]])).toEqual([
+                ['test/temperature', '1'],
+                ['test/humidity', '2'],
+                ['test/state', 'ON'],
+                ['test/allowedStates', 'ON,OFF'],
+                ['test/color', '100,0,102'],
+                ['test/nested-state', 'OFF'],
+                ['test/nested-color', '1,0,2'],
+            ]);
+        });
+
+        it('Should cache state', () => {
+            jest.spyOn(settings, 'get').mockReturnValue({
+                mqtt: {
+                    include_device_information: false,
+                },
+                advanced: {
+                    cache_state: true,
+                },
+                experimental: {
+                    output: 'json',
+                },
+            });
+
+            controller.publishEntityState('0x12345678', {temperature: 1});
+            controller.publishEntityState('0x12345678', {humidity: 2});
+            controller.publishEntityState('0x12345678', {humidity: 5, temperature: 4});
+            controller.publishEntityState('0x12345678', {temperature: 9});
+            controller.publishEntityState('0x12345678', {temperature: 3});
+            expect(mqttPublish).toHaveBeenCalledTimes(5);
+            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual({temperature: 1});
+            expect(JSON.parse(mqttPublish.mock.calls[1][1])).toStrictEqual({temperature: 1, humidity: 2});
+            expect(JSON.parse(mqttPublish.mock.calls[2][1])).toStrictEqual({temperature: 4, humidity: 5});
+            expect(JSON.parse(mqttPublish.mock.calls[3][1])).toStrictEqual({temperature: 9, humidity: 5});
+            expect(JSON.parse(mqttPublish.mock.calls[4][1])).toStrictEqual({temperature: 3, humidity: 5});
+        });
+
+        it('Shouldnt cache state', () => {
+            jest.spyOn(settings, 'get').mockReturnValue({
+                mqtt: {
+                    include_device_information: false,
+                },
+                advanced: {
+                    cache_state: false,
+                },
+                experimental: {
+                    output: 'json',
+                },
+            });
+
+            controller.publishEntityState('0x12345678', {temperature: 1});
+            controller.publishEntityState('0x12345678', {humidity: 2});
+            controller.publishEntityState('0x12345678', {temperature: 3});
+            expect(mqttPublish).toHaveBeenCalledTimes(3);
+            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual({temperature: 1});
+            expect(JSON.parse(mqttPublish.mock.calls[1][1])).toStrictEqual({humidity: 2});
+            expect(JSON.parse(mqttPublish.mock.calls[2][1])).toStrictEqual({temperature: 3});
+        });
+
+        it('Should not send empty messages', () => {
+            jest.spyOn(settings, 'get').mockReturnValue({
+                mqtt: {
+                    include_device_information: false,
+                },
+                advanced: {
+                    cache_state: true,
+                },
+                experimental: {
+                    output: 'json',
+                },
+            });
+
+            controller.publishEntityState('0x12345678', {});
+            expect(mqttPublish).toHaveBeenCalledTimes(0);
         });
     });
 });

test/data.test.js

@@ -1,8 +1,13 @@
 const data = require('../lib/util/data.js');
 const path = require('path');
+const utils = require('./utils');
 
 describe('Data', () => {
     describe('Get path', () => {
+        beforeEach(() => {
+            utils.stubLogger(jest);
+        });
+
         it('Should return correct path', () => {
             const expected = path.normalize(path.join(__dirname, '..', 'data'));
             const actual = data.getPath();

test/deviceBind.test.js

@@ -28,6 +28,11 @@ const devices = {
             return {inClusterList: [5, 6]};
         },
     },
+    occupancy_sensor: {
+        getSimpleDesc: () => {
+            return {outClusterList: [5, 6]};
+        },
+    },
 };
 
 const zigbee = {
@@ -46,12 +51,15 @@ const zigbee = {
             return devices.switch_ep2;
         } else if (ep == 3 && ID === 'switch_ep3') {
             return devices.switch_ep3;
+        } else if (ep == 2 && ID == 'occupancy_sensor') {
+            return devices.occupancy_sensor;
         }
 
         throw new Error(`No mock for ${ID} and ep ${ep}`);
     },
     getDevice: (ID) => {
         const lookup = {
+            'occupancy_sensor': 'SML002',
             'switch_ep2': 'lumi.sensor_86sw2.es1',
             'switch_ep3': 'DNCKAT_S003',
             'bulb': 'TRADFRI bulb E27 WS opal 980lm',
@@ -154,6 +162,17 @@ describe('DeviceBind', () => {
                 expect.any(Function)
             );
         });
+
+        it('Bind default ep when mapped', async () => {
+            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/occupancy_sensor', 'bulb');
+            expect(zigbee.bind).toHaveBeenCalledTimes(1);
+            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
+                devices.occupancy_sensor,
+                6,
+                devices.bulb,
+                expect.any(Function)
+            );
+        });
     });
 
     describe('Unbind devices', () => {

test/devicePublish.test.js

@@ -26,6 +26,11 @@ const cfg = {
         manufSpec: 0,
         disDefaultRsp: 0,
     },
+    disFeedbackRsp: {
+        manufSpec: 0,
+        disDefaultRsp: 0,
+        disFeedbackRsp: true,
+    },
 };
 
 describe('DevicePublish', () => {
@@ -481,7 +486,7 @@ describe('DevicePublish', () => {
                 'lightingColorCtrl',
                 'moveToColor',
                 'functional',
-                {colorx: 17085, colory: 44000, transtime: 0},
+                {colorx: 17806, colory: 43155, transtime: 0},
                 cfg.default,
                 null,
                 expect.any(Function));
@@ -490,7 +495,7 @@ describe('DevicePublish', () => {
             expect(publishEntityState).toHaveBeenCalledTimes(1);
             expect(publishEntityState).toHaveBeenNthCalledWith(1,
                 '0x00000016',
-                {color: {x: 0.2607, y: 0.6714}});
+                {color: {x: 0.2717, y: 0.6585}});
         });
 
         it('Should publish messages to zigbee devices with color rgb string', async () => {
@@ -505,7 +510,7 @@ describe('DevicePublish', () => {
                 'lightingColorCtrl',
                 'moveToColor',
                 'functional',
-                {colorx: 17085, colory: 44000, transtime: 0},
+                {colorx: 17806, colory: 43155, transtime: 0},
                 cfg.default,
                 null,
                 expect.any(Function));
@@ -514,7 +519,7 @@ describe('DevicePublish', () => {
             expect(publishEntityState).toHaveBeenCalledTimes(1);
             expect(publishEntityState).toHaveBeenNthCalledWith(1,
                 '0x00000017',
-                {color: {x: 0.2607, y: 0.6714}});
+                {color: {x: 0.2717, y: 0.6585}});
         }
         );
 
@@ -725,6 +730,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('my_device_id');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse get topic', () => {
@@ -733,6 +739,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('get');
             expect(parsed.ID).toBe('my_device_id2');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         });
 
 
@@ -768,6 +775,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('get');
             expect(parsed.ID).toBe('my_device_id2');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse topic with when deviceID has multiple slashes', () => {
@@ -776,6 +784,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('floor0/basement/my_device_id2');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse topic with when base and deviceID have multiple slashes', () => {
@@ -790,15 +799,26 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('floor0/basement/my_device_id2');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         }
         );
 
-        it('Should parse set with ieeAddr topic', () => {
+        it('Should parse set with attribute topic', () => {
+            const topic = 'zigbee2mqtt/0x12345689/set/foobar';
+            const parsed = devicePublish.parseTopic(topic);
+            expect(parsed.type).toBe('set');
+            expect(parsed.ID).toBe('0x12345689');
+            expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBe('foobar');
+        });
+
+        it('Should parse set with ieeeAddr topic', () => {
             const topic = 'zigbee2mqtt/0x12345689/set';
             const parsed = devicePublish.parseTopic(topic);
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('0x12345689');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse set with postfix topic', () => {
@@ -807,6 +827,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('0x12345689');
             expect(parsed.postfix).toBe('left');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse set with almost postfix topic', () => {
@@ -815,6 +836,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('wohnzimmer.light.wall.right');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse set with postfix topic', () => {
@@ -823,6 +845,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('0x12345689');
             expect(parsed.postfix).toBe('right');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Should parse set with postfix topic', () => {
@@ -831,6 +854,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('0x12345689');
             expect(parsed.postfix).toBe('bottom_left');
+            expect(parsed.attribute).toBeUndefined();
         });
 
         it('Shouldnt parse set with invalid postfix topic', () => {
@@ -839,6 +863,16 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('set');
             expect(parsed.ID).toBe('0x12345689/invalid');
             expect(parsed.postfix).toBe('');
+            expect(parsed.attribute).toBeUndefined();
+        });
+
+        it('Should parse set with postfix topic and attribute', () => {
+            const topic = 'zigbee2mqtt/0x12345689/bottom_left/set/foobar';
+            const parsed = devicePublish.parseTopic(topic);
+            expect(parsed.type).toBe('set');
+            expect(parsed.ID).toBe('0x12345689');
+            expect(parsed.postfix).toBe('bottom_left');
+            expect(parsed.attribute).toBe('foobar');
         });
 
         it('Should parse set with and slashes in base and deviceID postfix topic', () => {
@@ -853,6 +887,7 @@ describe('DevicePublish', () => {
             expect(parsed.type).toBe('get');
             expect(parsed.ID).toBe('my/device/in/basement/sensor');
             expect(parsed.postfix).toBe('bottom_left');
+            expect(parsed.attribute).toBeUndefined();
         }
         );
     });
@@ -896,11 +931,13 @@ describe('DevicePublish', () => {
         expect(zigbee.publish.mock.calls[1][2]).toBe('lightingColorCtrl');
         expect(zigbee.publish.mock.calls[1][3]).toBe('moveToColor');
         await wait(10);
-        expect(zigbee.publish).toHaveBeenCalledTimes(4);
-        expect(zigbee.publish.mock.calls[2][2]).toBe('genLevelCtrl');
+        expect(zigbee.publish).toHaveBeenCalledTimes(5);
+        expect(zigbee.publish.mock.calls[2][2]).toBe('genOnOff');
         expect(zigbee.publish.mock.calls[2][3]).toBe('read');
-        expect(zigbee.publish.mock.calls[3][2]).toBe('lightingColorCtrl');
+        expect(zigbee.publish.mock.calls[3][2]).toBe('genLevelCtrl');
         expect(zigbee.publish.mock.calls[3][3]).toBe('read');
+        expect(zigbee.publish.mock.calls[4][2]).toBe('lightingColorCtrl');
+        expect(zigbee.publish.mock.calls[4][3]).toBe('read');
         expect(publishEntityState).toHaveBeenCalledTimes(2);
         expect(publishEntityState).toHaveBeenNthCalledWith(1,
             '0x00000020',
@@ -1049,7 +1086,11 @@ describe('DevicePublish', () => {
     it('Home Assistant: should set state', async () => {
         zigbee.publish.mockClear();
         publishEntityState.mockClear();
-        jest.spyOn(settings, 'get').mockReturnValue({homeassistant: true, mqtt: {base_topic: 'zigbee2mqtt'}});
+        jest.spyOn(settings, 'get').mockReturnValue({
+            homeassistant: true,
+            mqtt: {base_topic: 'zigbee2mqtt'},
+            advanced: {last_seen: 'disable'},
+        });
         zigbee.getDevice = () => ({modelId: 'RB 185 C'});
         devicePublish.onMQTTMessage('zigbee2mqtt/0x00000001/set', JSON.stringify({state: 'ON'}));
         expect(zigbee.publish).toHaveBeenCalledTimes(1);
@@ -1073,7 +1114,11 @@ describe('DevicePublish', () => {
         zigbee.publish.mockClear();
         publishEntityState.mockClear();
         jest.spyOn(state, 'get').mockReturnValue({state: 'ON'});
-        jest.spyOn(settings, 'get').mockReturnValue({homeassistant: true, mqtt: {base_topic: 'zigbee2mqtt'}});
+        jest.spyOn(settings, 'get').mockReturnValue({
+            homeassistant: true,
+            mqtt: {base_topic: 'zigbee2mqtt'},
+            advanced: {last_seen: 'disable'},
+        });
         zigbee.getDevice = () => ({modelId: 'RB 185 C'});
         devicePublish.onMQTTMessage('zigbee2mqtt/0x00000001/set', JSON.stringify({state: 'ON', color_temp: 100}));
         expect(zigbee.publish).toHaveBeenCalledTimes(1);
@@ -1098,7 +1143,11 @@ describe('DevicePublish', () => {
         zigbee.publish.mockClear();
         publishEntityState.mockClear();
         jest.spyOn(state, 'get').mockReturnValue({state: 'ON'});
-        jest.spyOn(settings, 'get').mockReturnValue({homeassistant: true, mqtt: {base_topic: 'zigbee2mqtt'}});
+        jest.spyOn(settings, 'get').mockReturnValue({
+            homeassistant: true,
+            mqtt: {base_topic: 'zigbee2mqtt'},
+            advanced: {last_seen: 'disable'},
+        });
         zigbee.getDevice = () => ({modelId: 'RB 185 C'});
         devicePublish.onMQTTMessage(
             'zigbee2mqtt/0x00000001/set',
@@ -1126,7 +1175,11 @@ describe('DevicePublish', () => {
         zigbee.publish.mockClear();
         publishEntityState.mockClear();
         jest.spyOn(state, 'get').mockReturnValue({state: 'OFF'});
-        jest.spyOn(settings, 'get').mockReturnValue({homeassistant: true, mqtt: {base_topic: 'zigbee2mqtt'}});
+        jest.spyOn(settings, 'get').mockReturnValue({
+            homeassistant: true,
+            mqtt: {base_topic: 'zigbee2mqtt'},
+            advanced: {last_seen: 'disable'},
+        });
         zigbee.getDevice = () => ({modelId: 'RB 185 C'});
         devicePublish.onMQTTMessage(
             'zigbee2mqtt/0x00000001/set',
@@ -1162,4 +1215,123 @@ describe('DevicePublish', () => {
             '0x00000001',
             {color: {x: 0.41, y: 0.25}});
     });
+
+    it('Should publish message with disFeedbackRsp when set', async () => {
+        zigbee.publish.mockClear();
+        publishEntityState.mockClear();
+        zigbee.getDevice = () => ({modelId: 'HDC52EastwindFan'});
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000003/set', JSON.stringify({brightness: '92'}));
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenNthCalledWith(1,
+            '0x00000003',
+            'device',
+            'genLevelCtrl',
+            'moveToLevelWithOnOff',
+            'functional',
+            {level: 92, transtime: 0},
+            cfg.disFeedbackRsp,
+            null,
+            expect.any(Function));
+
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000003/set', JSON.stringify({state: 'OFF'}));
+        expect(zigbee.publish).toHaveBeenCalledTimes(2);
+        expect(zigbee.publish).toHaveBeenNthCalledWith(2,
+            '0x00000003',
+            'device',
+            'genOnOff',
+            'off',
+            'functional',
+            {},
+            cfg.disFeedbackRsp,
+            null,
+            expect.any(Function));
+    });
+
+    it('Should publish messages to zigbee devices', async () => {
+        jest.spyOn(settings, 'get').mockReturnValue({
+            mqtt: {base_topic: 'zigbee2mqtt'},
+            advanced: {last_seen: 'ISO_8601'},
+        });
+        publishEntityState.mockClear();
+        zigbee.getDevice = () => ({modelId: 'TRADFRI bulb E27 CWS opal 600lm'});
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000001/set', JSON.stringify({brightness: '200'}));
+        expect(publishEntityState).toHaveBeenCalledTimes(1);
+        expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('string');
+    });
+
+    it('HS2WD-E burglar warning', async () => {
+        zigbee.publish.mockClear();
+        publishEntityState.mockClear();
+        zigbee.getDevice = () => ({modelId: 'WarningDevice'});
+        const payload = {warning: {duration: 100, mode: 'burglar', strobe: true, level: 'high'}};
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000003/set', JSON.stringify(payload));
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenNthCalledWith(1,
+            '0x00000003',
+            'device',
+            'ssIasWd',
+            'startWarning',
+            'functional',
+            {startwarninginfo: 22, warningduration: 100},
+            cfg.default,
+            null,
+            expect.any(Function));
+    });
+
+    it('HS2WD-E emergency warning', async () => {
+        zigbee.publish.mockClear();
+        publishEntityState.mockClear();
+        zigbee.getDevice = () => ({modelId: 'WarningDevice'});
+        const payload = {warning: {duration: 10, mode: 'emergency', strobe: false, level: 'very_high'}};
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000003/set', JSON.stringify(payload));
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenNthCalledWith(1,
+            '0x00000003',
+            'device',
+            'ssIasWd',
+            'startWarning',
+            'functional',
+            {startwarninginfo: 51, warningduration: 10},
+            cfg.default,
+            null,
+            expect.any(Function));
+    });
+
+    it('HS2WD-E emergency without level', async () => {
+        zigbee.publish.mockClear();
+        publishEntityState.mockClear();
+        zigbee.getDevice = () => ({modelId: 'WarningDevice'});
+        const payload = {warning: {duration: 10, mode: 'emergency', strobe: false}};
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000003/set', JSON.stringify(payload));
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenNthCalledWith(1,
+            '0x00000003',
+            'device',
+            'ssIasWd',
+            'startWarning',
+            'functional',
+            {startwarninginfo: 49, warningduration: 10},
+            cfg.default,
+            null,
+            expect.any(Function));
+    });
+
+    it('HS2WD-E wrong payload (should use defaults)', async () => {
+        zigbee.publish.mockClear();
+        publishEntityState.mockClear();
+        zigbee.getDevice = () => ({modelId: 'WarningDevice'});
+        const payload = {warning: 'wrong'};
+        devicePublish.onMQTTMessage('zigbee2mqtt/0x00000003/set', JSON.stringify(payload));
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenNthCalledWith(1,
+            '0x00000003',
+            'device',
+            'ssIasWd',
+            'startWarning',
+            'functional',
+            {startwarninginfo: 53, warningduration: 10},
+            cfg.default,
+            null,
+            expect.any(Function));
+    });
 });

test/deviceReceive.test.js

@@ -40,6 +40,16 @@ describe('DeviceReceive', () => {
             expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'single'});
         });
 
+        it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', () => {
+            jest.spyOn(settings, 'get').mockReturnValue({homeassistant: true, advanced: {last_seen: 'disable'}});
+            const device = {ieeeAddr: '0x12345678'};
+            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
+            deviceReceive.onZigbeeMessage(message, device, WXKG11LM);
+            expect(publishEntityState).toHaveBeenCalledTimes(2);
+            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'single'});
+            expect(publishEntityState.mock.calls[1][1]).toStrictEqual({click: ''});
+        });
+
         it('Should handle a zigbee message which uses ep (left)', () => {
             const device = {ieeeAddr: '0x12345678'};
             const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);

test/group.test.js

@@ -0,0 +1,287 @@
+const Groups = require('../lib/extension/groups');
+const settings = require('../lib/util/settings');
+const utils = require('./utils');
+
+let groupExtension = null;
+let zigbee = null;
+
+describe('Groups', () => {
+    beforeEach(() => {
+        utils.stubLogger(jest);
+
+        zigbee = {
+            publish: jest.fn(),
+        };
+
+        groupExtension = new Groups(zigbee, null, null, null);
+    });
+
+    afterEach(() => {
+        jest.restoreAllMocks();
+    });
+
+    it('Apply group updates add', async () => {
+        zigbee.publish.mockClear();
+        const from = {};
+        const to = {'1': ['1', '2']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(2);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '2', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates remove', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1', '2', '3']};
+        const to = {'1': ['1']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(2);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '2', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates add 1', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1']};
+        const to = {'1': ['1', '2', '3']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(2);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '2', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates add and remove group', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1', '2']};
+        const to = {'2': ['1', '2']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(4);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '2', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '2', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates change 1', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1', '4', '2']};
+        const to = {'1': ['1', '2', '3']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(2);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '4', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates change 2', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1', '2', '3']};
+        const to = {'1': ['3', '1', '2', '4']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '4', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates change 3', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1', '2']};
+        const to = {'1': ['3', '1', '4', '2']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(2);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '4', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates change 4', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1']};
+        const to = {'2': ['3', '1']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(3);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates change 5', async () => {
+        zigbee.publish.mockClear();
+        const from = {'1': ['1']};
+        const to = {'1': ['3'], '2': ['3', '1']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(4);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '1', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '3', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates add with postfix', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        const from = {};
+        const to = {'1': ['0x12345689/right']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, 3, expect.any(Function)
+        );
+    });
+
+    it('Apply group updates add and remove with postfix', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        const from = {'1': ['0x12345689/right']};
+        const to = {'1': ['0x12345689'], '2': ['0x12345689/left']};
+        groupExtension.apply(from, to);
+        expect(zigbee.publish).toHaveBeenCalledTimes(3);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '2', groupname: ''}, null, 2, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, 3, expect.any(Function)
+        );
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
+        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
+        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/add', 'my_switch');
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
+        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
+        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/add', 'my_switch/right');
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'add', 'functional',
+            {groupid: '1', groupname: ''}, null, 3, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
+        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
+        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/remove', 'my_switch/left');
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'remove', 'functional',
+            {groupid: '1'}, null, 2, expect.any(Function)
+        );
+    });
+
+    it('Remove all group via MQTT', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
+        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/remove_all', 'my_switch');
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'removeAll', 'functional',
+            {}, null, null, expect.any(Function)
+        );
+    });
+
+    it('Remove all group via MQTT deprecated', async () => {
+        zigbee.publish.mockClear();
+        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
+        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
+        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
+        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
+        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/remove_all', 'my_switch');
+        expect(zigbee.publish).toHaveBeenCalledTimes(1);
+        expect(zigbee.publish).toHaveBeenCalledWith(
+            '0x12345689', 'device', 'genGroups', 'removeAll', 'functional',
+            {}, null, null, expect.any(Function)
+        );
+    });
+});

test/homeassistant.test.js

@@ -1,15 +1,19 @@
 const devices = require('zigbee-shepherd-converters').devices;
 const HomeassistantExtension = require('../lib/extension/homeassistant');
 const settings = require('../lib/util/settings');
+const utils = require('./utils');
 
 const WSDCGQ11LM = devices.find((d) => d.model === 'WSDCGQ11LM');
 const SV01 = devices.find((d) => d.model === 'SV01');
+const FAN99432 = devices.find((d) => d.model === '99432');
 
 describe('HomeAssistant extension', () => {
     let homeassistant = null;
     let mqtt = null;
 
     beforeEach(() => {
+        utils.stubLogger(jest);
+
         mqtt = {
             publish: jest.fn(),
         };
@@ -52,7 +56,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_temperature',
             'unique_id': '0x12345678_temperature_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -76,7 +80,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_humidity',
             'unique_id': '0x12345678_humidity_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -100,7 +104,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_pressure',
             'unique_id': '0x12345678_pressure_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -124,7 +128,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_battery',
             'unique_id': '0x12345678_battery_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -147,7 +151,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_linkquality',
             'unique_id': '0x12345678_linkquality_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -184,7 +188,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_temperature',
             'unique_id': '0x12345678_temperature_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -208,7 +212,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_humidity',
             'unique_id': '0x12345678_humidity_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -232,7 +236,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_pressure',
             'unique_id': '0x12345678_pressure_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -256,7 +260,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_battery',
             'unique_id': '0x12345678_battery_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -279,7 +283,7 @@ describe('HomeAssistant extension', () => {
             'name': 'my_device_linkquality',
             'unique_id': '0x12345678_linkquality_zigbee2mqtt',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -322,7 +326,7 @@ describe('HomeAssistant extension', () => {
             'expire_after': 90,
             'icon': 'mdi:test',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -348,7 +352,7 @@ describe('HomeAssistant extension', () => {
             'expire_after': 30,
             'icon': 'mdi:test',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -374,7 +378,7 @@ describe('HomeAssistant extension', () => {
             'expire_after': 30,
             'icon': 'mdi:test',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -400,7 +404,7 @@ describe('HomeAssistant extension', () => {
             'expire_after': 30,
             'icon': 'mdi:test',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -425,7 +429,7 @@ describe('HomeAssistant extension', () => {
             'expire_after': 30,
             'icon': 'mdi:test',
             'device': {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -440,6 +444,41 @@ describe('HomeAssistant extension', () => {
         expect(mqtt.publish.mock.calls[4][4]).toBe('homeassistant');
     });
 
+    it('Should discover devices with fan', () => {
+        let payload = null;
+        jest.spyOn(settings, 'getDevice').mockReturnValue({friendly_name: 'my_device'});
+
+        homeassistant.discover('0x12345678', FAN99432, false);
+        expect(mqtt.publish).toHaveBeenCalledTimes(3);
+
+        // 1
+        payload = {
+            name: 'my_device_fan',
+            state_topic: 'zigbee2mqtt/my_device',
+            state_value_template: '{{ value_json.fan_state }}',
+            command_topic: 'zigbee2mqtt/my_device/set/fan_state',
+            speed_state_topic: 'zigbee2mqtt/my_device',
+            speed_value_template: '{{ value_json.fan_mode }}',
+            speed_command_topic: 'zigbee2mqtt/my_device/set/fan_mode',
+            unique_id: '0x12345678_fan_zigbee2mqtt',
+            speeds: ['off', 'low', 'medium', 'high', 'on', 'auto', 'smart'],
+            device: {
+                'identifiers': ['zigbee2mqtt_0x12345678'],
+                'name': 'my_device',
+                'sw_version': 'Zigbee2mqtt test',
+                'manufacturer': 'Hampton Bay',
+                'model': 'Universal wink enabled white ceiling fan premier remote control (99432)',
+            },
+            availability_topic: 'zigbee2mqtt/bridge/state',
+            json_attributes_topic: 'zigbee2mqtt/my_device',
+        };
+
+        expect(JSON.parse(mqtt.publish.mock.calls[0][1])).toStrictEqual(payload);
+        expect(mqtt.publish.mock.calls[0][2]).toStrictEqual({retain: true, qos: 0});
+        expect(mqtt.publish.mock.calls[0][3]).toBeNull();
+        expect(mqtt.publish.mock.calls[0][4]).toBe('homeassistant');
+    });
+
     it('Should discover devices with cover_position', () => {
         let payload = null;
         jest.spyOn(settings, 'getDevice').mockReturnValue({friendly_name: 'my_device'});
@@ -457,7 +496,7 @@ describe('HomeAssistant extension', () => {
             value_template: '{{ value_json.position }}',
             unique_id: '0x12345678_cover_zigbee2mqtt',
             device: {
-                'identifiers': 'zigbee2mqtt_0x12345678',
+                'identifiers': ['zigbee2mqtt_0x12345678'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Smart vent (SV01)',
@@ -478,7 +517,7 @@ describe('HomeAssistant extension', () => {
             friendly_name: 'my_device',
             homeassistant: {
                 device: {
-                    identifiers: 'test123',
+                    identifiers: ['test123'],
                 },
                 temperature: {
                     expire_after: 90,
@@ -500,7 +539,7 @@ describe('HomeAssistant extension', () => {
             'unique_id': '0x12345678_temperature_zigbee2mqtt',
             'expire_after': 90,
             'device': {
-                'identifiers': 'test123',
+                'identifiers': ['test123'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -523,7 +562,7 @@ describe('HomeAssistant extension', () => {
                 temperature: {
                     expire_after: 90,
                     device: {
-                        identifiers: 'test',
+                        identifiers: ['test'],
                     },
                 },
             },
@@ -543,7 +582,7 @@ describe('HomeAssistant extension', () => {
             'unique_id': '0x12345678_temperature_zigbee2mqtt',
             'expire_after': 90,
             'device': {
-                'identifiers': 'test',
+                'identifiers': ['test'],
                 'name': 'my_device',
                 'sw_version': 'Zigbee2mqtt test',
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
@@ -567,7 +606,7 @@ describe('HomeAssistant extension', () => {
                 output: 'json',
             },
             advanced: {
-                homeassistant_discovery_topic: 'my_custom_topic',
+                homeassistant_discovery_topic: 'my_custom_discovery_topic',
             },
         });
 
@@ -579,7 +618,7 @@ describe('HomeAssistant extension', () => {
                 temperature: {
                     expire_after: 90,
                     device: {
-                        identifiers: 'test',
+                        identifiers: ['test'],
                     },
                 },
             },
@@ -588,6 +627,33 @@ describe('HomeAssistant extension', () => {
         homeassistant.discover('0x12345678', WSDCGQ11LM, false);
 
         expect(mqtt.publish).toHaveBeenCalledTimes(5);
-        expect(mqtt.publish.mock.calls[0][4]).toBe('my_custom_topic');
+        expect(mqtt.publish.mock.calls[0][4]).toBe('my_custom_discovery_topic');
+    });
+
+    it('Should subscribe to custom status topic', () => {
+        jest.spyOn(settings, 'get').mockReturnValue({
+            experimental: {
+                output: 'json',
+            },
+            advanced: {
+                homeassistant_status_topic: 'my_custom_status_topic',
+            },
+        });
+
+        const zigbee = {
+            getAllClients: jest.fn().mockReturnValue([]),
+        };
+
+        mqtt = {
+            subscribe: jest.fn(),
+        };
+
+
+        homeassistant = new HomeassistantExtension(zigbee, mqtt, null, null);
+
+        homeassistant.onMQTTConnected();
+
+        expect(mqtt.subscribe).toHaveBeenCalledTimes(1);
+        expect(mqtt.subscribe.mock.calls[0][0]).toBe('my_custom_status_topic');
     });
 });

test/settings.test.js

@@ -3,6 +3,7 @@ const objectAssignDeep = require('object-assign-deep');
 const data = require('../lib/util/data');
 const settings = require('../lib/util/settings.js');
 const fs = require('../lib/util/fs');
+const utils = require('./utils');
 
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
@@ -16,6 +17,8 @@ describe('Settings', () => {
     const files = new Map();
 
     beforeAll(() => {
+        utils.stubLogger(jest);
+
         jest.spyOn(fs, 'readYaml').mockImplementation((file) => {
             if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
             throw new Error(`Fake file not found: ${file}`);
@@ -203,12 +206,13 @@ describe('Settings', () => {
             const group = settings.getGroup('1');
             const expected = {
                 friendly_name: '123',
+                devices: [],
             };
 
             expect(group).toStrictEqual(expected);
         });
 
-        it('Should read groups form a separate file', () => {
+        it('Should read groups from a separate file', () => {
             const contentConfiguration = {
                 groups: 'groups.yaml',
             };
@@ -225,6 +229,7 @@ describe('Settings', () => {
             const group = settings.getGroup('1');
             const expected = {
                 friendly_name: '123',
+                devices: [],
             };
 
             expect(group).toStrictEqual(expected);
@@ -239,6 +244,7 @@ describe('Settings', () => {
             const contentGroups = {
                 '1': {
                     friendly_name: '123',
+                    devices: [],
                 },
             };
 
@@ -268,6 +274,7 @@ describe('Settings', () => {
             const group = settings.getGroup('1');
             const expectedGroup = {
                 friendly_name: '123',
+                devices: [],
             };
 
             expect(group).toStrictEqual(expectedGroup);
@@ -281,5 +288,35 @@ describe('Settings', () => {
 
             expect(settings.getDevice('0x1234')).toStrictEqual(expectedDevice2);
         });
+
+        it('Should add groups', () => {
+            write(configurationFile, {});
+
+            const added = settings.addGroup('test123');
+            const expected = {
+                '1': {
+                    friendly_name: 'test123',
+                },
+            };
+
+            expect(added).toStrictEqual(true);
+            expect(settings.get().groups).toStrictEqual(expected);
+        });
+
+        it('Should not add duplicate groups', () => {
+            write(configurationFile, {});
+
+            const added = settings.addGroup('test123');
+            const added2 = settings.addGroup('test123');
+            const expected = {
+                '1': {
+                    friendly_name: 'test123',
+                },
+            };
+
+            expect(added).toStrictEqual(true);
+            expect(added2).toStrictEqual(false);
+            expect(settings.get().groups).toStrictEqual(expected);
+        });
     });
 });

test/utils.js

@@ -6,6 +6,7 @@ module.exports = {
         jest.spyOn(logger, 'warn').mockReturnValue(undefined);
         jest.spyOn(logger, 'debug').mockReturnValue(undefined);
         jest.spyOn(logger, 'error').mockReturnValue(undefined);
+        jest.spyOn(logger, 'trace').mockReturnValue(undefined);
     },
     zigbeeMessage: (device, cid, type, data, epId, groupid=0) => {
         return {data: {cid, data}, type, groupid, endpoints: [{device, epId}]};

test/utils.test.js

@@ -0,0 +1,78 @@
+const utils = require('../lib/util/utils.js');
+const testUtils = require('./utils');
+
+describe('Utils', () => {
+    beforeAll(() => {
+        testUtils.stubLogger(jest);
+    });
+
+    describe('Is xiaomi device', () => {
+        it('Identify xiaomi device', () => {
+            const device = {type: 'Router', manufId: 4151, manufName: 'Xiaomi'};
+            expect(true).toBe(utils.isXiaomiDevice(device));
+        });
+
+        it('Identify xiaomi device without manufName', () => {
+            const device = {type: 'Router', manufId: 4447};
+            expect(true).toBe(utils.isXiaomiDevice(device));
+        });
+
+        it('Identify xiaomi device with different manufName', () => {
+            const device = {type: 'Router', manufId: 4151, manufName: 'Trust International B.V.\u0000'};
+            expect(false).toBe(utils.isXiaomiDevice(device));
+        });
+
+        it('Identify QBKG03LM as enddevice', () => {
+            const device = {type: 'Router', manufId: 4447, modelId: 'lumi.ctrl_neutral1'};
+            expect(false).toBe(utils.isRouter(device));
+            expect('EndDevice').toBe(utils.correctDeviceType(device));
+        });
+
+        it('Identify QBKG04LM as enddevice', () => {
+            const device = {type: 'Router', manufId: 4447, modelId: 'lumi.ctrl_neutral2'};
+            expect(false).toBe(utils.isRouter(device));
+            expect('EndDevice').toBe(utils.correctDeviceType(device));
+        });
+    });
+
+    describe('Get endpoint by id', () => {
+        it('Pick default ep', () => {
+            const zigbee = {
+                getDevice: (entityID) => {
+                    return {modelId: 'TRADFRI on/off switch'};
+                },
+                getEndpoint: (entityID, epId) => {
+                    return {epId: epId == null ? 1 : 0};
+                },
+            };
+            const endpoint = utils.getEndpointByEntityID(zigbee, '0x12345678', null);
+            expect(endpoint.epId).toBe(1);
+        });
+
+        it('Pick default ep from mapping when default defined', () => {
+            const zigbee = {
+                getDevice: (entityID) => {
+                    return {modelId: 'SML002'};
+                },
+                getEndpoint: (entityID, epId) => {
+                    return {epId};
+                },
+            };
+            const endpoint = utils.getEndpointByEntityID(zigbee, '0x12345678', null);
+            expect(endpoint.epId).toBe(2);
+        });
+
+        it('Pick default ep from mapping when not defined', () => {
+            const zigbee = {
+                getDevice: (entityID) => {
+                    return {modelId: 'lumi.sensor_86sw2.es1'};
+                },
+                getEndpoint: (entityID, epId) => {
+                    return {epId: epId == null ? 1 : 0};
+                },
+            };
+            const endpoint = utils.getEndpointByEntityID(zigbee, '0x12345678', null);
+            expect(endpoint.epId).toBe(1);
+        });
+    });
+});