0.1.7

* network map, more info

* add a get(All)Devices method to zigbee

* graphviz: display all devices

loop through all devices, display all devices even those that haven't
responded to the lqi scan.

* makes eslint happy :)

* remove null chars from network map graphviz output

makes graphviz happy :-)

* Improvements to graphviz network map

* Always add device type

.eslintrc.json

@@ -1,17 +1,20 @@
 {
     "env": {
+        "es6": true,
         "node": true
     },
     "extends": ["eslint:recommended", "google"],
     "parserOptions": {
-        "ecmaFeatures": {
-            "experimentalObjectRestSpread": true
-        },
+        "ecmaVersion": 2018,
         "sourceType": "module"
     },
     "rules": {
         "require-jsdoc": "off",
         "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }]
-    }
+        "max-len": ["error", { "code": 120 }],
+        "mocha/no-exclusive-tests": "error"
+    },
+    "plugins": [
+        "mocha"
+    ]
 }

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,11 @@
+Unsupported device
+First check: https://github.com/Koenkk/zigbee2mqtt/wiki/How-to-support-new-devices
+
+
+Bug report
+- First make sure your on the latest version of zigbee2mqtt.
+- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
+- Provide a clear description of the problem.
+- If applicable, provide steps how to reproduce the problem.
+- Provide a log with `log_level` set to `debug`, see https://github.com/Koenkk/zigbee2mqtt/wiki/Configuration
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.

.gitignore

@@ -60,4 +60,6 @@ typings/
 # data
 data/database.db
 data/config.json
-data/log*.txt
+data/log*.txt
+data/state.json
+data/log

.travis.yml

@@ -11,11 +11,12 @@ node_js:
 install:
   - npm install
 
-script:
+before_script:
+  - npm test
   - npm run verify-homeassistant-mapping
   - npm run eslint
 
-after_success:
+script:
   - ".travis/docker.sh"
   - ".travis/wiki.sh"
   - ".travis/trigger-downstream.sh"

.travis/trigger-downstream.sh

@@ -20,11 +20,11 @@ function travis_trigger() {
           "https://api.travis-ci.org/repo/${org}%2F${repo}/requests"
 }
 
-# Only trigger downstream if on master branch and not pull request
-if [ "$TRAVIS_BRANCH" = "master" -a "$TRAVIS_PULL_REQUEST" = "false" ]
+# Only trigger downstream if on dev branch and not pull request
+if [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
 then
   echo "Triggering build of downstream projects!"
   travis_trigger "danielwelch" "hassio-zigbee2mqtt" "master"
 else
-  echo "Not triggering build of downstream projects, triggered by pull request or not on master branch"
+  echo "Not triggering build of downstream projects, triggered by pull request or not on dev branch"
 fi

README.md

@@ -39,6 +39,6 @@ If you need assistance you can check [opened issues](https://github.com/Koenkk/z
 * [AndrewLinden](https://github.com/AndrewLinden)
 * [oskarn97](https://github.com/oskarn97)
 * [dgomes](https://github.com/dgomes)
-* [Koenkk](https://github.com/Koenk)
+* [Koenkk](https://github.com/Koenkk)
 * [kirovilya](https://github.com/kirovilya)
 * [ciotlosm](https://github.com/ciotlosm)

lib/controller.js

@@ -1,17 +1,19 @@
 const MQTT = require('./mqtt');
 const Zigbee = require('./zigbee');
+const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
+const ExtensionNetworkMap = require('./extension/networkMap');
+const ExtensionSoftReset = require('./extension/softReset');
+const ExtensionRouterPollXiaomi = require('./extension/routerPollXiaomi');
 const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
 const homeassistant = require('./homeassistant');
-const objectAssignDeep = require(`object-assign-deep`);
+const objectAssignDeep = require('object-assign-deep');
 
 const mqttConfigRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/\\w+`, 'g');
-const mqttDeviceRegex = new RegExp(`${settings.get().mqtt.base_topic}/[\\w\\s\\d]+/set`, 'g');
-const mqttDevicePrefixRegex = new RegExp(`${settings.get().mqtt.base_topic}/[\\w\\s\\d]+/[\\w\\s\\d]+/set`, 'g');
+const mqttDeviceRegex = new RegExp(`${settings.get().mqtt.base_topic}/[\\w\\s\\d.-]+/set`, 'g');
+const mqttDevicePrefixRegex = new RegExp(`${settings.get().mqtt.base_topic}/[\\w\\s\\d.-]+/[\\w\\s\\d.-]+/set`, 'g');
 
-const pollInterval = 60 * 1000; // seconds * 1000.
-const softResetTimeout = 3600 * 1000; // seconds * 1000.
 
 const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
 
@@ -30,17 +32,19 @@ if (settings.get().homeassistant && !cacheState) {
 
 class Controller {
     constructor() {
-        this.zigbee = new Zigbee();
-        this.mqtt = new MQTT();
-        this.stateCache = {};
-        this.configured = [];
         this.handleZigbeeMessage = this.handleZigbeeMessage.bind(this);
         this.handleMQTTMessage = this.handleMQTTMessage.bind(this);
+
+        this.zigbee = new Zigbee(this.handleZigbeeMessage);
+        this.mqtt = new MQTT();
+        this.state = new State();
+        this.configured = [];
+        this.extensions = [];
     }
 
     start() {
         this.startupLogVersion(() => {
-            this.zigbee.start(this.handleZigbeeMessage, (error) => {
+            this.zigbee.start((error) => {
                 if (error) {
                     logger.error('Failed to start', error);
                 } else {
@@ -57,12 +61,9 @@ class Controller {
                         logger.warn('`permit_join` set to  `true` in configuration.yaml.');
                         logger.warn('Allowing new devices to join.');
                         logger.warn('Set `permit_join` to `false` once you joined all devices.');
-                        this.zigbee.permitJoin(true);
                     }
 
-                    // Start timers.
-                    this.pollTimer(true);
-                    this.softResetTimeout(true);
+                    this.zigbee.permitJoin(settings.get().permit_join);
 
                     // Connect to MQTT broker
                     const subscriptions = [
@@ -93,73 +94,38 @@ class Controller {
             });
         }
 
+        // Initialize extensions.
+        this.extensions = [
+            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state),
+            new ExtensionSoftReset(this.zigbee, this.mqtt, this.state),
+            new ExtensionRouterPollXiaomi(this.zigbee, this.mqtt, this.state),
+        ];
+
         // Resend all cached states.
         this.sendAllCachedStates();
     }
 
     sendAllCachedStates() {
         this.zigbee.getAllClients().forEach((device) => {
-            if (this.stateCache.hasOwnProperty(device.ieeeAddr)) {
-                this.mqttPublishDeviceState(device.ieeeAddr, this.stateCache[device.ieeeAddr], false);
+            if (this.state.exists(device.ieeeAddr)) {
+                this.mqttPublishDeviceState(device, this.state.get(device.ieeeAddr), false);
             }
         });
     }
 
-    softResetTimeout(start) {
-        if (this._softResetTimer) {
-            clearTimeout(this._softResetTimer);
-            this._softResetTimer = null;
-        }
-
-        if (start) {
-            this._softResetTimer = setTimeout(() => {
-                this.zigbee.softReset((error) => {
-                    if (error) {
-                        logger.warn('Soft reset error', error);
-                        this.zigbee.stop((error) => {
-                            logger.warn('Zigbee stopped');
-                            this.zigbee.start(this.handleZigbeeMessage, (error) => {
-                                if (error) {
-                                    logger.error('Failed to restart!');
-                                }
-                            });
-                        });
-                    } else {
-                        logger.warn('Soft resetted zigbee');
-                    }
-
-                    this.softResetTimeout(true);
-                });
-            }, softResetTimeout);
-        }
-    }
-
-    pollTimer(start) {
-        // Some routers need polling to prevent them from sleeping.
-        if (start && !this._pollTimer) {
-            this._pollTimer = setInterval(() => {
-                const devices = this.zigbee.getAllClients().filter((d) => {
-                    const power = d.powerSource ? d.powerSource.toLowerCase().split(' ')[0] : 'unknown';
-                    return power !== 'battery' && power !== 'unknown' && d.type === 'Router';
-                });
-
-                devices.forEach((d) => this.zigbee.ping(d.ieeeAddr));
-            }, pollInterval);
-        } else if (!start && this._pollTimer) {
-            clearTimeout(this._pollTimer);
-            this._pollTimer = null;
-        }
-    }
-
     stop(callback) {
+        this.extensions.filter((e) => e.stop).forEach((e) => e.stop());
+        this.state.save();
         this.mqtt.disconnect();
-        this.pollTimer(false);
-        this.softResetTimeout(false);
         this.zigbee.stop(callback);
     }
 
     configureDevice(device) {
+        let friendlyName = 'unknown';
         const ieeeAddr = device.ieeeAddr;
+        if (settings.getDevice(ieeeAddr)) {
+            friendlyName = settings.getDevice(ieeeAddr).friendly_name;
+        }
         if (ieeeAddr && device.modelId && !this.configured.includes(ieeeAddr)) {
             const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
 
@@ -167,9 +133,9 @@ class Controller {
             if (mappedModel && mappedModel.configure) {
                 mappedModel.configure(ieeeAddr, this.zigbee.shepherd, this.zigbee.getCoordinator(), (ok, msg) => {
                     if (ok) {
-                        logger.info(`Succesfully configured ${ieeeAddr}`);
+                        logger.info(`Succesfully configured ${friendlyName} ${ieeeAddr}`);
                     } else {
-                        logger.error(`Failed to configure ${ieeeAddr}`);
+                        logger.error(`Failed to configure ${friendlyName} ${ieeeAddr}`);
                     }
                 });
             }
@@ -204,11 +170,36 @@ class Controller {
             `${friendlyDevice.vendor} ${friendlyDevice.description} (${type})`;
     }
 
-    handleZigbeeMessage(message) {
-        // Zigbee message receieved, reset soft reset timeout.
-        this.softResetTimeout(true);
+    getDeviceInfoForMqtt(device) {
+        const {type, ieeeAddr, nwkAddr, manufId, manufName, powerSource, modelId, status} = device;
+        const deviceSettings = settings.getDevice(device.ieeeAddr);
+
+        return {
+            ieeeAddr,
+            friendlyName: deviceSettings.friendly_name || '',
+            type,
+            nwkAddr,
+            manufId,
+            manufName,
+            powerSource,
+            modelId,
+            status,
+        };
+    }
+
+    handleZigbeeMessage(message) {
+        // Call extensions.
+        this.extensions.filter((e) => e.handleZigbeeMessage).forEach((e) => e.handleZigbeeMessage(message));
+
+        // Log the message.
+        let logMessage = `Recieved zigbee message of type '${message.type}' ` +
+                         `with data '${JSON.stringify(message.data)}'`;
+        if (message.endpoints && message.endpoints[0].device) {
+            const device = message.endpoints[0].device;
+            logMessage += ` of device '${device.modelId}' (${device.ieeeAddr})`;
+        }
+        logger.debug(logMessage);
 
-        logger.debug('Recieved zigbee message with data', JSON.stringify(message.data));
         if (message.type == 'devInterview' && !settings.getDevice(message.data)) {
             logger.info('Connecting with device...');
             this.mqtt.log('pairing', 'connecting with device');
@@ -312,7 +303,12 @@ class Controller {
                     }
                 });
 
-                this.mqttPublishDeviceState(device.ieeeAddr, payload, cache);
+                // Add device linkquality.
+                if (message.hasOwnProperty('linkquality')) {
+                    payload.linkquality = message.linkquality;
+                }
+
+                this.mqttPublishDeviceState(device, payload, cache);
             };
 
             const payload = converter.convert(mappedModel, message, publish, settings.getDevice(device.ieeeAddr));
@@ -326,6 +322,12 @@ class Controller {
     handleMQTTMessage(topic, message) {
         logger.debug(`Recieved mqtt message on topic '${topic}' with data '${message}'`);
 
+        // Find extensions that could handle this.
+        const extensions = this.extensions.filter((e) => e.handleMQTTMessage);
+
+        // Call extensions.
+        const extensionResults = extensions.map((e) => e.handleMQTTMessage(topic, message));
+
         if (topic.match(mqttConfigRegex)) {
             this.handleMQTTMessageConfig(topic, message);
         } else if (topic.match(mqttDeviceRegex) || topic.match(mqttDevicePrefixRegex)) {
@@ -337,13 +339,13 @@ class Controller {
                     clearTimeout(timer);
                 }, 20000);
             }
-        } else {
+        } else if (!extensionResults.includes(true)) {
             logger.warn(`Cannot handle MQTT message with topic '${topic}' and message '${message}'`);
         }
     }
 
     handleMQTTMessageConfig(topic, message) {
-        const option = topic.split('/')[3];
+        const option = topic.split('/').slice(-1)[0];
 
         if (option === 'permit_join') {
             this.zigbee.permitJoin(message.toString().toLowerCase() === 'true');
@@ -388,6 +390,9 @@ class Controller {
                 // Remove from configuration.yaml
                 settings.removeDevice(deviceID);
 
+                // Remove from state
+                this.state.remove(deviceID);
+
                 logger.info(`Successfully removed ${deviceID}`);
                 this.mqtt.log('device_removed', message);
             };
@@ -444,8 +449,8 @@ class Controller {
     }
 
     handleMQTTMessageDevice(topic, message, withPrefix) {
-        const friendlyName = topic.split('/')[1];
-        const topicPrefix = withPrefix ? topic.split('/')[2] : '';
+        const friendlyName = topic.split('/').slice(withPrefix ? -3 : -2)[0];
+        const topicPrefix = withPrefix ? topic.split('/').slice(-2)[0] : '';
 
         // Map friendlyName to deviceID.
         const deviceID = settings.getIDByFriendlyName(friendlyName);
@@ -498,15 +503,16 @@ class Controller {
 
             const callback = (error) => {
                 // Devices do not report when they go off, this ensures state (on/off) is always in sync.
-                if (!error && key.startsWith('state')) {
+                if (!error && (key.startsWith('state') || key === 'brightness')) {
                     const msg = {};
-                    const _key = topicPrefix ? `${key}_${topicPrefix}` : key;
-                    msg[_key] = json[key];
-                    this.mqttPublishDeviceState(deviceID, msg, true);
+                    const _key = topicPrefix ? `state_${topicPrefix}` : 'state';
+                    msg[_key] = key === 'brightness' ? 'ON' : json['state'];
+                    this.mqttPublishDeviceState(device, msg, true);
                 }
             };
 
-            this.zigbee.publish(deviceID, message.cid, message.cmd, message.zclData, ep, callback);
+            this.zigbee.publish(deviceID, message.cid, message.cmd, message.zclData,
+                message.cfg, ep, message.type, callback);
 
             published.push({message: message, converter: converter});
         });
@@ -541,26 +547,35 @@ class Controller {
         });
     }
 
-    mqttPublishDeviceState(deviceID, payload, cache) {
+    mqttPublishDeviceState(device, payload, cache) {
+        const deviceID = device.ieeeAddr;
+        const appSettings = settings.get();
+        let messagePayload = {...payload};
+
         if (cacheState) {
             // Add cached state to payload
-            if (this.stateCache[deviceID]) {
-                payload = objectAssignDeep.noMutate(this.stateCache[deviceID], payload);
+            if (this.state.exists(deviceID)) {
+                messagePayload = objectAssignDeep.noMutate(this.state.get(deviceID), payload);
             }
 
             // Update state cache with new state.
             if (cache) {
-                this.stateCache[deviceID] = payload;
+                this.state.set(deviceID, payload);
             }
         }
 
         const deviceSettings = settings.getDevice(deviceID);
+        const friendlyName = deviceSettings ? deviceSettings.friendly_name : deviceID;
         const options = {
-            retain: deviceSettings.retain,
-            qos: deviceSettings.qos ? deviceSettings.qos : 0,
+            retain: deviceSettings ? deviceSettings.retain : false,
+            qos: deviceSettings && deviceSettings.qos ? deviceSettings.qos : 0,
         };
 
-        this.mqtt.publish(deviceSettings.friendly_name, JSON.stringify(payload), options);
+        if (appSettings.mqtt.include_device_information) {
+            messagePayload.device = this.getDeviceInfoForMqtt(device);
+        }
+
+        this.mqtt.publish(friendlyName, JSON.stringify(messagePayload), options);
     }
 
     startupLogVersion(callback) {

lib/extension/networkMap.js

@@ -0,0 +1,87 @@
+
+const settings = require('../util/settings');
+const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+
+class NetworkMap {
+    constructor(zigbee, mqtt, state) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+
+        // Subscribe to topic.
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.mqtt.subscribe(this.topic);
+
+        // Set supported formats
+        this.supportedFormats = {
+            'raw': this.raw,
+            'graphviz': this.graphviz,
+        };
+    }
+
+    handleMQTTMessage(topic, message) {
+        message = message.toString();
+
+        if (topic === this.topic && this.supportedFormats.hasOwnProperty(message)) {
+            this.zigbee.networkScan((result)=> {
+                const converted = this.supportedFormats[message](this.zigbee, result);
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            });
+
+            return true;
+        }
+    }
+
+    raw(zigbee, topology) {
+        return JSON.stringify(topology);
+    }
+
+    graphviz(zigbee, topology) {
+        let text = 'digraph G {\nnode[shape=record];\n';
+        const lqiDevices = new Map(topology.map((d) => [d.ieeeAddr, d]));
+
+        zigbee.getDevices().forEach((device) => {
+            const labels = [];
+            const friendlyDevice = settings.getDevice(device.ieeeAddr);
+            const friendlyName = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
+
+            // Add friendly name
+            labels.push(friendlyName);
+
+            // Add the device type
+            labels.push(device.type);
+
+            // Add the device model
+            const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
+            if (mappedModel) {
+                labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
+            } else {
+                // This model is not supported by zigbee-shepherd-converters, add zigbee model information, if available
+                const zigbeeModel = [device.manufName, device.modelId].filter((a) => a).join(' ');
+                labels.push(zigbeeModel ? zigbeeModel : 'No model information available');
+            }
+
+            // Add the device status (online/offline)
+            labels.push(device.status);
+
+            // Add the device with its labels to the graph as a node.
+            text += `  "${device.ieeeAddr}" [label="{${labels.join('|')}}"];\n`;
+
+            /**
+             * Add an edge between the device and its parent to the graph
+             * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+             * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+             */
+            const lqiDevice = lqiDevices.get(device.ieeeAddr);
+            if (lqiDevice != undefined) {
+                text += `  "${device.ieeeAddr}" -> "${lqiDevice.parent}" [label="${lqiDevice.lqi}"]\n`;
+            }
+        });
+
+        text += '}';
+
+        return text.replace(/\0/g, '');
+    }
+}
+
+module.exports = NetworkMap;

lib/extension/routerPollXiaomi.js

@@ -0,0 +1,40 @@
+const utils = require('../util/utils');
+const interval = utils.secondsToMilliseconds(60);
+
+/**
+ * This extensions polls Xiaomi Zigbee routers to keep them awake.
+ */
+class RouterPollXiaomi {
+    constructor(zigbee, mqtt, state) {
+        this.zigbee = zigbee;
+        this.timer = null;
+
+        this.startTimer();
+    }
+
+    startTimer() {
+        this.clearTimer();
+        this.timer = setInterval(() => this.handleInterval(), interval);
+    }
+
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+
+    stop() {
+        this.clearTimer();
+    }
+
+    handleInterval() {
+        this.zigbee.getAllClients()
+            .filter((d) => utils.isXiaomiDevice(d)) // Filter Xiaomi devices
+            .filter((d) => d.type === 'Router') // Filter routers
+            .filter((d) => d.powerSource && d.powerSource !== 'Battery') // Remove battery powered devices
+            .forEach((d) => this.zigbee.ping(d.ieeeAddr)); // Ping devices.
+    }
+}
+
+module.exports = RouterPollXiaomi;

lib/extension/softReset.js

@@ -0,0 +1,66 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+
+/**
+ * This extensions soft resets the ZNP after a certain timeout.
+ */
+class SoftReset {
+    constructor(zigbee, mqtt, state) {
+        this.zigbee = zigbee;
+        this.timer = null;
+        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
+
+        if (this.timeout === 0) {
+            logger.debug(`Soft reset timeout disabled`);
+        } else {
+            logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+        }
+
+        this.resetTimer();
+    }
+
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+
+    resetTimer() {
+        if (this.timeout === 0) {
+            return;
+        }
+
+        this.clearTimer();
+        this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
+    }
+
+    handleTimeout() {
+        logger.warn('Soft reset timeout triggered');
+
+        this.zigbee.softReset((error) => {
+            if (error) {
+                logger.warn('Soft reset failed, trying stop/start');
+                this.zigbee.stop((error) => {
+                    logger.warn('Zigbee stopped');
+                    this.zigbee.start((error) => {
+                        if (error) {
+                            logger.error('Failed to restart!');
+                        }
+                    });
+                });
+            } else {
+                logger.warn('Soft resetted ZNP due to timeout');
+            }
+
+            this.resetTimer();
+        });
+    }
+
+    handleZigbeeMessage(message) {
+        this.resetTimer();
+    }
+}
+
+module.exports = SoftReset;

lib/homeassistant.js

@@ -46,6 +46,17 @@ const configurations = {
             json_attributes: ['battery', 'voltage'],
         },
     },
+    'binary_sensor_gas': {
+        type: 'binary_sensor',
+        object_id: 'gas',
+        discovery_payload: {
+            payload_on: true,
+            payload_off: false,
+            value_template: '{{ value_json.gas }}',
+            device_class: 'gas',
+            json_attributes: ['sensitivity'],
+        },
+    },
     'binary_sensor_router': {
         type: 'binary_sensor',
         object_id: 'router',
@@ -106,6 +117,7 @@ const configurations = {
             icon: 'mdi:toggle-switch',
             value_template: '{{ value_json.click }}',
             json_attributes: ['battery', 'voltage', 'action', 'duration'],
+            force_update: true,
         },
     },
     'sensor_power': {
@@ -124,7 +136,11 @@ const configurations = {
         discovery_payload: {
             icon: 'mdi:gesture-double-tap',
             value_template: '{{ value_json.action }}',
-            json_attributes: ['battery', 'voltage', 'angle', 'side', 'from_side', 'to_side', 'brightness'],
+            json_attributes: [
+                'battery', 'voltage', 'angle', 'side', 'from_side', 'to_side', 'brightness',
+                'angle_x_absolute', 'angle_y_absolute', 'angle_z', 'angle_y', 'angle_x', 'unknown_data',
+            ],
+            force_update: true,
         },
     },
     'sensor_brightness': {
@@ -137,6 +153,15 @@ const configurations = {
             json_attributes: [],
         },
     },
+    'sensor_lock': {
+        type: 'sensor',
+        object_id: 'lock',
+        discovery_payload: {
+            icon: 'mdi:lock',
+            value_template: '{{ value_json.inserted }}',
+            json_attributes: ['forgotten', 'keyerror'],
+        },
+    },
 
     // Light
     'light_brightness_colortemp_xy': {
@@ -191,28 +216,21 @@ const configurations = {
             command_topic: true,
         },
     },
-    'switch_left': {
+};
+
+const switchWithPostfix = (postfix) => {
+    return {
         type: 'switch',
-        object_id: 'switch_left',
+        object_id: `switch_${postfix}`,
         discovery_payload: {
             payload_off: 'OFF',
             payload_on: 'ON',
-            value_template: '{{ value_json.state_left }}',
+            value_template: `{{ value_json.state_${postfix} }}`,
             command_topic: true,
-            command_topic_prefix: 'left',
+            command_topic_prefix: postfix,
+            json_attributes: [`button_${postfix}`],
         },
-    },
-    'switch_right': {
-        type: 'switch',
-        object_id: 'switch_right',
-        discovery_payload: {
-            payload_off: 'OFF',
-            payload_on: 'ON',
-            value_template: '{{ value_json.state_right }}',
-            command_topic: true,
-            command_topic_prefix: 'right',
-        },
-    },
+    };
 };
 
 // Map homeassitant configurations to devices.
@@ -223,7 +241,7 @@ const mapping = {
     'WXKG03LM': [configurations.sensor_click],
     'WXKG02LM': [configurations.sensor_click],
     'QBKG04LM': [configurations.switch],
-    'QBKG03LM': [configurations.switch_left, configurations.switch_right],
+    'QBKG03LM': [switchWithPostfix('left'), switchWithPostfix('right')],
     'WSDCGQ01LM': [configurations.sensor_temperature, configurations.sensor_humidity],
     'WSDCGQ11LM': [configurations.sensor_temperature, configurations.sensor_humidity, configurations.sensor_pressure],
     'RTCGQ01LM': [configurations.binary_sensor_occupancy],
@@ -247,6 +265,7 @@ const mapping = {
     'PLUG EDP RE:DY': [configurations.switch, configurations.sensor_power],
     'CC2530.ROUTER': [configurations.binary_sensor_router],
     'AA70155': [configurations.light_brightness_colortemp],
+    '4058075816718': [configurations.light_brightness_colortemp_xy],
     'AA69697': [configurations.light_brightness_colortemp_xy],
     'HALIGHTDIMWWE27': [configurations.light_brightness],
     'AB3257001NJ': [configurations.switch],
@@ -281,7 +300,7 @@ const mapping = {
     'SL 110 W': [configurations.light_brightness],
     'AA68199': [configurations.light_brightness_colortemp],
     'QBKG11LM': [configurations.switch, configurations.sensor_power],
-    'QBKG12LM': [configurations.switch_left, configurations.switch_right, configurations.sensor_power],
+    'QBKG12LM': [switchWithPostfix('left'), switchWithPostfix('right'), configurations.sensor_power],
     'K2RGBW01': [configurations.light_brightness_colortemp_xy],
     '9290011370': [configurations.light_brightness],
     'DNCKATSW001': [configurations.switch],
@@ -293,11 +312,47 @@ const mapping = {
     '8718696598283': [configurations.light_brightness_colortemp],
     '73693': [configurations.light_brightness_colortemp_xy],
     '324131092621': [configurations.sensor_action],
+    '9290012607': [
+        configurations.binary_sensor_occupancy, configurations.sensor_temperature,
+        configurations.sensor_illuminance,
+    ],
     'GL-C-008': [configurations.light_brightness_colortemp_xy],
     'STSS-MULT-001': [configurations.binary_sensor_contact],
     'E11-G23': [configurations.light_brightness],
-    'AC03845': [configurations.light_brightness_colortemp_xy],
+    'AC03645': [configurations.light_brightness_colortemp_xy],
     'AC03641': [configurations.light_brightness],
+    'FB56+ZSW05HG1.2': [configurations.switch],
+    '72922-A': [configurations.switch],
+    'AC03642': [configurations.light_brightness_colortemp],
+    'DNCKATSW002': [switchWithPostfix('left'), switchWithPostfix('right')],
+    'DNCKATSW003': [switchWithPostfix('left'), switchWithPostfix('right'), switchWithPostfix('center')],
+    'DNCKATSW004': [
+        switchWithPostfix('bottom_left'), switchWithPostfix('bottom_right'),
+        switchWithPostfix('top_left'), switchWithPostfix('top_right'),
+    ],
+    'BY 165': [configurations.light_brightness],
+    'ZLED-2709': [configurations.light_brightness],
+    '8718696548738': [configurations.light_brightness_colortemp],
+    '4052899926110': [configurations.light_brightness_colortemp_xy],
+    'Z01-CIA19NAE26': [configurations.light_brightness],
+    'E11-N1EA': [configurations.light_brightness_colortemp_xy],
+    '74283': [configurations.light_brightness],
+    'JTQJ-BF-01LM/BW': [configurations.binary_sensor_gas],
+    '50045': [configurations.light_brightness],
+    'AV2010/22': [configurations.binary_sensor_occupancy],
+    '3210-L': [configurations.switch],
+    '7299355PH': [configurations.light_brightness_xy],
+    '45857GE': [configurations.light_brightness],
+    'A6121': [configurations.sensor_lock],
+    '433714': [configurations.light_brightness],
+    '3261030P7': [configurations.light_brightness_colortemp],
+    'DJT11LM': [configurations.sensor_action],
+    'E1603': [configurations.switch],
+    '7199960PH': [configurations.light_brightness_xy],
+    '74696': [configurations.light_brightness],
+    'AB35996': [configurations.light_brightness_colortemp_xy],
+    'AB401130055': [configurations.light_brightness_colortemp],
+    '74282': [configurations.light_brightness_colortemp],
 };
 
 // A map of all discoverd devices
@@ -314,7 +369,7 @@ function discover(deviceID, model, mqtt, force) {
 
     mapping[model].forEach((config) => {
         const topic = `${config.type}/${deviceID}/${config.object_id}/config`;
-        const payload = config.discovery_payload;
+        const payload = {...config.discovery_payload};
         payload.state_topic = `${settings.get().mqtt.base_topic}/${friendlyName}`;
         payload.availability_topic = `${settings.get().mqtt.base_topic}/bridge/state`;
 

lib/mqtt.js

@@ -12,7 +12,14 @@ class MQTT {
         const mqttSettings = settings.get().mqtt;
         logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
 
-        const options = {};
+        const options = {
+            will: {
+                topic: this.prefixTopic('bridge/state'),
+                payload: 'offline',
+                retain: true,
+            },
+        };
+
         if (mqttSettings.user && mqttSettings.password) {
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
@@ -23,6 +30,11 @@ class MQTT {
             options.clientId = mqttSettings.client_id;
         }
 
+        if (mqttSettings.hasOwnProperty('reject_unauthorized') && !mqttSettings.reject_unauthorized) {
+            logger.debug(`MQTT reject_unauthorized set false, ignoring certificate warnings.`);
+            options.rejectUnauthorized = false;
+        }
+
         this.client = mqtt.connect(mqttSettings.server, options);
 
         // Register callbacks.
@@ -58,7 +70,11 @@ class MQTT {
     handleConnect() {
         logger.info('Connected to MQTT server');
         this.publish('bridge/state', 'online', {retain: true, qos: 0});
-        this.subscriptions.forEach((topic) => this.client.subscribe(topic));
+        this.subscriptions.forEach((topic) => this.subscribe(topic));
+    }
+
+    subscribe(topic) {
+        this.client.subscribe(topic);
     }
 
     handleMessage(topic, message) {
@@ -68,8 +84,7 @@ class MQTT {
     }
 
     publish(topic, payload, options, callback, baseTopic) {
-        baseTopic = baseTopic ? baseTopic : settings.get().mqtt.base_topic;
-        topic = `${baseTopic}/${topic}`;
+        topic = this.prefixTopic(topic, baseTopic);
 
         if (!this.client || this.client.reconnecting) {
             logger.error(`Not connected to MQTT server!`);
@@ -81,6 +96,11 @@ class MQTT {
         this.client.publish(topic, payload, options, callback);
     }
 
+    prefixTopic(topic, baseTopic) {
+        baseTopic = baseTopic ? baseTopic : settings.get().mqtt.base_topic;
+        return `${baseTopic}/${topic}`;
+    }
+
     log(type, message) {
         const payload = {type: type, message: message};
         this.publish('bridge/log', JSON.stringify(payload), {retain: false});

lib/state.js

@@ -0,0 +1,55 @@
+const logger = require('./util/logger');
+const data = require('./util/data');
+const fs = require('fs');
+
+const saveInterval = 1000 * 60 * 5; // 5 minutes
+
+class State {
+    constructor() {
+        this.state = {};
+        this.file = data.joinPath('state.json');
+        this._load();
+
+        // Save the state on every interval
+        this.timer = setInterval(() => this.save(), saveInterval);
+    }
+
+    _load() {
+        if (fs.existsSync(this.file)) {
+            try {
+                this.state = JSON.parse(fs.readFileSync(this.file, 'utf8'));
+                logger.debug(`Loaded state from file ${this.file}`);
+            } catch (e) {
+                logger.debug(`Failed to load state from file ${this.file} (corrupt file?)`);
+            }
+        } else {
+            logger.debug(`Can't load state from file ${this.file} (doesn't exsist)`);
+        }
+    }
+
+    save() {
+        logger.debug(`Saving state to file ${this.file}`);
+        const json = JSON.stringify(this.state, null, 4);
+        fs.writeFileSync(this.file, json, 'utf8');
+    }
+
+    exists(ID) {
+        return this.state.hasOwnProperty(ID);
+    }
+
+    get(ID) {
+        return this.state[ID];
+    }
+
+    set(ID, state) {
+        this.state[ID] = state;
+    }
+
+    remove(ID) {
+        if (this.exists(ID)) {
+            delete this.state[ID];
+        }
+    }
+}
+
+module.exports = State;

lib/util/data.js

@@ -11,4 +11,5 @@ if (process.env.ZIGBEE2MQTT_DATA) {
 
 module.exports = {
     joinPath: (file) => path.join(dataPath, file),
+    getPath: () => dataPath,
 };

lib/util/logger.js

@@ -1,37 +1,62 @@
 const winston = require('winston');
-const data = require('./data');
+const moment = require('moment');
 const settings = require('./settings');
+const path = require('path');
+const fs = require('fs');
+const fx = require('mkdir-recursive');
+const rimraf = require('rimraf');
 
-let logLevel = '';
-if (process.env.DEBUG) {
-    logLevel = 'debug';
-} else if (settings.get().advanced && settings.get().advanced.log_level) {
-    logLevel = settings.get().advanced.log_level;
-} else {
-    logLevel = winston.level.info;
-}
+// Determine the log level.
+const level = settings.get().advanced.log_level;
 
+// Directoy to log to
+const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+
+// Make sure that log directoy exsists
+fx.mkdirSync(directory);
+
+// Create logger
 const logger = new (winston.Logger)({
     transports: [
         new (winston.transports.File)({
-            filename: data.joinPath('log.txt'),
+            filename: path.join(directory, 'log.txt'),
             json: false,
-            level: logLevel,
-            maxFiles: 3,
+            level: level,
+            maxFiles: 3, // Keep last 3 files
             maxsize: 10000000, // 10MB
+            timestamp: () => new Date().toLocaleString(),
         }),
         new (winston.transports.Console)({
             timestamp: () => new Date().toLocaleString(),
             formatter: function(options) {
-                return options.timestamp() + ' ' +
-                        winston.config.colorize(options.level, options.level.toUpperCase()) + ' ' +
-                        (options.message ? options.message : '') +
-                        (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' );
+                return winston.config.colorize(options.level, '  zigbee2mqtt:' + options.level.toLowerCase()) + ' ' +
+                    options.timestamp() + ' ' + (options.message ? options.message : '') +
+                    (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' );
             },
         }),
     ],
 });
 
-logger.transports.console.level = logLevel;
+logger.info(`Logging to directory: '${directory}'`);
+
+logger.transports.console.level = level;
+
+// Cleanup any old log directory.
+if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
+    const rootDirectory = path.join(directory, '..');
+
+    let directories = fs.readdirSync(rootDirectory).map((d) => {
+        d = path.join(rootDirectory, d);
+        return {path: d, birth: fs.statSync(d).birthtimeMs};
+    });
+
+    directories.sort((a, b) => b.birth - a.birth);
+    directories = directories.slice(10, directories.length);
+    directories.forEach((dir) => {
+        logger.debug(`Removing old log directory '${dir.path}'`);
+        rimraf.sync(dir.path);
+    });
+}
 
 module.exports = logger;

lib/util/settings.js

@@ -2,6 +2,17 @@ const yaml = require('js-yaml');
 const fs = require('fs');
 const data = require('./data');
 const file = data.joinPath('configuration.yaml');
+const objectAssignDeep = require(`object-assign-deep`);
+const path = require('path');
+
+const defaults = {
+    permit_join: false,
+    advanced: {
+        log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
+        log_level: process.env.DEBUG ? 'debug' : 'info',
+        soft_reset_timeout: 0,
+    },
+};
 
 let settings = read();
 
@@ -57,7 +68,7 @@ function changeFriendlyName(old, new_) {
 }
 
 module.exports = {
-    get: () => settings,
+    get: () => objectAssignDeep.noMutate(defaults, settings),
     write: () => write(),
     getDevice: (id) => settings.devices ? settings.devices[id] : null,
     addDevice: (id) => addDevice(id),

lib/util/utils.js

@@ -0,0 +1,8 @@
+// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
+const xiaomiManufacturerID = [4151, 4447];
+
+module.exports = {
+    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
+    secondsToMilliseconds: (seconds) => seconds * 1000,
+    isXiaomiDevice: (device) => xiaomiManufacturerID.includes(device.manufId),
+};

lib/zigbee.js

@@ -3,6 +3,7 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const data = require('./util/data');
 const zclPacket = require('zcl-packet');
+const utils = require('./util/utils');
 
 const advancedSettings = settings.get().advanced;
 
@@ -13,7 +14,7 @@ const shepherdSettings = {
     },
     dbPath: data.joinPath('database.db'),
     sp: {
-        baudrate: advancedSettings && advancedSettings.baudrate ? advancedSettings.baudrate : 115200,
+        baudRate: advancedSettings && advancedSettings.baudrate ? advancedSettings.baudrate : 115200,
         rtscts: advancedSettings && (typeof(advancedSettings.rtscts) === 'boolean') ? advancedSettings.rtscts : true,
     },
 };
@@ -21,13 +22,14 @@ const shepherdSettings = {
 logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettings)}'`);
 
 class Zigbee {
-    constructor() {
+    constructor(onMessage) {
+        this.onMessage = onMessage;
         this.handleReady = this.handleReady.bind(this);
         this.handleMessage = this.handleMessage.bind(this);
         this.handleError = this.handleError.bind(this);
     }
 
-    start(onMessage, callback) {
+    start(callback) {
         logger.info(`Starting zigbee-shepherd`);
 
         this.shepherd = new ZShepherd(settings.get().serial.port, shepherdSettings);
@@ -62,12 +64,11 @@ class Zigbee {
         this.shepherd.on('ready', this.handleReady);
         this.shepherd.on('ind', this.handleMessage);
         this.shepherd.on('error', this.handleError);
-
-        this.onMessage = onMessage;
     }
 
     _logStartupInfo() {
         logger.info('zigbee-shepherd started');
+        logger.info(`Coordinator firmware version: '${this.shepherd.info().firmware.revision}'`);
         logger.debug(`zigbee-shepherd info: ${JSON.stringify(this.shepherd.info())}`);
     }
 
@@ -83,12 +84,11 @@ class Zigbee {
     }
 
     handleReady() {
-        // Set all Xiaomi devices (manufId === 4151) to be online, so shepherd won't try
-        // to query info from devices (which would fail because they go tosleep).
-        // Xiaomi lumi.plug has manufId === 4447 and can be in the sleep mode too
+        // Set all Xiaomi devices to be online, so shepherd won't try
+        // to query info from devices (which would fail because they go to sleep).
         const devices = this.getAllClients();
         devices.forEach((d) => {
-            if ((d.manufId === 4151) || (d.manufId === 4447)) {
+            if (utils.isXiaomiDevice(d)) {
                 const device = this.shepherd.find(d.ieeeAddr, 1);
                 if (device) {
                     device.getDevice().update({
@@ -99,7 +99,7 @@ class Zigbee {
             }
         });
 
-        // Check if we have to turn of the led
+        // Check if we have to turn off the led
         if (settings.get().serial.disable_led) {
             this.shepherd.controller.request('UTIL', 'ledControl', {ledid: 3, mode: 0});
         }
@@ -127,7 +127,7 @@ class Zigbee {
     }
 
     getAllClients() {
-        return this.shepherd.list().filter((device) => device.type !== 'Coordinator');
+        return this.getDevices().filter((device) => device.type !== 'Coordinator');
     }
 
     removeDevice(deviceID, callback) {
@@ -153,11 +153,16 @@ class Zigbee {
     }
 
     ping(deviceID) {
+        let friendlyName = 'unknown';
         const device = this.shepherd._findDevByAddr(deviceID);
+        const ieeeAddr = device.ieeeAddr;
+        if (settings.getDevice(ieeeAddr)) {
+            friendlyName = settings.getDevice(ieeeAddr).friendly_name;
+        }
 
         if (device) {
             // Note: checkOnline has the callback argument but does not call callback
-            logger.debug(`Check online ${deviceID}`);
+            logger.debug(`Check online ${friendlyName} ${deviceID}`);
             this.shepherd.controller.checkOnline(device);
         }
     }
@@ -168,24 +173,49 @@ class Zigbee {
         }
     }
 
+    getDevices() {
+        return this.shepherd.list();
+    }
+
     getDevice(deviceID) {
-        return this.shepherd.list().find((d) => d.ieeeAddr === deviceID);
+        return this.getDevices().find((d) => d.ieeeAddr === deviceID);
     }
 
     getCoordinator() {
-        const device = this.shepherd.list().find((d) => d.type === 'Coordinator');
+        const device = this.getDevices().find((d) => d.type === 'Coordinator');
         return this.shepherd.find(device.ieeeAddr, 1);
     }
 
-    publish(deviceID, cid, cmd, zclData, ep, callback) {
+    publish(deviceID, cid, cmd, zclData, cfg, ep, type, callback) {
         const device = this._findDevice(deviceID, ep);
         if (!device) {
             logger.error(`Zigbee cannot publish message to device because '${deviceID}' not known by zigbee-shepherd`);
             return;
         }
 
-        logger.info(`Zigbee publish to '${deviceID}', ${cid} - ${cmd} - ${JSON.stringify(zclData)} - ${ep}`);
-        device.functional(cid, cmd, zclData, callback);
+        logger.info(
+            `Zigbee publish to '${deviceID}', ${cid} - ${cmd} - ` +
+            `${JSON.stringify(zclData)} - ${JSON.stringify(cfg)} - ${ep}`
+        );
+
+        const callback_ = (error) => {
+            if (error) {
+                logger.error(
+                    `Zigbee publish to '${deviceID}', ${cid} - ${cmd} - ${JSON.stringify(zclData)} ` +
+                    `- ${JSON.stringify(cfg)} - ${ep} ` +
+                    `failed with error ${error}`);
+            }
+
+            callback(error);
+        };
+
+        if (type === 'functional') {
+            device.functional(cid, cmd, zclData, cfg, callback_);
+        } else if (type === 'foundation') {
+            device.foundation(cid, cmd, [zclData], cfg, callback_);
+        } else {
+            logger.error(`Unknown zigbee publish type ${type}`);
+        }
     }
 
     read(deviceID, cid, attr, ep, callback) {
@@ -198,6 +228,14 @@ class Zigbee {
         device.read(cid, attr, callback);
     }
 
+    networkScan(callback) {
+        logger.info('Starting network scan...');
+        this.shepherd.lqiScan().then((result) => {
+            logger.info('Network scan completed');
+            callback(result);
+        });
+    }
+
     registerOnAfIncomingMsg(ieeeAddr, ep) {
         const device = this._findDevice(ieeeAddr, ep);
         device.onAfIncomingMsg = (message) => {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "0.1.2",
+  "version": "0.1.7",
   "description": "Zigbee to MQTT bridge using zigbee-shepherd",
   "main": "index.js",
   "repository": {
@@ -17,10 +17,12 @@
     "cc2531"
   ],
   "scripts": {
-    "start": "node index.js",
+    "coverage": "nyc --all report --reporter=text mocha --recursive --timeout=3000",
     "docgen": "node support/docgen.js",
-    "verify-homeassistant-mapping": "node support/verify-homeassistant-mapping.js",
-    "eslint": "node_modules/.bin/eslint ."
+    "eslint": "node_modules/.bin/eslint .",
+    "start": "node index.js",
+    "test": "mocha --recursive",
+    "verify-homeassistant-mapping": "node support/verify-homeassistant-mapping.js"
   },
   "author": "Koen Kanters",
   "license": "GPL-3.0",
@@ -29,18 +31,27 @@
   },
   "homepage": "https://github.com/Koenkk/zigbee2mqtt/wiki",
   "dependencies": {
-    "object-assign-deep": "*",
-    "mqtt": "*",
-    "winston": "2.4.2",
-    "zigbee-shepherd": "git+https://github.com/Koenkk/zigbee-shepherd.git#2078557a485466c032cc26b2cd2b8ac29edeceac",
-    "zigbee-shepherd-converters": "2.0.34",
-    "json2yaml": "*",
-    "zcl-packet": "git+https://github.com/Koenkk/zcl-packet.git#3ea8912f9d228b59fad554064d15ed9be7897a7f",
+    "git-last-commit": "*",
     "js-yaml": "*",
-    "git-last-commit": "*"
+    "json2yaml": "*",
+    "mkdir-recursive": "*",
+    "moment": "*",
+    "mqtt": "*",
+    "object-assign-deep": "*",
+    "rimraf": "*",
+    "winston": "2.4.2",
+    "zcl-packet": "git+https://github.com/Koenkk/zcl-packet.git#fbd8c936bbd4be0597ad3e934be0ca722b0128a6",
+    "zigbee-shepherd": "git+https://github.com/Koenkk/zigbee-shepherd.git#c3fdef2c30267573aa5821a0f2c51f21a700eff2",
+    "zigbee-shepherd-converters": "4.0.3"
   },
   "devDependencies": {
+    "chai": "*",
+    "chai-spies": "*",
     "eslint": "*",
-    "eslint-config-google": "*"
+    "eslint-config-google": "*",
+    "eslint-plugin-mocha": "*",
+    "mocha": "*",
+    "nyc": "*",
+    "proxyquire": "*"
   }
 }

test/data.test.js

@@ -0,0 +1,23 @@
+/* global describe, it */
+
+const chai = require('chai');
+const proxyquire = require('proxyquire').noPreserveCache();
+const data = () => proxyquire('../lib/util/data.js', {});
+const path = require('path');
+
+describe('Data', () => {
+    describe('Get path', () => {
+        it('Should return correct path', () => {
+            const expected = path.normalize(path.join(__dirname, '..', 'data'));
+            const actual = data().getPath();
+            chai.assert.strictEqual(actual, expected);
+        });
+
+        it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
+            const expected = path.join('var', 'zigbee2mqtt');
+            process.env.ZIGBEE2MQTT_DATA = expected;
+            const actual = data().getPath();
+            chai.assert.strictEqual(actual, expected);
+        });
+    });
+});