1.20.0

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -3,6 +3,7 @@
 .git
 .github
 .gitignore
+.idea
 .npmrc
 .travis*
 .travis.yml
@@ -12,4 +13,4 @@ images
 node_modules
 scripts
 test
-update.sh
+update.sh

.github/FUNDING.yml

@@ -1,2 +1,4 @@
 github: [koenkk]
-custom: https://www.paypal.me/koenkk
+custom:
+  - https://www.paypal.me/koenkk
+  - https://www.buymeacoffee.com/koenkk

.github/ISSUE_TEMPLATE/config.yml

@@ -7,7 +7,7 @@ contact_links:
     url: https://github.com/nurikk/z2m-frontend/issues
     about: Issues related to the frontend.
   - name: Home Assistant addon issues
-    url: https://github.com/danielwelch/hassio-zigbee2mqtt/issues
+    url: https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/issues
     about: Issues related to the Home Assistant addon.
   - name: FAQ
     url: https://www.zigbee2mqtt.io/information/FAQ.html

.github/workflows/ci.yml

@@ -31,6 +31,9 @@ jobs:
     - name: Docker login
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
       run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
+    - name: Docker login ghcr.io
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
     - name: Docker setup - QEMU
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
       uses: docker/setup-qemu-action@v1
@@ -50,7 +53,7 @@ jobs:
           --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
-          -t koenkk/zigbee2mqtt:latest-dev \
+          -t koenkk/zigbee2mqtt:latest-dev -t ghcr.io/koenkk/zigbee2mqtt:latest-dev \
           .
     - name: Docker build release
       if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
@@ -61,22 +64,13 @@ jobs:
           --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
-          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
+          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" -t ghcr.io/koenkk/zigbee2mqtt:latest -t "ghcr.io/koenkk/zigbee2mqtt:$TAG" \
           .
     - name: Publish to npm
       if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
       run: npm publish
       env:
         NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN }}
-    - name: Trigger danielwelch/hassio-zigbee2mqtt build
-      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
-      run: |
-        curl -s \
-          --request POST \
-          --header "Content-Type: application/json" \
-          --data '{"definition": {"id": 1}}' \
-          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
     - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
       if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
       run: |

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+# Contributing to Zigbee2MQTT
+
+Everybody is invited and welcome to contribute to Zigbee2MQTT. Zigbee2MQTT is writen in JavaScript and is based upon [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) and [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters). Zigbee-herdsman-converters contains all device definition, zigbee-herdsman is responsible for handling all communication with the adapter.
+
+- Pull requests are always created against the [**dev**](https://github.com/Koenkk/zigbee2mqtt/tree/dev) branch.
+- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/getting_started/running_zigbee2mqtt.html#running-zigbee2mqtt) to do this.
+- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if you code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
+- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).

README.md

@@ -53,7 +53,7 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
 - Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">

docker/Dockerfile

@@ -1,7 +1,7 @@
-FROM node:12-alpine3.12 as base
+FROM node:14-alpine3.12 as base
 
 WORKDIR /app
-RUN apk add --no-cache tzdata eudev
+RUN apk add --no-cache tzdata eudev tini
 
 COPY package.json ./
 
@@ -19,7 +19,7 @@ FROM base as release
 
 COPY --from=dependencies /app/node_modules ./node_modules
 COPY lib ./lib
-COPY index.js data/configuration.yaml ./
+COPY LICENSE index.js data/configuration.yaml ./
 
 COPY docker/docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
@@ -30,4 +30,4 @@ ARG COMMIT
 RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
 
 ENTRYPOINT ["docker-entrypoint.sh"]
-CMD ["npm", "start"]
+CMD [ "/sbin/tini", "--", "node", "index.js"]

index.js

@@ -1,38 +1,60 @@
 const semver = require('semver');
 const engines = require('./package.json').engines;
+const indexJsRestart = 'indexjs.restart';
 
-const version = engines.node;
-if (!semver.satisfies(process.version, version)) {
-    console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+let controller;
+let stopping = false;
+
+async function restart() {
+    await stop(indexJsRestart);
+    await start();
 }
 
-// Validate settings
-const settings = require('./lib/util/settings');
-const errors = settings.validate();
-if (errors.length > 0) {
-    console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
-    console.log('            READ THIS CAREFULLY\n');
-    console.log(`Refusing to start because configuration is not valid, found the following errors:`);
-    for (const error of errors) {
-        console.log(`- ${error}`);
+async function exit(code, reason) {
+    if (reason !== indexJsRestart) {
+        process.exit(code);
     }
-    console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`);
-    console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
-    process.exit(1);
 }
 
-const Controller = require('./lib/controller');
-const controller = new Controller();
-controller.start();
+async function start() {
+    const version = engines.node;
+    if (!semver.satisfies(process.version, version)) {
+        console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+    }
+
+    // Validate settings
+    const settings = require('./lib/util/settings');
+    settings.reRead();
+    const errors = settings.validate();
+    if (errors.length > 0) {
+        console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
+        console.log('            READ THIS CAREFULLY\n');
+        console.log(`Refusing to start because configuration is not valid, found the following errors:`);
+        for (const error of errors) {
+            console.log(`- ${error}`);
+        }
+        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`); // eslint-disable-line
+        console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
+        exit(1);
+    }
+
+    const Controller = require('./lib/controller');
+    controller = new Controller(restart, exit);
+    await controller.start();
+}
+
+async function stop(reason=null) {
+    await controller.stop(reason);
+}
+
+async function handleQuit() {
+    if (!stopping && controller) {
+        stopping = true;
+        await stop();
+    }
+}
 
 process.on('SIGINT', handleQuit);
 process.on('SIGTERM', handleQuit);
 
-let stopping = false;
-
-function handleQuit() {
-    if (!stopping) {
-        stopping = true;
-        controller.stop();
-    }
-}
+start();

lib/controller.js

@@ -4,12 +4,8 @@ const EventBus = require('./eventBus');
 const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
-const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
 const stringify = require('json-stable-stringify-without-jsonify');
-const fs = require('fs');
-const data = require('./util/data');
-const path = require('path');
 const assert = require('assert');
 
 // Extensions
@@ -26,33 +22,37 @@ const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
 const ExtensionAvailability = require('./extension/availability');
 const ExtensionBind = require('./extension/bind');
-const ExtensionReport = require('./extension/report');
+const ExtensionReport = require('./extension/legacy/report');
 const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
 const ExtensionExternalConverters = require('./extension/externalConverters');
+const ExtensionExternalExtension = require('./extension/externalExtension');
 
 const AllExtensions = [
     ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
     ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
     ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
-    ExtensionExternalConverters,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension,
 ];
 
 class Controller {
-    constructor() {
+    constructor(restartCallback, exitCallback) {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
         this.eventBus = new EventBus();
         this.state = new State(this.eventBus);
+        this.restartCallback = restartCallback;
+        this.exitCallback = exitCallback;
 
         this.publishEntityState = this.publishEntityState.bind(this);
         this.enableDisableExtension = this.enableDisableExtension.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
+        this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
         const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionBridge(...args, this.enableDisableExtension),
+            new ExtensionBridge(...args, this.enableDisableExtension, this.restartCallback),
             new ExtensionPublish(...args),
             new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
@@ -73,6 +73,10 @@ class Controller {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
 
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
+        }
+
         if (settings.get().homeassistant) {
             this.extensions.push(new ExtensionHomeAssistant(...args));
         }
@@ -85,18 +89,7 @@ class Controller {
         if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
-        if (settings.get().external_converters.length) {
-            this.extensions.push(new ExtensionExternalConverters(...args));
-        }
-
-        const extensionPath = data.joinPath('extension');
-        if (fs.existsSync(extensionPath)) {
-            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
-            for (const extension of extensions) {
-                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
-                this.extensions.push(new Extension(...args, settings, logger));
-            }
-        }
+        this.extensions.push(new ExtensionExternalExtension(...args, this.addExtension, this.enableDisableExtension));
     }
 
     async start() {
@@ -107,16 +100,27 @@ class Controller {
         logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
         // Start zigbee
+        let startResult;
         try {
-            await this.zigbee.start();
+            startResult = await this.zigbee.start();
             this.callExtensionMethod('onZigbeeStarted', []);
             this.zigbee.on('event', this.onZigbeeEvent.bind(this));
             this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
         } catch (error) {
             logger.error('Failed to start zigbee');
+            // eslint-disable-next-line
+            logger.error('Check https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start for possible solutions');
             logger.error('Exiting...');
             logger.error(error.stack);
-            process.exit(1);
+            this.exitCallback(1);
+        }
+
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
         }
 
         // Log zigbee clients on startup
@@ -184,7 +188,11 @@ class Controller {
         }
     }
 
-    async stop() {
+    addExtension(extension) {
+        this.extensions.push(extension);
+    }
+
+    async stop(reason=null) {
         // Call extensions
         await this.callExtensionMethod('stop', []);
 
@@ -194,10 +202,11 @@ class Controller {
 
         try {
             await this.zigbee.stop();
-            process.exit(0);
+            logger.info('Stopped Zigbee2MQTT');
+            this.exitCallback(0, reason);
         } catch (error) {
-            logger.error('Failed to stop zigbee');
-            process.exit(1);
+            logger.error('Failed to stop Zigbee2MQTT');
+            this.exitCallback(1, reason);
         }
     }
 
@@ -252,10 +261,14 @@ class Controller {
             }
         } else if (type === 'deviceAnnounce') {
             logger.debug(`Device '${name}' announced itself`);
+        } else if (type === 'deviceNetworkAddressChanged') {
+            logger.debug(`Device '${name}' changed network address`);
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
-                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                const settingsDevice = settings.getDevice(data.ieeeAddr);
+                logger.warn(`Device '${settingsDevice ? settingsDevice.friendlyName : data.ieeeAddr}' ` +
+                    `left the network`);
                 this.state.remove(data.ieeeAddr);
             }
         }
@@ -280,12 +293,9 @@ class Controller {
         }
 
         let messagePayload = {...payload};
-        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
-            this.state.get(resolvedEntity.settings.ID) : {};
-        const newState = objectAssignDeep.noMutate(currentState, payload);
 
         // Update state cache with new state.
-        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
+        const newState = this.state.set(resolvedEntity.settings.ID, payload, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
@@ -333,6 +343,12 @@ class Controller {
             resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
+        for (const extension of this.extensions) {
+            if (extension.adjustMessagePayloadBeforePublish) {
+                extension.adjustMessagePayloadBeforePublish(resolvedEntity, messagePayload);
+            }
+        }
+
         if (Object.entries(messagePayload).length) {
             const output = settings.get().experimental.output;
             if (output === 'attribute_and_json' || output === 'json') {
@@ -344,7 +360,7 @@ class Controller {
             }
         }
 
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
+        this.eventBus.emit('publishEntityState', {messagePayload, entity: resolvedEntity, stateChangeReason, payload});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {

lib/extension/availability.js

@@ -9,10 +9,18 @@ const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/(.*)/availabil
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
 const pingableEndDevices = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-N1EA'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
 ];
 
 const Hours25 = 1000 * 60 * 60 * 25;
+const AvailabilityLagRatio = 0.1;
+
+
+function timeoutLag(timeout, ratio) {
+    const lag = timeout * ratio;
+    return Math.floor(Math.random() * Math.floor(lag));
+}
 
 /**
  * This extensions pings devices to check if they are online.
@@ -136,10 +144,11 @@ class Availability extends Extension {
     }
 
     setTimerPingable(device) {
+        const timeout = this.availability_timeout + timeoutLag(this.availability_timeout, AvailabilityLagRatio);
         clearTimeout(this.timers[device.ieeeAddr]);
         this.timers[device.ieeeAddr] = setTimeout(async () => {
             await this.handleIntervalPingable(device);
-        }, utils.secondsToMilliseconds(this.availability_timeout));
+        }, utils.secondsToMilliseconds(timeout));
     }
 
     async stop() {

lib/extension/bind.js

@@ -5,16 +5,121 @@ const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const Extension = require('./extension');
 const stringify = require('json-stable-stringify-without-jsonify');
+const debounce = require('debounce');
+const zigbeeHersdman = require('zigbee-herdsman');
 
-const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
+    'hvacThermostat', 'msTemperatureMeasurement'];
 
-// See zigbee-herdsman-converters devices.js
+// See zigbee-herdsman-converters
 const defaultBindGroup = {type: 'group_number', ID: 901};
 
+const defaultReportConfiguration = {
+    minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: 1,
+};
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') == null) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const reportClusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultReportConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultReportConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultReportConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultReportConfiguration},
+    ],
+};
+
+const pollOnMessage = [
+    {
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genLevelCtrl: [
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+                {type: 'commandToggle', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+];
+
 class Bind extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.legacyApi = settings.get().advanced.legacy_api;
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d));
+        this.pollDebouncers = {};
     }
 
     parseMQTTMessage(topic, message) {
@@ -22,6 +127,7 @@ class Bind extends Extension {
         let sourceKey = null;
         let targetKey = null;
         let clusters = null;
+        let skipDisableReporting = false;
 
         if (this.legacyApi && topic.match(legacyTopicRegex)) {
             topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
@@ -34,14 +140,16 @@ class Bind extends Extension {
             sourceKey = message.from;
             targetKey = message.to;
             clusters = message.clusters;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
         }
 
-        return {type, sourceKey, targetKey, clusters};
+        return {type, sourceKey, targetKey, clusters, skipDisableReporting};
     }
 
     async onMQTTMessage(topic, message) {
-        const {type, sourceKey, targetKey, clusters} = this.parseMQTTMessage(topic, message);
+        const {type, sourceKey, targetKey, clusters, skipDisableReporting} = this.parseMQTTMessage(topic, message);
         if (!type) return null;
+        message = utils.parseJSON(message, message);
 
         let error = null;
         const source = this.zigbee.resolveEntity(sourceKey);
@@ -59,24 +167,35 @@ class Bind extends Extension {
             const failedClusters = [];
             const attemptedClusters = [];
 
+            let bindTarget = null;
+            if (target.type === 'group') bindTarget = target.group;
+            else if (target.type === 'group_number') bindTarget = target.ID;
+            else bindTarget = target.endpoint;
+
             // Find which clusters are supported by both the source and target.
             // Groups are assumed to support all clusters.
             for (const cluster of clusterCandidates) {
                 if (clusters && !clusters.includes(cluster)) continue;
+                let matchingClusters = false;
 
-                const targetValid = target.type === 'group' || target.type === 'group_number' ||
-                    target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+                const anyClusterValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator';
 
-                if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                if (!anyClusterValid) {
+                    matchingClusters = ((target.endpoint.supportsInputCluster(cluster) &&
+                            source.endpoint.supportsOutputCluster(cluster)) ||
+                            (source.endpoint.supportsInputCluster(cluster) &&
+                            target.endpoint.supportsOutputCluster(cluster)) );
+                }
+
+                const sourceValid = source.endpoint.supportsInputCluster(cluster) ||
+                source.endpoint.supportsOutputCluster(cluster);
+
+                if ( sourceValid && (anyClusterValid || matchingClusters)) {
                     logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
                     attemptedClusters.push(cluster);
 
                     try {
-                        let bindTarget = null;
-                        if (target.type === 'group') bindTarget = target.group;
-                        else if (target.type === 'group_number') bindTarget = target.ID;
-                        else bindTarget = target.endpoint;
-
                         if (type === 'bind') {
                             await source.endpoint.bind(cluster, bindTarget);
                         } else {
@@ -133,6 +252,15 @@ class Bind extends Extension {
 
             responseData[`clusters`] = successfulClusters;
             responseData[`failed`] = failedClusters;
+
+            if (successfulClusters.length !== 0) {
+                if (type === 'bind') {
+                    await this.setupReporting(source.endpoint.binds.filter((b) =>
+                        successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
+                } else if ((target.type !== 'group_number') && !skipDisableReporting) {
+                    await this.disableUnnecessaryReportings(bindTarget);
+                }
+            }
         }
 
         const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
@@ -147,6 +275,150 @@ class Bind extends Extension {
             this.eventBus.emit(`devicesChanged`);
         }
     }
+
+    async groupMembersChanged(data) {
+        if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
+                .flat().filter((b) => b.target === data.group.group);
+            await this.setupReporting(bindsToGroup);
+        } else { // action === remove/remove_all
+            if (!data.skipDisableReporting) {
+                await this.disableUnnecessaryReportings(data.endpoint);
+            }
+        }
+    }
+
+    getSetupReportingEndpoints(bind, coordinatorEp) {
+        const endpoints = bind.target.constructor.name === 'Group' ? bind.target.members : [bind.target];
+        return endpoints.filter((e) => {
+            const supportsInputCluster = e.supportsInputCluster(bind.cluster.name);
+            const hasConfiguredReporting = !!e.configuredReportings.find((c) => c.cluster.name === bind.cluster.name);
+            const hasBind = !!e.binds.find((b) => b.cluster.name === bind.cluster.name && b.target === coordinatorEp);
+            return supportsInputCluster && !(hasBind && hasConfiguredReporting);
+        });
+    }
+
+    async setupReporting(binds) {
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        for (const bind of binds.filter((b) => b.cluster.name in reportClusters)) {
+            for (const endpoint of this.getSetupReportingEndpoints(bind, coordinatorEndpoint)) {
+                const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+                try {
+                    await endpoint.bind(bind.cluster.name, coordinatorEndpoint);
+
+                    const items = await reportClusters[bind.cluster.name]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(bind.cluster.name, items);
+                    logger.info(`Succesfully setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                } catch (error) {
+                    logger.warn(`Failed to setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                }
+            }
+        }
+
+        this.eventBus.emit(`devicesChanged`);
+    }
+
+    async disableUnnecessaryReportings(target) {
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        const endpoints = target.constructor.name === 'Group' ? target.members : [target];
+        for (const endpoint of endpoints) {
+            const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+            const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
+                .map((b) => b.cluster.name);
+            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
+                .flat().filter((bind) => {
+                    if (bind.target.constructor.name === 'Group') {
+                        return bind.target.members.includes(endpoint);
+                    } else {
+                        return bind.target === endpoint;
+                    }
+                }).map((b) => b.cluster.name).filter((v, i, a) => a.indexOf(v) === i);
+
+            for (const cluster of boundClusters.filter((c) => !requiredClusters.includes(c) && c in reportClusters)) {
+                try {
+                    await endpoint.unbind(cluster, coordinatorEndpoint);
+                    const items = await reportClusters[cluster]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a, maximumReportInterval: 0xFFFF};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(cluster, items);
+                    logger.info(`Succesfully disabled reporting for '${entity}' cluster '${cluster}'`);
+                } catch (error) {
+                    logger.warn(`Failed to disable reporting for '${entity}' cluster '${cluster}'`);
+                }
+            }
+
+            this.eventBus.emit('reportingDisabled', {device: endpoint.getDevice()});
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'message') {
+            this.poll(data);
+        }
+    }
+
+    poll(message) {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
+            utils.equalsPartial(message.data, c.data)),
+        );
+
+        if (polls.length) {
+            let toPoll = [];
+
+            // Add bound devices
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
+            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
+            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
+            toPoll = toPoll.map((e) => e.target);
+
+            // If message is published to a group, add members of the group
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
+            if (group) {
+                toPoll = toPoll.concat(group.members);
+            }
+
+            toPoll = new Set(toPoll);
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
+                        continue;
+                    }
+
+                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, poll.read.attributes);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
+            }
+        }
+    }
 }
 
 module.exports = Bind;

lib/extension/bridge.js

@@ -6,15 +6,18 @@ const settings = require('../util/settings');
 const Transport = require('winston-transport');
 const stringify = require('json-stable-stringify-without-jsonify');
 const objectAssignDeep = require(`object-assign-deep`);
+const {updatedDiff, addedDiff} = require('deep-object-diff');
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
 class Bridge extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.enableDisableExtension = enableDisableExtension;
+        this.restartCallback = restartCallback;
         this.lastJoinedDeviceIeeeAddr = null;
         this.setupMQTTLogging();
+        this.restartRequired = false;
 
         this.requestLookup = {
             'device/options': this.deviceOptions.bind(this),
@@ -26,14 +29,17 @@ class Bridge extends Extension {
             'group/remove': this.groupRemove.bind(this),
             'group/rename': this.groupRename.bind(this),
             'permit_join': this.permitJoin.bind(this),
-            'config/last_seen': this.configLastSeen.bind(this),
-            'config/homeassistant': this.configHomeAssistant.bind(this),
-            'config/elapsed': this.configElapsed.bind(this),
-            'config/log_level': this.configLogLevel.bind(this),
+            'restart': this.restart.bind(this),
             'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
             'touchlink/identify': this.touchlinkIdentify.bind(this),
             'touchlink/scan': this.touchlinkScan.bind(this),
             'health_check': this.healthCheck.bind(this),
+            'options': this.bridgeOptions.bind(this),
+            // Below are deprecated
+            'config/last_seen': this.configLastSeen.bind(this),
+            'config/homeassistant': this.configHomeAssistant.bind(this),
+            'config/elapsed': this.configElapsed.bind(this),
+            'config/log_level': this.configLogLevel.bind(this),
         };
     }
 
@@ -42,7 +48,17 @@ class Bridge extends Extension {
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
         this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
-        this.eventBus.on(`devicesChanged`, () => this.publishDevices());
+        this.eventBus.on(`devicesChanged`, () => {
+            this.publishDevices();
+            this.publishInfo();
+        });
+        this.eventBus.on(`deviceRenamed`, () => {
+            this.publishInfo();
+        });
+        this.eventBus.on(`groupRenamed`, () => {
+            this.publishInfo();
+        });
+        this.zigbee.on('permitJoinChanged', (data) => this.permitJoinChanged(data));
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
@@ -58,7 +74,13 @@ class Bridge extends Extension {
             }
         }
 
-        logger.add(new EventTransport({level: logger.getLevel()}));
+        logger.addTransport(new EventTransport());
+    }
+
+    permitJoinChanged(data) {
+        if (!this.zigbee.isStopping()) {
+            this.publishInfo();
+        }
     }
 
     async onMQTTMessage(topic, message) {
@@ -94,13 +116,18 @@ class Bridge extends Extension {
                 if (data.status === 'successful') {
                     const definition = resolvedEntity.definition;
                     payload.supported = !!definition;
-                    payload.definition = this.getDefinitionPayload(definition);
+                    payload.definition = this.getDefinitionPayload(definition, resolvedEntity);
                 }
             } else if (type === 'deviceAnnounce') {
                 payload = {
                     friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
                 };
-            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
+            } else {
+                // deviceLeave
+                const settingsDevice = settings.getDevice(ieeeAddress);
+                payload = {ieee_address: ieeeAddress,
+                    friendly_name: settingsDevice ? settingsDevice.friendlyName : null};
+            }
 
             await this.mqtt.publish(
                 'bridge/event',
@@ -109,7 +136,8 @@ class Bridge extends Extension {
             );
         }
 
-        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started') ||
+            'deviceNetworkAddressChanged' === type) {
             await this.publishDevices();
         }
     }
@@ -126,6 +154,49 @@ class Bridge extends Extension {
         return this.changeEntityOptions('group', message);
     }
 
+    async bridgeOptions(message) {
+        if (typeof message !== 'object' || typeof message.options !== 'object') {
+            throw new Error(`Invalid payload`);
+        }
+
+        const diffUpdated = updatedDiff(settings.get(), message.options);
+        const diffAdded = addedDiff(settings.get(), message.options);
+        const newSettings = objectAssignDeep.noMutate(diffUpdated, diffAdded);
+
+        // deep-object-diff converts arrays to objects, set original array back here
+        const convertBackArray = (before, after) => {
+            for (const [key, afterValue] of Object.entries(after)) {
+                const beforeValue = before[key];
+                if (Array.isArray(beforeValue)) {
+                    after[key] = beforeValue;
+                } else if (afterValue && typeof beforeValue === 'object') {
+                    convertBackArray(beforeValue, afterValue);
+                }
+            }
+        };
+        convertBackArray(message.options, newSettings);
+
+        const restartRequired = settings.apply(newSettings);
+        if (restartRequired) this.restartRequired = true;
+
+        // Apply some settings on-the-fly.
+        if (newSettings.hasOwnProperty('permit_join')) {
+            await this.zigbee.permitJoin(newSettings.permit_join);
+        }
+
+        if (newSettings.hasOwnProperty('homeassistant')) {
+            this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
+        }
+
+        if (newSettings.hasOwnProperty('advanced') && newSettings.advanced.hasOwnProperty('log_level')) {
+            logger.setLevel(newSettings.advanced.log_level);
+        }
+
+        logger.info('Succesfully changed options');
+        this.publishInfo();
+        return utils.getResponse(message, {restart_required: this.restartRequired}, null);
+    }
+
     async deviceRemove(message) {
         return this.removeEntity('device', message);
     }
@@ -159,15 +230,24 @@ class Bridge extends Extension {
         return this.renameEntity('group', message);
     }
 
+    async restart(message) {
+        // Wait 500 ms before restarting so response can be send.
+        setTimeout(this.restartCallback, 500);
+        logger.info('Restarting Zigbee2MQTT');
+        return utils.getResponse(message, {}, null);
+    }
+
     async permitJoin(message) {
         if (typeof message === 'object' && !message.hasOwnProperty('value')) {
             throw new Error('Invalid payload');
         }
 
         let value;
+        let time;
         let resolvedEntity;
         if (typeof message === 'object') {
             value = message.value;
+            time = message.time;
             if (message.device) {
                 resolvedEntity = this.zigbee.resolveEntity(message.device);
                 if (!resolvedEntity || resolvedEntity.type !== 'device') {
@@ -178,13 +258,18 @@ class Bridge extends Extension {
             value = message;
         }
 
-        await this.zigbee.permitJoin(value, resolvedEntity);
-        await this.publishInfo();
-        return utils.getResponse(
-            message, resolvedEntity ? {value: value, device: message.device} : {value: value}, null,
-        );
+        if (typeof value === 'string') {
+            value = value.toLowerCase() === 'true';
+        }
+
+        await this.zigbee.permitJoin(value, resolvedEntity, time);
+        const response = {value};
+        if (resolvedEntity) response.device = message.device;
+        if (time) response.time = message.time;
+        return utils.getResponse(message, response, null);
     }
 
+    // Deprecated
     configLastSeen(message) {
         const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
         const value = this.getValue(message);
@@ -192,11 +277,12 @@ class Bridge extends Extension {
             throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
         }
 
-        settings.set(['advanced', 'last_seen'], message);
+        settings.set(['advanced', 'last_seen'], value);
         this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
+    // Deprecated
     configHomeAssistant(message) {
         const allowed = [true, false];
         const value = this.getValue(message);
@@ -210,6 +296,7 @@ class Bridge extends Extension {
         return utils.getResponse(message, {value}, null);
     }
 
+    // Deprecated
     configElapsed(message) {
         const allowed = [true, false];
         const value = this.getValue(message);
@@ -222,6 +309,7 @@ class Bridge extends Extension {
         return utils.getResponse(message, {value}, null);
     }
 
+    // Deprecated
     configLogLevel(message) {
         const allowed = ['error', 'warn', 'info', 'debug'];
         const value = this.getValue(message);
@@ -294,7 +382,7 @@ class Bridge extends Extension {
         }
     }
 
-    changeEntityOptions(entityType, message) {
+    async changeEntityOptions(entityType, message) {
         if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
             throw new Error(`Invalid payload`);
         }
@@ -308,6 +396,10 @@ class Bridge extends Extension {
         };
         const oldOptions = cleanup(entity.settings);
         const newOptions = cleanup(settings.getEntity(ID));
+        await this.publishInfo();
+
+        logger.info(`Changed config for ${entityType} ${ID}`);
+
         return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
     }
 
@@ -320,6 +412,11 @@ class Bridge extends Extension {
 
         const ID = message.id;
         const endpoint = this.getEntity('device', ID).endpoint;
+
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        await endpoint.bind(message.cluster, coordinatorEndpoint);
+
         await endpoint.configureReporting(message.cluster, [{
             attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
             maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
@@ -347,11 +444,6 @@ class Bridge extends Extension {
             throw new Error('No device has joined since start');
         }
 
-        const validationErrors = utils.validateFriendlyName(message.to);
-        if (validationErrors.length !== 0) {
-            throw new Error(validationErrors[0]);
-        }
-
         const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
         const to = message.to;
         const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
@@ -473,11 +565,15 @@ class Bridge extends Extension {
             coordinator: this.coordinatorVersion,
             network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
             log_level: logger.getLevel(),
-            permit_join: await this.zigbee.getPermitJoin(),
+            permit_join: this.zigbee.getPermitJoin(),
+            permit_join_timeout: this.zigbee.getPermitJoinTimeout(),
+            restart_required: this.restartRequired,
             config,
+            config_schema: settings.schema,
         };
 
-        await this.mqtt.publish('bridge/info', stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish(
+            'bridge/info', stringify(payload), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
     async publishDevices() {
@@ -512,7 +608,7 @@ class Bridge extends Extension {
                 for (const configuredReporting of endpoint.configuredReportings) {
                     data.configured_reportings.push({
                         cluster: configuredReporting.cluster.name,
-                        attribute: configuredReporting.attribute.name,
+                        attribute: configuredReporting.attribute.name || configuredReporting.attribute.ID,
                         minimum_report_interval: configuredReporting.minimumReportInterval,
                         maximum_report_interval: configuredReporting.maximumReportInterval,
                         reportable_change: configuredReporting.reportableChange,
@@ -528,7 +624,7 @@ class Bridge extends Extension {
                 network_address: device.networkAddress,
                 supported: !!definition,
                 friendly_name: resolved.name,
-                definition: this.getDefinitionPayload(definition),
+                definition: this.getDefinitionPayload(definition, resolved),
                 power_source: device.powerSource,
                 software_build_id: device.softwareBuildID,
                 date_code: device.dateCode,
@@ -539,7 +635,8 @@ class Bridge extends Extension {
             };
         });
 
-        await this.mqtt.publish('bridge/devices', stringify(devices), {retain: true, qos: 0});
+        await this.mqtt.publish(
+            'bridge/devices', stringify(devices), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
     async publishGroups() {
@@ -557,16 +654,35 @@ class Bridge extends Extension {
             };
         });
 
-        await this.mqtt.publish('bridge/groups', stringify(groups), {retain: true, qos: 0});
+        await this.mqtt.publish(
+            'bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    getDefinitionPayload(definition) {
+    getDefinitionPayload(definition, resolvedDevice) {
         if (definition) {
+            let icon = null;
+            if (resolvedDevice && resolvedDevice.hasOwnProperty('settings')) {
+                icon = resolvedDevice.settings.icon;
+            }
+            if (!icon) {
+                icon = definition.icon;
+            }
+            if (icon) {
+                let zigbeeModel = '';
+                /* istanbul ignore else */
+                if (resolvedDevice && resolvedDevice.hasOwnProperty('device')) {
+                    zigbeeModel = resolvedDevice.device.modelID;
+                }
+                icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(zigbeeModel));
+                icon = icon.replace('${model}', utils.sanitizeImageParameter(definition.model));
+            }
             return {
                 model: definition.model,
                 vendor: definition.vendor,
                 description: definition.description,
                 exposes: definition.exposes,
+                supports_ota: !!definition.ota,
+                icon,
             };
         } else {
             return null;

lib/extension/configure.js

@@ -3,6 +3,7 @@ const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 const stringify = require('json-stable-stringify-without-jsonify');
+const zhc = require('zigbee-herdsman-converters');
 
 /**
  * This extension calls the zigbee-herdsman-converters definition configure() method
@@ -31,18 +32,18 @@ class Configure extends Extension {
         }
 
         if (this.shouldConfigure(resolvedEntity)) {
-            this.configure(resolvedEntity);
+            this.configure(resolvedEntity, 'reporting_disabled');
         }
     }
 
-    shouldConfigure(resolvedEntity) {
+    shouldConfigure(resolvedEntity, event) {
         if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
             return false;
         }
 
         if (resolvedEntity.device.meta &&
             resolvedEntity.device.meta.hasOwnProperty('configured') &&
-            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            resolvedEntity.device.meta.configured === zhc.getConfigureKey(resolvedEntity.definition)) {
             return false;
         }
 
@@ -50,6 +51,11 @@ class Configure extends Extension {
             return false;
         }
 
+        // Only configure end devices when a message is received, otherwise it will likely fails as they are sleeping.
+        if (resolvedEntity.device.type === 'EndDevice' && event !== 'message_received') {
+            return false;
+        }
+
         return true;
     }
 
@@ -95,7 +101,7 @@ class Configure extends Extension {
 
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
-            if (this.shouldConfigure(resolvedEntity)) {
+            if (this.shouldConfigure(resolvedEntity, 'started')) {
                 await this.configure(resolvedEntity);
             }
         }
@@ -109,7 +115,7 @@ class Configure extends Extension {
             device.save();
         }
 
-        if (this.shouldConfigure(resolvedEntity)) {
+        if (this.shouldConfigure(resolvedEntity, 'message_received')) {
             this.configure(resolvedEntity);
         }
     }
@@ -130,7 +136,7 @@ class Configure extends Extension {
         try {
             await resolvedEntity.definition.configure(device, this.coordinatorEndpoint, logger);
             logger.info(`Successfully configured '${resolvedEntity.name}'`);
-            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.meta.configured = zhc.getConfigureKey(resolvedEntity.definition);
             device.save();
             this.eventBus.emit(`devicesChanged`);
         } catch (error) {

lib/extension/externalExtension.js

@@ -0,0 +1,113 @@
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const fs = require('fs');
+const data = require('./../util/data');
+const path = require('path');
+const logger = require('./../util/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
+
+class ExternalExtension extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, addExtension, enableDisableExtension) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
+        this.addExtension = addExtension;
+        this.enableDisableExtension = enableDisableExtension;
+        this.extensionsBaseDir = 'extension';
+        this.loadExtension = this.loadExtension.bind(this);
+        this.requestLookup = {
+            'save': this.saveExtension.bind(this),
+            'remove': this.removeExtension.bind(this),
+        };
+        this.loadUserDefinedExtensions();
+    }
+    getExtensionsBasePath() {
+        return data.joinPath(this.extensionsBaseDir);
+    }
+
+    getListOfUserDefinedExtensions() {
+        const basePath = this.getExtensionsBasePath();
+        if (fs.existsSync(basePath)) {
+            return fs.readdirSync(basePath).filter((f) => f.endsWith('.js')).map((fileName) => {
+                const extensonFilePath = path.join(basePath, fileName);
+                return {'name': fileName, 'code': fs.readFileSync(extensonFilePath, 'utf-8')};
+            });
+        } else {
+            return [];
+        }
+    }
+    removeExtension(message) {
+        const {name} = message;
+        const extensions = this.getListOfUserDefinedExtensions();
+        const extensionToBeRemoved = extensions.find((e) => e.name === name);
+
+        if (extensionToBeRemoved) {
+            this.enableDisableExtension(false, extensionToBeRemoved.name);
+            const basePath = this.getExtensionsBasePath();
+            const extensonFilePath = path.join(basePath, path.basename(name));
+            fs.unlinkSync(extensonFilePath);
+            this.publishExtensions();
+            logger.info(`Extension ${name} removed`);
+            return utils.getResponse(message, {}, null);
+        } else {
+            return utils.getResponse(message, {}, `Extension ${name} doesn't exists`);
+        }
+    }
+    saveExtension(message) {
+        const {name, code} = message;
+        const ModuleConstructor = utils.loadModuleFromText(code);
+        this.loadExtension(ModuleConstructor);
+        const basePath = this.getExtensionsBasePath();
+        /* istanbul ignore else */
+        if (!fs.existsSync(basePath)) {
+            fs.mkdirSync(basePath);
+        }
+        const extensonFilePath = path.join(basePath, path.basename(name));
+        fs.writeFileSync(extensonFilePath, code);
+        this.publishExtensions();
+        logger.info(`Extension ${name} loaded`);
+        return utils.getResponse(message, {}, null);
+    }
+
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    loadExtension(ConstructorClass) {
+        this.enableDisableExtension(false, ConstructorClass.name);
+        this.addExtension(new ConstructorClass(...this.args, settings, logger));
+    }
+
+    loadUserDefinedExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        extensions
+            .map(({code}) => utils.loadModuleFromText(code))
+            .map(this.loadExtension);
+    }
+    async onMQTTConnected() {
+        this.publishExtensions();
+    }
+
+    async publishExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        await this.mqtt.publish('bridge/extensions', stringify(extensions), {
+            retain: true,
+            qos: 0,
+        }, settings.get().mqtt.base_topic, true);
+    }
+}
+
+module.exports = ExternalExtension;

lib/extension/frontend.js

@@ -19,8 +19,8 @@ class Frontend extends Extension {
         this.onRequest = this.onRequest.bind(this);
         this.onUpgrade = this.onUpgrade.bind(this);
         this.mqttBaseTopic = settings.get().mqtt.base_topic;
-        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
-        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
+        this.onMQTTPublishMessage = this.onMQTTPublishMessage.bind(this);
+        this.mqtt.on('publishMessage', this.onMQTTPublishMessage);
         this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
         this.server = http.createServer(this.onRequest);
         this.server.on('upgrade', this.onUpgrade);
@@ -46,6 +46,7 @@ class Frontend extends Extension {
 
     async stop() {
         for (const client of this.wss.clients) {
+            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
             client.close();
         }
 
@@ -105,7 +106,7 @@ class Frontend extends Extension {
         }
     }
 
-    onMQTTPublishedMessage(data) {
+    onMQTTPublishMessage(data) {
         let {topic, payload, options} = data;
         if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
             // Send topic without base_topic

lib/extension/groups.js

@@ -11,6 +11,21 @@ const topicRegex =
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
+const stateProperties = {
+    'state': (value, definition) => true,
+    'brightness': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
 class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
@@ -82,20 +97,26 @@ class Groups extends Extension {
             return;
         }
 
-        const properties = ['state', 'brightness', 'color_temp', 'color'];
         const payload = {};
 
-        properties.forEach((prop) => {
-            if (data.to.hasOwnProperty(prop)) {
-                payload[prop] = data.to[prop];
+        let endpointName;
+        for (let [prop, value] of Object.entries(data.update)) {
+            const endpointNameMatch = utils.getEndpointNames().find((n) => prop.endsWith(`_${n}`));
+            if (endpointNameMatch) {
+                prop = prop.substring(0, prop.length - endpointNameMatch.length - 1);
+                endpointName = endpointNameMatch;
             }
-        });
+
+            if (prop in stateProperties) {
+                payload[prop] = value;
+            }
+        }
 
         if (Object.keys(payload).length) {
-            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
+            const resolvedEntity = this.zigbee.resolveEntity(`${data.ID}${endpointName ? `/${endpointName}` : ''}`);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
-                return settingsGroup && settingsGroup.optimistic;
+                return settingsGroup && (!settingsGroup.hasOwnProperty('optimistic') || settingsGroup.optimistic);
             });
 
             if (resolvedEntity.type === 'device') {
@@ -109,9 +130,28 @@ class Groups extends Extension {
                     }
                 }
             } else {
+                // Invalidate the last optimistic group state when group state is changed directly.
+                delete this.lastOptimisticState[resolvedEntity.group.groupID];
+
                 const groupIDsToPublish = new Set();
                 for (const member of resolvedEntity.group.members) {
-                    await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
+                    const resolvedEntity = this.zigbee.resolveEntity(member);
+                    const memberPayload = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], resolvedEntity.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = resolvedEntity.endpointName;
+                    if (endpointName) {
+                        Object.keys(memberPayload).forEach((key) => {
+                            memberPayload[`${key}_${endpointName}`] = memberPayload[key];
+                            delete memberPayload[key];
+                        });
+                    }
+
+                    await this.publishEntityState(member.getDevice().ieeeAddr, memberPayload, reason);
                     for (const zigbeeGroup of zigbeeGroups) {
                         if (zigbeeGroup.hasMember(member)) {
                             if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
@@ -150,6 +190,7 @@ class Groups extends Extension {
         let groupKey = null;
         let deviceKey = null;
         let triggeredViaLegacyApi = false;
+        let skipDisableReporting = false;
 
         /* istanbul ignore else */
         const topicRegexMatch = topic.match(topicRegex);
@@ -205,6 +246,7 @@ class Groups extends Extension {
             type = topicRegexMatch[1];
             message = JSON.parse(message);
             deviceKey = message.device;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
 
             if (type !== 'remove_all') {
                 groupKey = message.group;
@@ -224,16 +266,17 @@ class Groups extends Extension {
 
         return {
             resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
-            triggeredViaLegacyApi,
+            triggeredViaLegacyApi, skipDisableReporting,
         };
     }
 
     async onMQTTMessage(topic, message) {
         let {
             resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
-            groupKey, deviceKey,
+            groupKey, deviceKey, skipDisableReporting,
         } = this.parseMQTTMessage(topic, message);
         if (!type) return;
+        message = utils.parseJSON(message, message);
 
         const responseData = {device: deviceKey};
         if (groupKey) {
@@ -318,7 +361,9 @@ class Groups extends Extension {
         if (error) {
             logger.error(error);
         } else {
-            this.eventBus.emit('groupMembersChanged', {group: resolvedEntityGroup});
+            this.eventBus.emit('groupMembersChanged',
+                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint,
+                    skipDisableReporting: skipDisableReporting});
         }
     }
 }

lib/extension/legacy/bridgeLegacy.js

@@ -382,7 +382,7 @@ class BridgeLegacy extends Extension {
             coordinator,
             network: await this.zigbee.getNetworkParameters(),
             log_level: logger.getLevel(),
-            permit_join: await this.zigbee.getPermitJoin(),
+            permit_join: this.zigbee.getPermitJoin(),
         };
 
         await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});

lib/extension/legacy/report.js

@@ -1,11 +1,9 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const logger = require('../util/logger');
-const settings = require('../util/settings');
+const logger = require('../../util/logger');
+const settings = require('../../util/settings');
 const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
-const utils = require('../util/utils');
-const Extension = require('./extension');
-const debounce = require('debounce');
-const ZigbeeHerdsman = require('zigbee-herdsman');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
 
 const defaultConfiguration = {
     minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
@@ -60,64 +58,11 @@ const clusters = {
     ],
 };
 
-const pollOnMessage = [
-    {
-        // Key is used this.pollDebouncers uniqueness
-        key: 1,
-        // On messages that have the cluster and type of below
-        cluster: {
-            manuSpecificPhilips: [
-                {type: 'commandHueNotification', data: {button: 2}},
-                {type: 'commandHueNotification', data: {button: 3}},
-            ],
-            genLevelCtrl: [
-                {type: 'commandStep', data: {}},
-                {type: 'commandStepWithOnOff', data: {}},
-                {type: 'commandStop', data: {}},
-                {type: 'commandMoveWithOnOff', data: {}},
-                {type: 'commandStopWithOnOff', data: {}},
-                {type: 'commandMove', data: {}},
-            ],
-        },
-        // Read the following attributes
-        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
-        // When the bound devices/members of group have the following manufacturerIDs
-        manufacturerIDs: [
-            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
-            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
-            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
-            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
-        ],
-    },
-    {
-        key: 2,
-        cluster: {
-            genOnOff: [
-                {type: 'commandOn', data: {}},
-                {type: 'commandOff', data: {}},
-                {type: 'commandOffWithEffect', data: {}},
-            ],
-            manuSpecificPhilips: [
-                {type: 'commandHueNotification', data: {button: 1}},
-                {type: 'commandHueNotification', data: {button: 4}},
-            ],
-        },
-        read: {cluster: 'genOnOff', attributes: ['onOff']},
-        manufacturerIDs: [
-            ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
-            ZigbeeHerdsman.Zcl.ManufacturerCode.ATMEL,
-            ZigbeeHerdsman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
-            ZigbeeHerdsman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
-        ],
-    },
-];
-
 class Report extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.queue = new Set();
         this.failed = new Set();
-        this.pollDebouncers = {};
         this.enabled = settings.get().advanced.report;
     }
 
@@ -242,61 +187,6 @@ class Report extends Extension {
         if (this.shouldSetupReporting(resolvedEntity, type)) {
             await this.setupReporting(resolvedEntity);
         }
-
-        if (type === 'message') {
-            this.poll(data);
-        }
-    }
-
-    poll(message) {
-        /**
-         * This method poll bound endpoints and group members for state changes.
-         *
-         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
-         * Hue bulbs only report their on/off state.
-         * When dimming the bulb via the dimmer switch the state is therefore not reported.
-         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
-         */
-        const polls = pollOnMessage.filter((p) =>
-            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
-            utils.equalsPartial(message.data, c.data)),
-        );
-
-        if (polls.length) {
-            let toPoll = [];
-
-            // Add bound devices
-            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
-                e.binds.map((e) => e).filter((e) => e.target))));
-            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
-            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
-            toPoll = toPoll.map((e) => e.target);
-
-            // If message is published to a group, add members of the group
-            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
-            if (group) {
-                toPoll = toPoll.concat(group.members);
-            }
-
-            toPoll = new Set(toPoll);
-
-            for (const endpoint of toPoll) {
-                for (const poll of polls) {
-                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
-                        continue;
-                    }
-
-                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${poll.key}`;
-                    if (!this.pollDebouncers[key]) {
-                        this.pollDebouncers[key] = debounce(async () => {
-                            await endpoint.read(poll.read.cluster, poll.read.attributes);
-                        }, 1000);
-                    }
-
-                    this.pollDebouncers[key]();
-                }
-            }
-        }
     }
 }
 

lib/extension/networkMap.js

@@ -84,7 +84,7 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
@@ -132,7 +132,7 @@ class NetworkMap extends Extension {
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
-                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const textRoutes = e.routes.map((r) => utils.toNetworkAddressHex(r.destinationAddress));
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
                 text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
@@ -160,7 +160,7 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             text.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
             );
 
@@ -212,9 +212,25 @@ class NetworkMap extends Extension {
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
+            await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
             const resolvedEntity = this.zigbee.resolveEntity(device);
+
+            const doRequest = async (request, firstAttempt = true) => {
+                try {
+                    return await request();
+                } catch (error) {
+                    if (!firstAttempt) {
+                        throw error;
+                    } else {
+                        // Network is possibly congested, sleep 5 seconds to let the network settle.
+                        await utils.sleep(5);
+                        return await doRequest(request, false);
+                    }
+                }
+            };
+
             try {
-                const result = await device.lqi();
+                const result = await doRequest(async () => await device.lqi());
                 lqis.set(device, result);
                 logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
@@ -224,7 +240,7 @@ class NetworkMap extends Extension {
 
             if (includeRoutes) {
                 try {
-                    const result = await device.routingTable();
+                    const result = await doRequest(async () => await device.routingTable());
                     routingTables.set(device, result);
                     logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {
@@ -244,7 +260,7 @@ class NetworkMap extends Extension {
                 model: resolvedEntity.definition.model,
                 vendor: resolvedEntity.definition.vendor,
                 description: resolvedEntity.definition.description,
-                supports: Array.from(new Set((resolvedEntity.definition.exposes || []).map((e) => {
+                supports: Array.from(new Set((resolvedEntity.definition.exposes).map((e) => {
                     return e.hasOwnProperty('name') ?
                         e.name :
                         `${e.type} (${e.features.map((f) => f.name).join(', ')})`;

lib/extension/otaUpdate.js

@@ -8,7 +8,6 @@ const topicRegex =
     new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
 
 const Extension = require('./extension');
-const MINUTES_10 = 1000 * 60 * 10;
 
 class OTAUpdate extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
@@ -36,15 +35,17 @@ class OTAUpdate extends Extension {
     }
 
     async onZigbeeEvent(type, data, resolvedEntity) {
+        if (settings.get().ota.disable_automatic_update_check) return;
         if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
 
         const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
         if (supportsOTA) {
             // When a device does a next image request, it will usually do it a few times after each other
-            // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
-            // each time.
+            // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
+            // each time, so this interval can be set by the user. The default is 1,440 minutes (one day).
+            const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
             const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
-                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > MINUTES_10 : true;
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
             if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
@@ -199,7 +200,7 @@ class OTAUpdate extends Extension {
                     const onProgress = (progress, remaining) => {
                         let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
                         if (remaining) {
-                            msg += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                            msg += `, ≈ ${Math.round(remaining / 60)} minutes remaining`;
                         }
 
                         logger.info(msg);
@@ -234,6 +235,7 @@ class OTAUpdate extends Extension {
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
                     }
                 } catch (e) {
+                    logger.debug(`Update of '${resolvedEntity.name}' failed (${e})`);
                     error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
 
                     this.removeProgressAndRemainingFromState(resolvedEntity.device);

lib/extension/publish.js

@@ -160,19 +160,28 @@ class EntityPublish extends Extension {
 
         // For each attribute call the corresponding converter
         const usedConverters = {};
+        const toPublish = {};
+        const addToToPublish = (ID, payload) => {
+            if (!(ID in toPublish)) toPublish[ID] = {};
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
         for (let [key, value] of entries) {
             let endpointName = topic.endpointName;
             let actualTarget = target;
+            let endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
                 const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
                 if (endpointNameMatch) {
                     endpointName = endpointNameMatch;
-                    key = key.replace(`_${endpointNameMatch}`, '');
+                    const regex = new RegExp(`(_${endpointNameMatch})$`, 'gm');
+                    key = key.replace(regex, ``);
 
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+                    endpointOrGroupID = endpointName;
                     if (!actualTarget) {
                         logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
                         continue;
@@ -180,7 +189,6 @@ class EntityPublish extends Extension {
                 }
             }
 
-            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
             if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
@@ -222,14 +230,12 @@ class EntityPublish extends Extension {
                 if (topic.type === 'set' && converter.convertSet) {
                     logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
-                    const isGroup = actualTarget.constructor.name == 'Group';
-                    // Optimistic behaviour of groups and devices is different.
-                    const publishOptimistic = isGroup || (!options.hasOwnProperty('optimistic') || options.optimistic);
-                    if (result && result.state && publishOptimistic) {
+                    const optimistic = !options.hasOwnProperty('optimistic') || options.optimistic;
+                    if (result && result.state && optimistic) {
                         const msg = result.state;
 
                         if (endpointName) {
-                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp', 'color_mode']) {
                                 if (msg.hasOwnProperty(key)) {
                                     msg[`${key}_${endpointName}`] = msg[key];
                                     delete msg[key];
@@ -237,12 +243,17 @@ class EntityPublish extends Extension {
                             }
                         }
 
-                        this.publishEntityState(resolvedEntity.settings.ID, msg);
+                        // filter out attribute listed in filtered_optimistic
+                        if (options.hasOwnProperty('filtered_optimistic')) {
+                            options.filtered_optimistic.forEach((a) => delete msg[a]);
+                        }
+
+                        addToToPublish(resolvedEntity.settings.ID, msg);
                     }
 
-                    if (result && result.membersState && publishOptimistic) {
+                    if (result && result.membersState && optimistic) {
                         for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
-                            this.publishEntityState(ieeeAddr, state);
+                            addToToPublish(ieeeAddr, state);
                         }
                     }
 
@@ -253,6 +264,7 @@ class EntityPublish extends Extension {
                     // When reporting is requested for a device (report: true in device-specific settings) we won't
                     // ever issue a read here, as we assume the device will properly report changes.
                     // Only do this when the retrieve_state option is enabled for this device.
+                    // retrieve_state == decprecated
                     if (
                         resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
                         resolvedEntity.settings.retrieve_state
@@ -285,6 +297,10 @@ class EntityPublish extends Extension {
             usedConverters[endpointOrGroupID].push(converter);
         }
 
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            this.publishEntityState(ID, payload);
+        }
+
         return true;
     }
 }

lib/extension/receive.js

@@ -119,7 +119,7 @@ class Receive extends Extension {
         });
 
         // Check if there is an available converter, genOta messages are not interesting.
-        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
+        if (!converters.length && !['genOta', 'genTime', 'genBasic'].includes(data.cluster)) {
             logger.debug(
                 `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
                 `and type '${data.type}' and data '${stringify(data.data)}'`,

lib/mqtt.js

@@ -116,6 +116,8 @@ class MQTT extends events.EventEmitter {
             this.publishedTopics.add(topic);
         }
 
+        this.emit('publishMessage', {topic, payload, options});
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -135,7 +137,6 @@ class MQTT extends events.EventEmitter {
 
         return new Promise((resolve) => {
             this.client.publish(topic, payload, actualOptions, () => {
-                this.emit('publishedMessage', {topic, payload, options});
                 resolve();
             });
         });

lib/state.js

@@ -75,8 +75,10 @@ class State {
         return this.state[ID];
     }
 
-    set(ID, state, reason=null) {
-        const toState = objectAssignDeep.noMutate(state);
+    set(ID, update, reason=null) {
+        const fromState = this.state[ID] || {};
+        const toState = objectAssignDeep.noMutate(fromState, update);
+        const result = {...toState};
 
         for (const property of Object.keys(toState)) {
             if (dontCacheProperties.find((p) => property.match(p))) {
@@ -84,11 +86,9 @@ class State {
             }
         }
 
-        const fromState = this.state[ID];
-
         this.state[ID] = toState;
-
-        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason, update});
+        return result;
     }
 
     removeKey(ID, path) {

lib/util/logger.js

@@ -19,12 +19,25 @@ const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', time
 // Make sure that log directoy exsists when not logging to stdout only
 if (output.includes('file')) {
     fx.mkdirSync(directory);
+
+    if (settings.get().advanced.log_symlink_current) {
+        const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
+        const actual = './' + timestamp;
+        if (fs.existsSync(current)) {
+            fs.unlinkSync(current);
+        }
+        fs.symlinkSync(actual, current);
+    }
 }
 
+const z2mToWinstonLevel = (level) => level === 'warn' ? 'warning' : level;
+const winstonToZ2mLevel = (level) => level === 'warning' ? 'warn' : level;
+
 // Determine the log level.
-const level = settings.get().advanced.log_level;
+let level = settings.get().advanced.log_level;
 const validLevels = ['info', 'error', 'warn', 'debug'];
 assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
+level = z2mToWinstonLevel(level);
 
 const levelWithCompensatedLength = {
     'info': 'info ',
@@ -45,7 +58,7 @@ const transportsToUse = [
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
                 let {timestamp, level, message} = info;
-                level = level === 'warning' ? 'warn' : level;
+                level = winstonToZ2mLevel(level);
                 const prefix = colorizer.colorize(level, `Zigbee2MQTT:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
@@ -63,7 +76,7 @@ const transportFileOptions = {
         winston.format.timestamp({format: timestampFormat}),
         winston.format.printf(/* istanbul ignore next */(info) => {
             let {timestamp, level, message} = info;
-            level = level === 'warning' ? 'warn' : level;
+            level = winstonToZ2mLevel(level);
             return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
         }),
     ),
@@ -129,11 +142,16 @@ function logOutput() {
     }
 }
 
+logger.addTransport = (transport) => {
+    transport.level = transportsToUse[0].level;
+    logger.add(transport);
+};
 logger.cleanup = cleanup;
 logger.logOutput = logOutput;
-logger.getLevel = () => transportsToUse[0].level;
+logger.getLevel = () => winstonToZ2mLevel(transportsToUse[0].level);
 logger.setLevel = (level) => {
-    transportsToUse.forEach((transport) => transport.level = level);
+    level = z2mToWinstonLevel(level);
+    logger.transports.forEach((transport) => transport.level = level);
 };
 
 // winston.config.syslog.levels doesnt have warn, but is required for syslog.

lib/util/settings.js

@@ -6,7 +6,10 @@ const objectAssignDeep = require(`object-assign-deep`);
 const path = require('path');
 const yaml = require('./yaml');
 const Ajv = require('ajv');
-const ajv = new Ajv({allErrors: true});
+const schema = require('./settings.schema.json');
+const ajvSetting = new Ajv({allErrors: true}).compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword('requiresRestart', {validate: (v) => !v}).compile(schema);
 
 const defaults = {
     passlist: [],
@@ -54,6 +57,7 @@ const defaults = {
     advanced: {
         legacy_api: true,
         log_rotation: true,
+        log_symlink_current: false,
         log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
         log_file: 'log.txt',
@@ -121,6 +125,15 @@ const defaults = {
          */
         homeassistant_status_topic: 'hass/status',
 
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
         /**
          * Home Assistant legacy triggers, when enabled:
          * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
@@ -134,175 +147,20 @@ const defaults = {
          */
         timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
+    ota: {
+        /**
+         * Minimal time delta in minutes between polling third party server for potential firmware updates
+         */
+        update_check_interval: 24 * 60,
+        /**
+         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
+         * If set to true, only a user-initiated update search will be possible.
+         */
+        disable_automatic_update_check: false,
+    },
     external_converters: [],
 };
 
-const schema = {
-    type: 'object',
-    properties: {
-        device_options: {type: 'object'},
-        homeassistant: {type: 'boolean'},
-        permit_join: {type: 'boolean'},
-        mqtt: {
-            type: 'object',
-            properties: {
-                base_topic: {type: 'string'},
-                server: {type: 'string'},
-                keepalive: {type: 'number'},
-                ca: {type: 'string'},
-                key: {type: 'string'},
-                cert: {type: 'string'},
-                user: {type: 'string'},
-                password: {type: 'string'},
-                client_id: {type: 'string'},
-                reject_unauthorized: {type: 'boolean'},
-                include_device_information: {type: 'boolean'},
-                version: {type: 'number'},
-                force_disable_retain: {type: 'boolean'},
-            },
-            required: ['base_topic', 'server'],
-        },
-        serial: {
-            type: 'object',
-            properties: {
-                port: {type: ['string', 'null']},
-                disable_led: {type: 'boolean'},
-                adapter: {type: 'string', enum: ['deconz', 'zstack', 'zigate']},
-            },
-        },
-        blocklist: {type: 'array', items: {type: 'string'}},
-        passlist: {type: 'array', items: {type: 'string'}},
-        // Deprecated: use block/passlist
-        whitelist: {type: 'array', items: {type: 'string'}},
-        ban: {type: 'array', items: {type: 'string'}},
-        experimental: {
-            type: 'object',
-            properties: {
-                transmit_power: {type: 'number'},
-                output: {type: 'string', enum: ['attribute_and_json', 'attribute', 'json']},
-            },
-        },
-        advanced: {
-            type: 'object',
-            properties: {
-                legacy_api: {type: 'boolean'},
-                pan_id: {type: 'number'},
-                ext_pan_id: {type: 'array', items: {type: 'number'}},
-                channel: {type: 'number', minimum: 11, maximum: 26},
-                cache_state: {type: 'boolean'},
-                cache_state_persistent: {type: 'boolean'},
-                cache_state_send_on_startup: {type: 'boolean'},
-                log_rotation: {type: 'boolean'},
-                log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
-                log_output: {type: 'array', items: {type: 'string'}},
-                log_directory: {type: 'string'},
-                log_file: {type: 'string'},
-                baudrate: {type: 'number'},
-                rtscts: {type: 'boolean'},
-                soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: ['array', 'string'], items: {type: 'number'}},
-                last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
-                elapsed: {type: 'boolean'},
-                availability_timeout: {type: 'number', minimum: 0},
-                availability_blocklist: {type: 'array', items: {type: 'string'}},
-                availability_passlist: {type: 'array', items: {type: 'string'}},
-                // Deprecated, use block/passlist
-                availability_blacklist: {type: 'array', items: {type: 'string'}},
-                availability_whitelist: {type: 'array', items: {type: 'string'}},
-                report: {type: 'boolean'},
-                homeassistant_discovery_topic: {type: 'string'},
-                homeassistant_status_topic: {type: 'string'},
-                timestamp_format: {type: 'string'},
-                adapter_concurrent: {type: 'number'},
-                adapter_delay: {type: 'number'},
-            },
-        },
-        map_options: {
-            type: 'object',
-            properties: {
-                graphviz: {
-                    type: 'object',
-                    properties: {
-                        colors: {
-                            type: 'object',
-                            properties: {
-                                fill: {
-                                    type: 'object',
-                                    properties: {
-                                        enddevice: {type: 'string'},
-                                        coordinator: {type: 'string'},
-                                        router: {type: 'string'},
-                                    },
-                                },
-                                font: {
-                                    type: 'object',
-                                    properties: {
-                                        enddevice: {type: 'string'},
-                                        coordinator: {type: 'string'},
-                                        router: {type: 'string'},
-                                    },
-                                },
-                                line: {
-                                    type: 'object',
-                                    properties: {
-                                        active: {type: 'string'},
-                                        inactive: {type: 'string'},
-                                    },
-                                },
-                            },
-                        },
-                    },
-                },
-            },
-        },
-        devices: {
-            type: 'object',
-            propertyNames: {
-                pattern: '^0x[\\d\\w]{16}$',
-            },
-            patternProperties: {
-                '^.*$': {
-                    type: 'object',
-                    properties: {
-                        friendly_name: {type: 'string'},
-                        retain: {type: 'boolean'},
-                        retention: {type: 'number'},
-                        qos: {type: 'number'},
-                        filtered_attributes: {type: 'array', items: {type: 'string'}},
-                    },
-                    required: ['friendly_name'],
-                },
-            },
-        },
-        groups: {
-            type: 'object',
-            propertyNames: {
-                pattern: '^[\\w].*$',
-            },
-            patternProperties: {
-                '^.*$': {
-                    type: 'object',
-                    properties: {
-                        friendly_name: {type: 'string'},
-                        retain: {type: 'boolean'},
-                        devices: {type: 'array', items: {type: 'string'}},
-                        optimistic: {type: 'boolean'},
-                        qos: {type: 'number'},
-                        filtered_attributes: {type: 'array', items: {type: 'string'}},
-                    },
-                    required: ['friendly_name'],
-                },
-            },
-        },
-    },
-    external_converters: {
-        type: 'array',
-        items: {type: 'string'},
-    },
-    required: ['homeassistant', 'permit_join', 'mqtt'],
-};
-
-
 let _settings;
 let _settingsWithDefaults;
 
@@ -314,7 +172,12 @@ function write() {
     const actual = yaml.read(file);
 
     // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
-    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+    for (const path of [
+        ['mqtt', 'user'],
+        ['mqtt', 'password'],
+        ['advanced', 'network_key'],
+        ['frontend', 'auth_token'],
+    ]) {
         if (actual[path[0]] && actual[path[0]][path[1]]) {
             const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
             if (match) {
@@ -366,10 +229,8 @@ function validate() {
         return [error.message];
     }
 
-    const validate = ajv.compile(schema);
-    validate(_settings);
-    if (validate.errors) {
-        return validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+    if (!ajvSetting(_settings)) {
+        return ajvSetting.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
     }
 
     const errors = [];
@@ -378,6 +239,11 @@ function validate() {
         errors.push(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
     }
 
+    if (_settings.advanced && _settings.advanced.pan_id && typeof _settings.advanced.pan_id === 'string' &&
+        _settings.advanced.pan_id !== 'GENERATE') {
+        errors.push(`advanced.pan_id: should be number or 'GENERATE' (is '${_settings.advanced.pan_id}')`);
+    }
+
     // Verify that all friendly names are unique
     const names = [];
     const check = (name) => {
@@ -439,6 +305,10 @@ function read() {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
+    if (s.frontend && s.frontend.auth_token) {
+        s.frontend.auth_token = interpetValue(s.frontend.auth_token);
+    }
+
     // Read devices/groups configuration from separate file if specified.
     const readDevicesOrGroups = (type) => {
         if (typeof s[type] === 'string' || Array.isArray(s[type])) {
@@ -461,7 +331,7 @@ function applyEnvironmentVariables(settings) {
     const iterate = (obj, path) => {
         Object.keys(obj).forEach((key) => {
             if (key !== 'type') {
-                if (key !== 'properties') {
+                if (key !== 'properties' && obj[key]) {
                     const type = (obj[key].type || 'object').toString();
                     const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
                     const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
@@ -486,7 +356,7 @@ function applyEnvironmentVariables(settings) {
                     }
                 }
 
-                if (typeof obj[key] === 'object') {
+                if (typeof obj[key] === 'object' && obj[key]) {
                     const newPath = [...path];
                     if (key !== 'properties') {
                         newPath.push(key);
@@ -543,16 +413,34 @@ function set(path, value) {
     write();
 }
 
+function apply(newSettings) {
+    ajvSetting(newSettings);
+    const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
+    if (errors.length) {
+        const error = errors[0];
+        throw new Error(`${error.dataPath.substring(1)} ${error.message}`);
+    }
+
+    get(); // Ensure _settings is intialized.
+    _settings = objectAssignDeep.noMutate(_settings, newSettings);
+    write();
+
+    ajvRestartRequired(newSettings);
+    const restartRequired = ajvRestartRequired.errors &&
+        !!ajvRestartRequired.errors.find((e) => e.keyword === 'requiresRestart');
+    return restartRequired;
+}
+
 function getGroup(IDorName) {
     const settings = getWithDefaults();
     const byID = settings.groups[IDorName];
     if (byID) {
-        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+        return {devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
     }
 
     for (const [ID, group] of Object.entries(settings.groups)) {
         if (group.friendly_name === IDorName) {
-            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+            return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
         }
     }
 
@@ -562,7 +450,7 @@ function getGroup(IDorName) {
 function getGroups() {
     const settings = getWithDefaults();
     return Object.entries(settings.groups).map(([ID, group]) => {
-        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
     });
 }
 
@@ -669,10 +557,21 @@ function removeDevice(IDorName) {
     const device = getDeviceThrowIfNotExists(IDorName);
     const settings = get();
     delete settings.devices[device.ID];
+
+    // Remove device from groups
+    if (settings.groups) {
+        const regex =
+            new RegExp(`^(${device.friendly_name}|${device.ID})(/(\\d|${utils.getEndpointNames().join('|')}))?$`);
+        for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
+            group.devices = group.devices.filter((device) => !device.match(regex));
+        }
+    }
+
     write();
 }
 
 function addGroup(name, ID=null) {
+    utils.validateFriendlyName(name, true);
     if (getGroup(name) || getDevice(name)) {
         throw new Error(`friendly_name '${name}' is already in use`);
     }
@@ -756,8 +655,10 @@ function changeEntityOptions(IDorName, newOptions) {
     delete newOptions.devices;
     if (getDevice(IDorName)) {
         objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.devices[getDevice(IDorName).ID]);
     } else if (getGroup(IDorName)) {
         objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.groups[getGroup(IDorName).ID]);
     } else {
         throw new Error(`Device or group '${IDorName}' does not exist`);
     }
@@ -766,6 +667,7 @@ function changeEntityOptions(IDorName, newOptions) {
 }
 
 function changeFriendlyName(IDorName, newName) {
+    utils.validateFriendlyName(newName, true);
     if (getGroup(newName) || getDevice(newName)) {
         throw new Error(`friendly_name '${newName}' is already in use`);
     }
@@ -786,6 +688,7 @@ module.exports = {
     validate,
     get: getWithDefaults,
     set,
+    apply,
     getDevice,
     getGroup,
     getGroups,
@@ -801,15 +704,16 @@ module.exports = {
     removeDeviceFromGroup,
     changeEntityOptions,
     changeFriendlyName,
-
-    // For tests only
-    _write: write,
-    _reRead: () => {
+    schema,
+    reRead: () => {
         _settings = null;
         get();
         _settingsWithDefaults = null;
         getWithDefaults();
     },
+
+    // For tests only
+    _write: write,
     _clear: () => {
         _settings = null;
         _settingsWithDefaults = null;

lib/util/settings.schema.json

@@ -0,0 +1,767 @@
+{
+  "type": "object",
+  "properties": {
+    "device_options": {
+      "type": "object"
+    },
+    "homeassistant": {
+      "title": "Home Assistant integration",
+      "type": "boolean",
+      "description": "Home Assistant integration (MQTT discovery)",
+      "default": false
+    },
+    "permit_join": {
+      "type": "boolean",
+      "default": false,
+      "title": "Permit join",
+      "description": "Allow new devices to join (re-applied at restart)"
+    },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
+    },
+    "mqtt": {
+      "type": "object",
+      "title": "MQTT",
+      "properties": {
+        "base_topic": {
+          "type": "string",
+          "title": "Base topic",
+          "requiresRestart": true,
+          "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
+          "examples": ["zigbee2mqtt"]
+        },
+        "server": {
+          "type": "string",
+          "title": "MQTT server",
+          "requiresRestart": true,
+          "description": "MQTT server URL (use mqtts:// for SSL/TLS connection)",
+          "examples": ["mqtt://localhost:1883"]
+        },
+        "keepalive": {
+          "type": "number",
+          "title": "Keepalive",
+          "requiresRestart": true,
+          "description": "MQTT keepalive in second",
+          "default": 60
+        },
+        "ca": {
+          "type": "string",
+          "title": "Certificate authority",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS certificate of CA used to sign server and client certificates",
+          "examples": ["/etc/ssl/mqtt-ca.crt"]
+        },
+        "key": {
+          "type": "string",
+          "title": "SSL/TLS key",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS key for client-authentication",
+          "examples": ["/etc/ssl/mqtt-client.key"]
+        },
+        "cert": {
+          "type": "string",
+          "title": "SSL/TLS certificate",
+          "description": "Absolute path to SSL/TLS certificate for client-authentication",
+          "requiresRestart": true,
+          "examples": ["/etc/ssl/mqtt-client.crt"]
+        },
+        "user": {
+          "type": "string",
+          "title": "User",
+          "requiresRestart": true,
+          "description": "MQTT server authentication user",
+          "examples": ["johnnysilverhand"]
+        },
+        "password": {
+          "type": "string",
+          "title": "Password",
+          "requiresRestart": true,
+          "description": "MQTT server authentication password",
+          "examples": ["ILOVEPELMENI"]
+        },
+        "client_id": {
+          "type": "string",
+          "title": "Client ID",
+          "requiresRestart": true,
+          "description": "MQTT client ID",
+          "examples": ["MY_CLIENT_ID"]
+        },
+        "reject_unauthorized": {
+          "type": "boolean",
+          "title": "Reject unauthorized",
+          "requiresRestart": true,
+          "description": "Disable self-signed SSL certificate",
+          "default": true
+        },
+        "include_device_information": {
+          "type": "boolean",
+          "title": "Include device information",
+          "description": "Include device information to mqtt messages",
+          "default": false
+        },
+        "version": {
+          "type": ["number", "null"],
+          "title": "Version",
+          "requiresRestart": true,
+          "description": "MQTT protocol version",
+          "default": 4,
+          "examples": [5]
+        },
+        "force_disable_retain": {
+          "type": "boolean",
+          "title": "Force disable retain",
+          "requiresRestart": true,
+          "description": "Disable retain for all send messages. ONLY enable if you MQTT broker doesn't support retained message (e.g. AWS IoT core, Azure IoT Hub, Google Cloud IoT core, IBM Watson IoT Platform). Enabling will break the Home Assistant integration",
+          "default": false
+        }
+      },
+      "required": ["base_topic", "server"]
+    },
+    "serial": {
+      "type": "object",
+      "title": "Serial",
+      "properties": {
+        "port": {
+          "type": ["string", "null"],
+          "title": "Port",
+          "requiresRestart": true,
+          "description": "Location of the adapter. To autodetect the port, set null",
+          "examples": ["/dev/ttyACM0"]
+        },
+        "disable_led": {
+          "type": "boolean",
+          "title": "Disable led",
+          "requiresRestart": true,
+          "description": "Disable LED of the adapter if supported",
+          "default": false
+        },
+        "adapter": {
+          "type": ["string"],
+          "enum": ["deconz", "zstack", "zigate", "ezsp", "auto"],
+          "title": "Adapter",
+          "default": "auto",
+          "requiresRestart": true,
+          "description": "Adapter type, not needed unless you are experiencing problems"
+        }
+      }
+    },
+    "blocklist": {
+      "title": "Blocklist",
+      "requiresRestart": true,
+      "description": "Block devices from the network (by ieeeAddr)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "passlist": {
+      "title": "Passlist",
+      "requiresRestart": true,
+      "description": "Allow only certain devices to join the network (by ieeeAddr). Note that all devices not on the passlist will be removed from the network!",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
+      "properties": {
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [15, 20, 25]
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_symlink_current": {
+          "type": "boolean",
+          "title": "Log symlink current",
+          "requiresRestart": true,
+          "description": "Create symlink to current logs in the log directory",
+          "default": false
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to supress logging"
+        },
+        "log_directory": {
+          "type": "string",
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
+        },
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "availability_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "default": 0,
+          "requiresRestart": true,
+          "title": "Availability Timeout",
+          "description": "Availability timeout in seconds when enabled, devices will be checked if they are still online. Only AC powered routers are checked for availability"
+        },
+        "availability_blocklist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability Blocklist",
+          "description": "Prevent devices from being checked for availability"
+        },
+        "availability_passlist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability passlist",
+          "description": "Only enable availability check for certain devices"
+        },
+        "availability_blacklist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "title": "Availability blacklist (deprecated, use availability_blocklist)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "availability_whitelist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "items": {
+            "type": "string"
+          },
+          "title": "Availability whitelist (deprecated, use passlist)"
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+          "default": true
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        }
+      }
+    },
+    "map_options": {
+      "type": "object",
+      "title": "Networkmap",
+      "properties": {
+        "graphviz": {
+          "type": "object",
+          "properties": {
+            "colors": {
+              "type": "object",
+              "properties": {
+                "fill": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "font": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "line": {
+                  "type": "object",
+                  "properties": {
+                    "active": {
+                      "type": "string"
+                    },
+                    "inactive": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "ota": {
+      "type": "object",
+      "title": "OTA updates",
+      "properties": {
+        "update_check_interval": {
+          "type": "number",
+          "title": "Update check interval",
+          "description": "Your device may request a check for a new firmware update. This value determines how frequently third party servers may actually be contacted to look for firmware updates. The value is set in minutes, and the default is 1 day.",
+          "default": 1440
+        },
+        "disable_automatic_update_check": {
+          "type": "boolean",
+          "title": "Disable automatic update check",
+          "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
+          "default": false
+        }
+      }
+    },
+    "devices": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^0x[\\d\\w]{16}$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/device"
+        }
+      }
+    },
+    "groups": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[\\w].*$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/group"
+        }
+      }
+    },
+    "frontend": {
+      "type": "object",
+      "title": "Frontend",
+      "properties": {
+        "port": {
+          "type": "number",
+          "title": "Port",
+          "description": "Frontend binding port",
+          "default": 8080,
+          "requiresRestart": true
+        },
+        "host": {
+          "type": "string",
+          "title": "Bind host",
+          "description": "Frontend binding host",
+          "default":" 0.0.0.0",
+          "requiresRestart": true
+        },
+        "auth_token": {
+          "type": ["string", "null"],
+          "title": "Auth token",
+          "description": "Enables authentication, disabled by default",
+          "requiresRestart": true
+        }
+      }
+    }
+  },
+  "required": ["mqtt"],
+  "definitions": {
+    "device": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string",
+          "title": "Friendly name",
+          "description": "Used in the MQTT topic of a device. By default this is the device ID",
+          "readOnly": true
+        },
+        "retain": {
+          "type": "boolean",
+          "title": "Retain",
+          "description": "Retain MQTT messages of this device"
+        },
+        "retention": {
+          "type": "number",
+          "title": "Retention",
+          "description": "Sets the MQTT Message Expiry in seconds, Make sure to set mqtt.version to 5"
+        },
+        "qos": {
+          "type": "number",
+          "title": "QoS",
+          "descritption": "QoS level for MQTT messages of this device"
+        },
+        "debounce": {
+          "type": "number",
+          "title": "Debounce",
+          "description": "Debounces messages of this device"
+        },
+        "debounce_ignore": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["action"],
+          "title": "Ignore debounce",
+          "description": "Protects unique payload values of specified payload properties from overriding within debounce time"
+        },
+        "optimistic": {
+          "type": "boolean",
+          "title": "Optimistic",
+          "description": "Publish optimistic state after set",
+          "default": true
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["color_mode", "color_temp", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["temperature", "battery", "action"],
+          "title": "Filtered publish attributes",
+          "description": "Filter attributes from publish payload."
+        },
+        "icon": {
+          "type": "string",
+          "title": "Icon",
+          "description": "The user-defined device icon for the frontend. It can be a link to an image (not a path to a file) or base64 encoded data URL like: image/svg+xml;base64,PHN2ZyB3aW....R0aD"
+        }
+      },
+      "required": ["friendly_name"]
+    },
+    "group": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string"
+        },
+        "retain": {
+          "type": "boolean"
+        },
+        "devices": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "optimistic": {
+          "type": "boolean"
+        },
+        "qos": {
+          "type": "number"
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": ["friendly_name"]
+    }
+  }
+}

lib/util/utils.js

@@ -3,6 +3,7 @@ const humanizeDuration = require('humanize-duration');
 const data = require('./data');
 const vm = require('vm');
 const fs = require('fs');
+const path = require('path');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -35,12 +36,13 @@ const endpointNames = [
     'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
     'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
     'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
     'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
     'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
     'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
     'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-    'heat', 'cool', 'water',
+    'heat', 'cool', 'water', 'meter', 'wifi',
 ];
 
 function capitalize(s) {
@@ -124,7 +126,7 @@ function equalsPartial(object, expected) {
 }
 
 function getObjectProperty(object, key, defaultValue) {
-    return object.hasOwnProperty(key) ? object[key] : defaultValue;
+    return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
 }
 
 function getResponse(request, data, error) {
@@ -144,6 +146,28 @@ function parseJSON(value, failedReturnValue) {
     }
 }
 
+function loadModuleFromText(moduleCode) {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+        setTimeout,
+        clearTimeout,
+        setInterval,
+        clearInterval,
+        setImmediate,
+        clearImmediate,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    return sandbox.module.exports;
+}
+
+function loadModuleFromFile(modulePath) {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
 function* getExternalConvertersDefinitions(settings) {
     const externalConverters = settings.get().external_converters;
 
@@ -151,13 +175,7 @@ function* getExternalConvertersDefinitions(settings) {
         let converter;
 
         if (moduleName.endsWith('.js')) {
-            const sandbox = {
-                require,
-                module: {},
-            };
-            const converterCode = fs.readFileSync(data.joinPath(moduleName), {encoding: 'utf8'});
-            vm.runInNewContext(converterCode, sandbox);
-            converter = sandbox.module.exports;
+            converter = loadModuleFromFile(data.joinPath(moduleName));
         } else {
             converter = require(moduleName);
         }
@@ -172,6 +190,32 @@ function* getExternalConvertersDefinitions(settings) {
     }
 }
 
+function removeNullPropertiesFromObject(obj) {
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value == null) {
+            delete obj[key];
+        } else if (typeof value === 'object') {
+            removeNullPropertiesFromObject(value);
+        }
+    }
+}
+
+function getKey(object, value, fallback, convertTo) {
+    for (const key in object) {
+        if (object[key]===value) {
+            return convertTo ? convertTo(key) : key;
+        }
+    }
+
+    return fallback;
+}
+
+function toNetworkAddressHex(value) {
+    const hex = value.toString(16);
+    return `0x${'0'.repeat(4 - hex.length)}${hex}`;
+}
+
 function toSnakeCase(value) {
     if (typeof value === 'object') {
         value = {...value};
@@ -188,15 +232,39 @@ function toSnakeCase(value) {
     }
 }
 
-function validateFriendlyName(name) {
+function validateFriendlyName(name, throwFirstError=false) {
     const errors = [];
+    for (const endpointName of endpointNames) {
+        if (name.toLowerCase().endsWith('/' + endpointName)) {
+            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
+        }
+    }
+
+    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
+    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
+    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
     if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
     if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
     if (name.includes('#') || name.includes('+')) {
         errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
     }
+
+    if (throwFirstError && errors.length) {
+        throw new Error(errors[0]);
+    }
+
     return errors;
 }
+function sleep(seconds) {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+function sanitizeImageParameter(parameter) {
+    const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
+    let sanitized = parameter;
+    replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
+    return sanitized;
+}
 
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
@@ -205,6 +273,7 @@ module.exports = {
     getDependencyVersion,
     objectHasProperties,
     toSnakeCase,
+    sleep,
     getObjectProperty,
     getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
@@ -216,7 +285,13 @@ module.exports = {
     equalsPartial,
     getResponse,
     capitalize,
+    toNetworkAddressHex,
     parseJSON,
     getExternalConvertersDefinitions,
     validateFriendlyName,
+    loadModuleFromFile,
+    loadModuleFromText,
+    getKey,
+    sanitizeImageParameter,
+    removeNullPropertiesFromObject,
 };

lib/zigbee.js

@@ -40,6 +40,7 @@ class Zigbee extends events.EventEmitter {
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
                 delay: settings.get().advanced.adapter_delay,
+                disableLED: settings.get().serial.disable_led,
             },
         };
 
@@ -53,23 +54,33 @@ class Zigbee extends events.EventEmitter {
             herdsmanSettings.network.networkKey = newKey;
         }
 
+        if (herdsmanSettings.network.panID === 'GENERATE') {
+            const newPanID = Math.floor(Math.random() * (0xFFFF - 2)) + 1;
+            settings.set(['advanced', 'pan_id'], newPanID);
+            herdsmanSettings.network.panID = newPanID;
+        }
+
+        let startResult;
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
-            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
-            await this.herdsman.start();
+            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, logger);
+            startResult = await this.herdsman.start();
         } catch (error) {
             logger.error(`Error while starting zigbee-herdsman`);
             throw error;
         }
 
         this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
+        this.herdsman.on('deviceNetworkAddressChanged', (data) =>
+            this.emit('event', 'deviceNetworkAddressChanged', data));
         this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
         this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
         this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
         this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
         this.herdsman.on('message', (data) => this.emit('event', 'message', data));
+        this.herdsman.on('permitJoinChanged', (data) => this.emit('permitJoinChanged', data));
 
-        logger.info('zigbee-herdsman started');
+        logger.info(`zigbee-herdsman started (${startResult})`);
         logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
         logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
@@ -99,12 +110,18 @@ class Zigbee extends events.EventEmitter {
             await this.herdsman.setTransmitPower(transmitPower);
             logger.info(`Set transmit power to '${transmitPower}'`);
         }
+
+        return startResult;
     }
 
     async getCoordinatorVersion() {
         return this.herdsman.getCoordinatorVersion();
     }
 
+    isStopping() {
+        return this.herdsman.isStopping();
+    }
+
     async getNetworkParameters() {
         return this.herdsman.getNetworkParameters();
     }
@@ -114,26 +131,31 @@ class Zigbee extends events.EventEmitter {
     }
 
     async stop() {
+        logger.info('Stopping zigbee-herdsman...');
         await this.herdsman.stop();
-        logger.info('zigbee-herdsman stopped');
+        logger.info('Stopped zigbee-herdsman');
     }
 
-    async permitJoin(permit, resolvedEntity) {
+    async permitJoin(permit, resolvedEntity, time=undefined) {
         permit ?
             logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`) :
             logger.info('Zigbee: disabling joining new devices.');
 
         if (resolvedEntity && permit) {
-            await this.herdsman.permitJoin(permit, resolvedEntity.device);
+            await this.herdsman.permitJoin(permit, resolvedEntity.device, time);
         } else {
-            await this.herdsman.permitJoin(permit);
+            await this.herdsman.permitJoin(permit, undefined, time);
         }
     }
 
-    async getPermitJoin() {
+    getPermitJoin() {
         return this.herdsman.getPermitJoin();
     }
 
+    getPermitJoinTimeout() {
+        return this.herdsman.getPermitJoinTimeout();
+    }
+
     getClients() {
         return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
     }
@@ -168,10 +190,16 @@ class Zigbee extends events.EventEmitter {
     resolveEntity(key) {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            key.constructor.name === 'Device' || key.constructor.name === 'Group' ||
+            key.constructor.name === 'Endpoint',
             `Wrong type '${typeof key}'`,
         );
 
+        const getEndpointName = (endpointNames, endpoint) => {
+            return endpoint ?
+                utils.getKey(endpointNames, endpoint.ID, null, ((v) => v === 'default' ? null : v)) : null;
+        };
+
         const deviceOptions = settings.get().device_options;
         if (typeof key === 'string' || typeof key === 'number') {
             if (typeof key === 'number') {
@@ -186,6 +214,7 @@ class Zigbee extends events.EventEmitter {
                     endpoint: coordinator.getEndpoint(1),
                     settings: {friendlyName: 'Coordinator'},
                     name: 'Coordinator',
+                    endpointName: null,
                 };
             }
 
@@ -208,20 +237,21 @@ class Zigbee extends events.EventEmitter {
                 }
 
                 const definition = zigbeeHerdsmanConverters.findByDevice(device);
-                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
+                const endpointNames = definition && definition.endpoint ? definition.endpoint(device) : null;
                 let endpoint;
                 if (endpointKey) {
                     if (endpointByNumber) {
                         endpoint = device.getEndpoint(endpointKey);
                     } else {
                         assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
-                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
-                        const endpointID = endpoints[endpointKey];
+                        assert(endpointNames != null,
+                            `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpointNames[endpointKey];
                         assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
                         endpoint = device.getEndpoint(endpointID);
                     }
-                } else if (endpoints && endpoints['default']) {
-                    endpoint = device.getEndpoint(endpoints['default']);
+                } else if (endpointNames && endpointNames['default']) {
+                    endpoint = device.getEndpoint(endpointNames['default']);
                 } else {
                     endpoint = device.endpoints[0];
                 }
@@ -229,21 +259,29 @@ class Zigbee extends events.EventEmitter {
                 return {
                     type: 'device', device, endpoint, settings: {...deviceOptions, ...entity},
                     name: entity.friendlyName, definition,
+                    endpointName: getEndpointName(endpointNames, endpoint),
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
                 return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
             }
-        } else if (key.constructor.name === 'Device') {
-            const setting = settings.getEntity(key.ieeeAddr);
+        } else if (key.constructor.name === 'Device' || key.constructor.name === 'Endpoint') {
+            const device = key.constructor.name === 'Endpoint' ? key.getDevice() : key;
+            const setting = settings.getEntity(device.ieeeAddr);
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const name = setting ? setting.friendlyName :
+                (device.type === 'Coordinator' ? 'Coordinator' : device.ieeeAddr);
+            const endpointNames = definition && definition.endpoint ? definition.endpoint(device) : null;
+
+            let endpoint;
+            if (key.constructor.name === 'Endpoint') endpoint = key;
+            else if (endpointNames && endpointNames['default']) endpoint = device.getEndpoint(endpointNames['default']);
+            else endpoint = device.endpoints[0];
+
             return {
-                type: 'device',
-                device: key,
-                endpoint: key.endpoints[0],
-                settings: {...deviceOptions, ...(setting || {})},
-                name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
-                definition: zigbeeHerdsmanConverters.findByDevice(key),
+                type: 'device', definition, name, device, endpoint, settings: {...deviceOptions, ...(setting || {})},
+                endpointName: getEndpointName(endpointNames, endpoint),
             };
         } else { // Group
             const setting = settings.getEntity(key.groupID);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.17.0",
+  "version": "1.20.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^14 || ^15"
+    "node": "^10 || ^12 || ^14 || ^15 || ^16"
   },
   "keywords": [
     "xiaomi",
@@ -35,6 +35,7 @@
   "dependencies": {
     "ajv": "^6.12.4",
     "debounce": "^1.2.0",
+    "deep-object-diff": "^1.1.0",
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.1.2",
     "git-last-commit": "^1.0.0",
@@ -51,9 +52,9 @@
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
     "ws": "^7.3.1",
-    "zigbee-herdsman": "0.13.46",
-    "zigbee-herdsman-converters": "14.0.3",
-    "zigbee2mqtt-frontend": "0.2.129"
+    "zigbee-herdsman": "0.13.117",
+    "zigbee-herdsman-converters": "14.0.185",
+    "zigbee2mqtt-frontend": "0.4.29"
   },
   "devDependencies": {
     "eslint": "*",
@@ -72,10 +73,10 @@
       }
     },
     "collectCoverageFrom": [
-      "lib/**"
+      "lib/**/*.js"
     ]
   },
   "bin": {
-    "zigbee2mqtt": "./cli.js"
+    "zigbee2mqtt": "cli.js"
   }
 }

test/assets/exampleExtension.js

@@ -1,6 +1,7 @@
 class Example {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.mqtt = mqtt;
+        this.mqtt.publish('example/extension', 'call from constructor')
     }
 
     onMQTTConnected() {

test/assets/mock-external-converter-multiple.js

@@ -11,10 +11,23 @@ const homeassistantSwitch = {
 
 const mockDevices = [{
     mock: 1,
-    model: 'external_converters_device',
+    model: 'external_converters_device_1',
     homeassistant: [homeassistantSwitch],
+    zigbeeModel: ['external_converter_device_1'],
+    vendor: 'external_1',
+    description: 'external_1',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
 }, {
-    mock: 2
+    mock: 2,
+    model: 'external_converters_device_2',
+    zigbeeModel: ['external_converter_device_2'],
+    vendor: 'external_2',
+    description: 'external_2',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
 }];
 
 module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -2,10 +2,11 @@ const mockDevice = {
     mock: true,
     zigbeeModel: ['external_converter_device'],
     vendor: 'external',
-    model: ['external_converter_device'],
-    definition: {
-        foo: 'bar'
-    }
+    model: 'external_converter_device',
+    description: 'external',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
 };
 
 module.exports = mockDevice;

test/availability.test.js

@@ -25,11 +25,11 @@ describe('Availability', () => {
 
     beforeEach(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
         jest.useFakeTimers();
         settings.set(['advanced', 'availability_timeout'], 10);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         mocksClear.forEach((m) => m.mockClear());
         await controller.start();
         await flushPromises();
@@ -172,16 +172,14 @@ describe('Availability', () => {
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(4);
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorTemperature']);
     });
 
     it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
         MQTT.publish.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
+        const device = zigbeeHerdsman.devices.E11_G13;
         const endpoint = device.getEndpoint(1);
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceJoined({device});
@@ -203,7 +201,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         device.ping.mockClear();
@@ -217,7 +215,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         device.ping.mockClear();
@@ -231,7 +229,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         device.ping.mockClear();
@@ -258,7 +256,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         device.ping.mockClear();
@@ -272,7 +270,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         device.ping.mockClear();
@@ -286,7 +284,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         device.ping.mockClear();
@@ -301,7 +299,7 @@ describe('Availability', () => {
         settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
         await controller.stop();
         await flushPromises();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         jest.advanceTimersByTime(11 * 1000);
@@ -387,7 +385,7 @@ describe('Availability', () => {
         await controller.stop();
         await flushPromises();
         MQTT.publish.mockClear();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         getExtension().state[device.ieeeAddr] = false;
         await controller.start();
         await flushPromises();

test/bind.test.js

@@ -6,6 +6,8 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const stringify = require('json-stable-stringify-without-jsonify');
+jest.mock('debounce', () => jest.fn(fn => fn));
+const debounce = require('debounce');
 
 describe('Bind', () => {
     let controller;
@@ -23,30 +25,49 @@ describe('Bind', () => {
 
     beforeEach(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
-        controller = new Controller();
+        zigbeeHerdsman.groups.group_1.members = [];
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        debounce.mockClear();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         MQTT.publish.mockClear();
     });
 
-    it('Should bind', async () => {
+    it('Should bind to device and configure reporting', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
+
+        // Setup
+        const originalDeviceOutputClusters = device.getEndpoint(1).outputClusters;
+        device.getEndpoint(1).outputClusters = [...device.getEndpoint(1).outputClusters, 768];
+        const originalTargetBinds = target.binds;
+        target.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target.getClusterAttributeValue.mockImplementationOnce((cluster, value) =>  undefined);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({transaction: "1234", from: 'remote', to: 'bulb_color'}));
         await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(target.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorCapabilities' ]);
+        expect(endpoint.bind).toHaveBeenCalledTimes(4);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(endpoint.bind).toHaveBeenCalledWith("lightingColorCtrl", target);
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1}]);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"transaction": "1234","data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl", "lightingColorCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
 
@@ -56,6 +77,10 @@ describe('Bind', () => {
           { retain: true, qos: 0 },
           expect.any(Function)
         );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).outputClusters = originalDeviceOutputClusters;
     });
 
     it('Should bind only specifief clusters', async () => {
@@ -92,19 +117,44 @@ describe('Bind', () => {
     it('Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+
+        // setup
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+        const originalRemoteBinds = device.getEndpoint(1).binds;
+        device.getEndpoint(1).binds = [];
+        const originalTargetBinds = target.binds;
+        target.binds = [
+            {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'lightingColorCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}
+        ];
+
         const endpoint = device.getEndpoint(1);
         mockClear(device);
+        delete zigbeeHerdsman.devices.bulb_color.meta.configured;
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(undefined);
         MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'bulb_color'}));
         await flushPromises();
         expect(endpoint.unbind).toHaveBeenCalledTimes(3);
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+
+        // Disable reporting
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1}]);
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/unbind',
             stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).binds = originalRemoteBinds;
     });
 
     it('Should unbind coordinator', async () => {
@@ -129,7 +179,10 @@ describe('Bind', () => {
     it('Should bind to groups', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
         mockClear(device);
         MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'group_1'}));
         await flushPromises();
@@ -137,11 +190,89 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
             stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+
+        // Should configure reproting for device added to group
+        target1Member.configureReporting.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb');
+        await flushPromises();
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+    });
+
+    it('Should unbind from group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from group with skip_disable_reporting=true', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set to false, we don't expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(0);
+        endpoint.binds = originalBinds;
+    });
+
+    it('Should unbind from group with skip_disable_reporting=false', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: false}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set, we expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 65535, "minimumReportInterval": 5, "reportableChange": 1}])
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 65535, "minimumReportInterval": 0, "reportableChange": 0}])
+        endpoint.binds = originalBinds;
     });
 
     it('Should bind to group by number', async () => {
@@ -194,6 +325,22 @@ describe('Bind', () => {
         );
     });
 
+    it('Should bind server clusters to client clusters', async () => {
+        const device = zigbeeHerdsman.devices.temperature_sensor;
+        const target = zigbeeHerdsman.devices.heating_actuator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'temperature_sensor', to: 'heating_actuator'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("msTemperatureMeasurement", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"temperature_sensor","to":"heating_actuator","clusters":["msTemperatureMeasurement"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Should bind to default endpoint returned by endpoints()', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
@@ -267,7 +414,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -298,7 +444,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -319,7 +464,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -339,7 +483,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -359,7 +502,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -413,7 +555,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -421,4 +562,37 @@ describe('Bind', () => {
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
     });
+
+    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
+        const remote = zigbeeHerdsman.devices.remote;
+        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
+        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+    });
+
+    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read.mockClear();
+        const remote = zigbeeHerdsman.devices.tradfri_remote;
+        const data = {"stepmode":0,"stepsize":43,"transtime":5};
+        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
+
+        // Should also only debounce once
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(4);
+
+        // Should only call Hue bulb, not e.g. tradfri
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+    });
 });

test/configure.test.js

@@ -26,6 +26,20 @@ describe('Configure', () => {
         expect(device.meta.configured).toBe(1);
     }
 
+    expectBulbConfigured = () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.read).toHaveBeenCalledTimes(2);
+        expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities']);
+        expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorTempPhysicalMin', 'colorTempPhysicalMax' ]);
+    }
+
+    expectBulbNotConfigured = () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.read).toHaveBeenCalledTimes(0);
+    }
+
     expectRemoteNotConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
@@ -44,65 +58,68 @@ describe('Configure', () => {
     beforeEach(async () => {
         jest.useRealTimers();
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
         this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
     });
 
-    it('Should configure on startup', async () => {
-        expectRemoteConfigured();
+    it('Should configure Router on startup', async () => {
+        expectBulbConfigured();
+    });
+
+    it('Should not configure EndDevice on startup', async () => {
+        expectRemoteNotConfigured();
     });
 
     it('Should re-configure when device rejoins', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         await flushPromises();
         mockClear(device);
         const payload = {device};
         zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
     it('Should reconfigure reporting on reportingDisabled event', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         mockClear(device);
-        expectRemoteNotConfigured();
+        expectBulbNotConfigured();
         controller.eventBus.emit('reportingDisabled', {device})
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
     it('Should not configure twice', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expectBulbNotConfigured();
     });
 
     it('Should configure on zigbee message when not configured yet', async () => {
-        const device = zigbeeHerdsman.devices.remote;
+        const device = zigbeeHerdsman.devices.bulb;
         delete device.meta.configured;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
     it('Should allow to configure via MQTT', async () => {
-        expectRemoteConfigured();
         mockClear(zigbeeHerdsman.devices.remote);
         expectRemoteNotConfigured();
         await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
@@ -137,11 +154,11 @@ describe('Configure', () => {
     });
 
     it('Fail to configure via MQTT when device has no configure', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb", transaction: 20}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "0x90fd9ffffe4b64ax", transaction: 20}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
+            stringify({"data":{"id": "0x90fd9ffffe4b64ax"},"status":"error","error": "Device 'ZNLDP12LM' cannot be configured","transaction":20}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -161,9 +178,9 @@ describe('Configure', () => {
     });
 
     it('Legacy api: Should skip reconfigure when device does not require this', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x90fd9ffffe4b64ax');
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
     it('Should not configure when interviewing', async () => {

test/controller.test.js

@@ -3,14 +3,13 @@ const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const path = require('path');
-const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
 const flushPromises = () => new Promise(setImmediate);
 const tmp = require('tmp');
 const mocksClear = [
-    zigbeeHerdsman.permitJoin, mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
     MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
     zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
 ];
@@ -19,24 +18,26 @@ const fs = require('fs');
 
 describe('Controller', () => {
     let controller;
+    let mockExit;
 
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
-        controller = new Controller();
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeDefaultState();
     });
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
         expect(logger.info).toHaveBeenCalledWith(`Currently ${Object.values(zigbeeHerdsman.devices).length - 1} devices are joined:`)
         expect(logger.info).toHaveBeenCalledWith('bulb (0x000b57fffec6a5b2): LED1545G12 - IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (Router)');
         expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
@@ -93,12 +94,15 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
-    it('Should generate network_key when set to GENERATE', async () => {
+    it('Should generate network_key and pan_id when set to GENERATE', async () => {
         settings.set(['advanced', 'network_key'], 'GENERATE');
+        settings.set(['advanced', 'pan_id'], 'GENERATE');
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.panID).toStrictEqual(expect.any(Number));
         expect(data.read().advanced.network_key.length).toStrictEqual(16);
+        expect(data.read().advanced.pan_id).toStrictEqual(expect.any(Number));
     });
 
     it('Start controller should publish cached states', async () => {
@@ -202,7 +206,7 @@ describe('Controller', () => {
         settings.set(['permit_join'], false);
         await controller.start();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
     });
 
     it('Start controller with disable_led', async () => {
@@ -225,7 +229,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(0);
+        expect(mockExit).toHaveBeenCalledWith(0, null);
     });
 
     it('Start controller and stop', async () => {
@@ -235,7 +239,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, null);
     });
 
     it('Start controller adapter disconnects', async () => {
@@ -246,7 +250,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, null);
     });
 
     it('Handle mqtt message', async () => {
@@ -599,16 +603,6 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
-    it('Load user extension', async () => {
-        const extensionPath = path.join(data.mockDir, 'extension');
-        fs.mkdirSync(extensionPath);
-        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
-    });
-
     it('Start controller with force_disable_retain', async () => {
         settings.set(['mqtt', 'force_disable_retain'], true);
         await controller.start();
@@ -629,4 +623,13 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
     });
+
+    it('Should disable legacy options on new network start', async () => {
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        expect(settings.get().advanced.legacy_api).toBeTruthy();
+        zigbeeHerdsman.start.mockReturnValueOnce('reset');
+        await controller.start();
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().advanced.legacy_api).toBeFalsy();
+    });
 });

test/externalConverters.test.js

@@ -47,12 +47,12 @@ describe('Loads external converters', () => {
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
         data.writeEmptyState();
-        settings._reRead();
+        settings.reRead();
     });
 
     it('Does not load external converters', async () => {
         settings.set(['external_converters'], []);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
@@ -62,7 +62,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter.js']);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
@@ -70,10 +70,11 @@ describe('Loads external converters', () => {
             mock: true,
             zigbeeModel: ['external_converter_device'],
             vendor: 'external',
-            model: ['external_converter_device'],
-            definition: {
-                foo: 'bar'
-            }
+            model: 'external_converter_device',
+            description: 'external',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
         });
     });
 
@@ -81,22 +82,35 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1,
-            model: 'external_converters_device',
+            model: 'external_converters_device_1',
+            zigbeeModel: ['external_converter_device_1'],
+            vendor: 'external_1',
+            description: 'external_1',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
         });
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
-            mock: 2
+            mock: 2,
+            model: 'external_converters_device_2',
+            zigbeeModel: ['external_converter_device_2'],
+            vendor: 'external_2',
+            description: 'external_2',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
         });
     });
 
     it('Loads external converters from package', async () => {
         settings.set(['external_converters'], ['mock-external-converter-module']);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
@@ -107,7 +121,7 @@ describe('Loads external converters', () => {
 
     it('Loads multiple external converters from package', async () => {
         settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);

test/externalExtension.test.js

@@ -0,0 +1,102 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const tmp = require('tmp');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
+const mkdirSyncSpy = jest.spyOn(fs, 'mkdirSync');
+const unlinkSyncSpy = jest.spyOn(fs, 'unlinkSync');
+
+describe('User extensions', () => {
+    let controller;
+    let mockExit;
+
+    beforeEach(() => {
+        zigbeeHerdsman.returnDevices.splice(0);
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        data.writeDefaultState();
+    });
+    afterEach(() => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.rmdirSync(extensionPath, {recursive: true});
+    })
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "exampleExtension.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Load user extension from api call', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "foo.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'call from constructor', { retain: false, qos: 0 }, expect.any(Function));
+        expect(mkdirSyncSpy).toHaveBeenCalledWith(extensionPath);
+    });
+
+    it('Do not load corrupted extensions', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = "definetly not a correct javascript code";
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', stringify({"data":{},"error":"Unexpected identifier","status":"error"}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('Removes user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        const extensionFilePath = path.join(extensionPath, 'exampleExtension.js')
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), extensionFilePath)
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "exampleExtension.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/remove', stringify({"name": "exampleExtension.js"}));
+        await flushPromises();
+        expect(unlinkSyncSpy).toHaveBeenCalledWith(extensionFilePath);
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/remove', stringify({"name": "non existing.js"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/extension/remove',
+            stringify({"data":{},"status":"error","error":"Extension non existing.js doesn't exists"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+});

test/frontend.test.js

@@ -74,7 +74,7 @@ describe('Frontend', () => {
         mockWS.implementation.clients = [];
         data.writeDefaultConfiguration();
         data.writeDefaultState();
-        settings._reRead();
+        settings.reRead();
         settings.set(['frontend'], {port: 8081, host: "127.0.0.1"});
         settings.set(['homeassistant'], true);
         zigbeeHerdsman.devices.bulb.linkquality = 10;
@@ -85,7 +85,7 @@ describe('Frontend', () => {
     });
 
     it('Start/stop', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
         expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
@@ -93,6 +93,7 @@ describe('Frontend', () => {
         const mockWSClient = {
             implementation: {
                 close: jest.fn(),
+                send: jest.fn(),
             },
             events: {},
         };
@@ -103,7 +104,7 @@ describe('Frontend', () => {
     });
 
     it('Websocket interaction', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
 
         // Connect
@@ -118,18 +119,18 @@ describe('Frontend', () => {
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
 
-        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[0])).toStrictEqual({topic: 'bridge/state', payload: 'online'});
-        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}});
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bridge/state', payload: 'online'}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255}}));
 
         // Message
         MQTT.publish.mockClear();
         mockWSClient.implementation.send.mockClear();
         mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
-            stringify({state: 'ON'}),
+            stringify({state: 'ON', linkquality: null}),
             { retain: false, qos: 0 },
             expect.any(Function)
         );
@@ -139,8 +140,8 @@ describe('Frontend', () => {
         await flushPromises();
 
         // Received message on socket
-        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(4);
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON'}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(1);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', linkquality: null}}));
 
         // Shouldnt set when not ready
         mockWSClient.implementation.send.mockClear();
@@ -154,11 +155,11 @@ describe('Frontend', () => {
         settings.set(['advanced'], {last_seen: 'ISO_8601'});
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
-        expect(JSON.parse(mockWSClient.implementation.send.mock.calls[12])).toStrictEqual({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}});
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z"}}));
     });
 
     it('onReques/onUpgrade', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
 
         const mockSocket = {destroy: jest.fn()};
@@ -176,7 +177,7 @@ describe('Frontend', () => {
     });
 
     it('Static server', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
 
         expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
@@ -185,7 +186,7 @@ describe('Frontend', () => {
     it('Authentification', async () => {
         const authToken = 'sample-secure-token'
         settings.set(['frontend'], {auth_token: authToken});
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
 
         const mockSocket = {destroy: jest.fn()};

test/group.test.js

@@ -9,6 +9,7 @@ zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45724');
 
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
@@ -20,11 +21,12 @@ describe('Groups', () => {
 
     beforeEach(() => {
         data.writeEmptyState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         MQTT.publish.mockClear();
+        zigbeeHerdsman.groups.gledopto_group.command.mockClear();
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     })
 
@@ -98,6 +100,17 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
 
+    it('Should resolve device friendly names', async () => {
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'friendly_name'], 'bulb_friendly_name');
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb_friendly_name', 'bulb_color']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
+            zigbeeHerdsman.devices.bulb.getEndpoint(1),
+            zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
+        ]);
+    });
+
     it('Legacy api: Add to group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -304,11 +317,12 @@ describe('Groups', () => {
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/ha_discovery_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -328,6 +342,67 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
+    it('Should publish state change for group when members state change', async () => {
+        // Created for https://github.com/Koenkk/zigbee2mqtt/issues/5725
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state of device with endpoint name', async () => {
+        const group = zigbeeHerdsman.groups.gledopto_group;
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/gledopto_group/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/GLEDOPTO_2ID", stringify({"state_cct":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/gledopto_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(group.command).toHaveBeenCalledTimes(1);
+        expect(group.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+    });
+
+    it('Should publish state of group when specific state of specific endpoint is changed', async () => {
+        const group = zigbeeHerdsman.groups.gledopto_group;
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/GLEDOPTO_2ID/set', stringify({state_cct: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/GLEDOPTO_2ID", stringify({"state_cct":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/gledopto_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(group.command).toHaveBeenCalledTimes(0);
+    });
+
     it('Should publish state change of all members when a group changes its state, filtered', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -462,6 +537,34 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
+    it('Should only update group state with changed properties', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF', color_temp: 200}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON', color_temp: 250}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 300}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode": "color_temp","color_temp":300,"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
     it('Should publish state change off even when missing current state', async () => {
         const device_1 = zigbeeHerdsman.devices.bulb_color;
         const device_2 = zigbeeHerdsman.devices.bulb;
@@ -498,14 +601,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({transaction: "123", group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/group/members/add',
-            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"transaction": "123", "status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -616,6 +719,27 @@ describe('Groups', () => {
         );
     });
 
+    it('Remove from group via MQTT keeping device reporting', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -750,4 +874,31 @@ describe('Groups', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
+
+    it('onlythis Should only include relevant properties when publishing member states', async () => {
+        const bulbColor = zigbeeHerdsman.devices.bulb_color;
+        const bulbColorTemp = zigbeeHerdsman.devices.bulb;
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(bulbColor.getEndpoint(1));
+        group.members.push(bulbColorTemp.getEndpoint(1));
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [bulbColor.ieeeAddr, bulbColorTemp.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 50}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color: {x: 0.5, y: 0.3}}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":548}), {"retain": true, qos: 0}, expect.any(Function));
+    });
 });

test/homeassistant.test.js

@@ -18,7 +18,7 @@ describe('HomeAssistant extension', () => {
         this.version = `Zigbee2MQTT ${this.version.version}`;
         jest.useRealTimers();
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
         MQTT.publish.mockClear();
         settings.set(['homeassistant'], true);
@@ -44,16 +44,72 @@ describe('HomeAssistant extension', () => {
         expect(duplicated).toHaveLength(0);
     });
 
-    it('Should discover devices', async () => {
+    it('Should discover devices and groups', async () => {
         controller = new Controller(false);
         await controller.start();
 
         let payload;
         await flushPromises();
 
+        payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":[
+                  "zigbee2mqtt_9"
+               ],
+               "name":"ha_discovery_group",
+               "sw_version":this.version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_9"],
+               "name":"ha_discovery_group",
+               "sw_version":this.version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "unique_id":"9_switch_zigbee2mqtt",
+            "value_template":"{{ value_json.state }}"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/9/switch/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -79,6 +135,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -104,6 +161,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -129,6 +187,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'battery',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.battery }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -153,7 +212,9 @@ describe('HomeAssistant extension', () => {
 
         payload = {
             'icon': 'mdi:signal',
+            'enabled_by_default': false,
             'unit_of_measurement': 'lqi',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.linkquality }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -248,7 +309,10 @@ describe('HomeAssistant extension', () => {
             ],
             "brightness":true,
             "brightness_scale":254,
-            "color_temp":true,
+            "color_mode": true,
+            "supported_color_modes": ["color_temp"],
+            "min_mireds": 250,
+            "max_mireds": 454,
             "command_topic":"zigbee2mqtt/bulb/set",
             "device":{
                 "identifiers":[
@@ -268,13 +332,11 @@ describe('HomeAssistant extension', () => {
                 "finish_effect",
                 "stop_effect"
             ],
-            "hs":false,
             "json_attributes_topic":"zigbee2mqtt/bulb",
             "name":"bulb",
             "schema":"json",
             "state_topic":"zigbee2mqtt/bulb",
             "unique_id":"0x000b57fffec6a5b2_light_zigbee2mqtt",
-            "xy":false
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -303,6 +365,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -328,6 +391,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -353,6 +417,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -409,6 +474,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -436,6 +502,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -529,6 +596,24 @@ describe('HomeAssistant extension', () => {
         expect(topics).not.toContain('homeassistant/sensor/0x0017880104e45522/temperature/config')
     });
 
+    it('Shouldnt discover sensor when set to null', async () => {
+        logger.error.mockClear();
+        settings.set(['devices', '0x0017880104e45522'], {
+            homeassistant: {humidity: null},
+            friendly_name: 'weather_sensor',
+            retain: false,
+        })
+
+        controller = new Controller(false);
+        await controller.start();
+
+        await flushPromises();
+
+        const topics = MQTT.publish.mock.calls.map((c) => c[0]);
+        expect(topics).not.toContain('homeassistant/sensor/0x0017880104e45522/humidity/config')
+        expect(topics).toContain('homeassistant/sensor/0x0017880104e45522/temperature/config')
+    });
+
     it('Should discover devices with fan', async () => {
         controller = new Controller(false);
         await controller.start();
@@ -544,12 +629,10 @@ describe('HomeAssistant extension', () => {
             "speed_command_topic":"zigbee2mqtt/fan/set/fan_mode",
             "speed_value_template":"{{ value_json.fan_mode }}",
             "speeds":[
-               "off",
                "low",
                "medium",
                "high",
                "on",
-               "auto",
                "smart"
             ],
             "json_attributes_topic":"zigbee2mqtt/fan",
@@ -583,8 +666,6 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
 
         payload = {
-            "action_template":"{% set values = {'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'} %}{{ values[value_json.running_state] }}",
-            "action_topic":"zigbee2mqtt/TS0601_thermostat",
             'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
             "away_mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/away_mode",
             "away_mode_state_template":"{{ value_json.away_mode }}",
@@ -649,7 +730,7 @@ describe('HomeAssistant extension', () => {
             position_topic: 'zigbee2mqtt/smart vent',
             set_position_topic: 'zigbee2mqtt/smart vent/set',
             set_position_template: '{ "position": {{ position }} }',
-            value_template: '{{ value_json.position }}',
+            position_template: '{{ value_json.position }}',
             json_attributes_topic: 'zigbee2mqtt/smart vent',
             name: 'smart vent',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
@@ -683,6 +764,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -720,6 +802,83 @@ describe('HomeAssistant extension', () => {
         expect(logger.warn).toHaveBeenCalledWith("In order for HomeAssistant integration to work properly set `cache_state: true");
     });
 
+    it('Should set missing values to null', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/6987
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/weather_sensor',
+            stringify({"battery":null,"humidity":null,"linkquality":null,"pressure":null,"temperature":-0.85,"voltage":null}),
+            { retain: false, qos: 1 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should copy hue/saturtion to h/s if present', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const data = {currentHue: 0, currentSaturation: 254}
+        const payload = {data, cluster: 'lightingColorCtrl', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"hue": 0, "saturation": 100, "h": 0, "s": 100}, "color_mode": "hs", "linkquality": null, "state": null}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should not copy hue/saturtion if properties are missing', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const data = {currentX: 29991, currentY: 26872};
+        const payload = {data, cluster: 'lightingColorCtrl', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({"color": {"x": 0.4576,"y": 0.41}, "color_mode": "xy", "linkquality": null,"state": null}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should not copy hue/saturtion if color is missing', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({"linkquality": null,"state": "ON"}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
     it('Shouldt discover when already discovered', async () => {
         controller = new Controller(false);
         await controller.start();
@@ -748,6 +907,7 @@ describe('HomeAssistant extension', () => {
         const payloadHA = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -772,7 +932,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt discover when device leaves', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         controller.extensions.find((e) => e.constructor.name === 'HomeAssistant').discovered = {};
@@ -786,7 +946,7 @@ describe('HomeAssistant extension', () => {
     it('Should send all status when home assistant comes online (default topic)', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         expect(MQTT.subscribe).toHaveBeenCalledWith('homeassistant/status');
         await flushPromises();
@@ -797,13 +957,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update":{"state":"idle"},"update_available":false}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"brightness":255,"update_available":false,"update":{"state":"idle"}}),
+            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -812,7 +972,7 @@ describe('HomeAssistant extension', () => {
     it('Should send all status when home assistant comes online', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(MQTT.subscribe).toHaveBeenCalledWith('hass/status');
@@ -823,13 +983,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false,"update":{"state":"idle"}}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"brightness":255,"update_available":false,"update":{"state":"idle"}}),
+            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -838,7 +998,7 @@ describe('HomeAssistant extension', () => {
     it('Shouldnt send all status when home assistant comes offline', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -852,7 +1012,7 @@ describe('HomeAssistant extension', () => {
     it('Shouldnt send all status when home assistant comes online with different topic', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -874,6 +1034,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -960,6 +1121,7 @@ describe('HomeAssistant extension', () => {
         const payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
@@ -1030,6 +1192,7 @@ describe('HomeAssistant extension', () => {
         const payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
@@ -1061,6 +1224,7 @@ describe('HomeAssistant extension', () => {
             "payload_on":true,
             "payload_off":false,
             "value_template":"{{ value_json.update_available}}",
+            "enabled_by_default": false,
             "state_topic":"zigbee2mqtt/bulb",
             "json_attributes_topic":"zigbee2mqtt/bulb",
             "name":"bulb update available",
@@ -1091,7 +1255,7 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
 
         const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(4);
+        expect(discovered.length).toBe(5);
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/click/config');
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
 
@@ -1166,21 +1330,21 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "single", click: "single"}),
+            stringify({action: "single", click: "single", battery: null, linkquality: null, voltage: null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: ""}),
+            stringify({action: "", battery: null, linkquality: null, voltage: null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({click: ""}),
+            stringify({click: "", action: null, battery: null, linkquality: null, voltage: null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -1238,6 +1402,28 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
     });
 
+    it('Should not discover device_automation when disabled', async () => {
+        settings.set(['device_options'], {
+            homeassistant: {device_automation: null},
+        })
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const payload1 = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
+        await flushPromises();
+
+        expect(MQTT.publish).not.toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/action_single/config',
+            expect.any(String),
+            expect.any(Object),
+            expect.any(Function),
+        );
+    });
+
     it('Should not discover sensor_click when legacy: false is set', async () => {
         settings.set(['devices', '0x0017880104e45520'], {
             legacy: false,
@@ -1249,7 +1435,7 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
 
         const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(3);
+        expect(discovered.length).toBe(4);
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/battery/config');
         expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/linkquality/config');
@@ -1262,7 +1448,7 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
 
         const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
-        expect(discovered.length).toBe(2);
+        expect(discovered.length).toBe(3);
         expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/click/config');
         expect(discovered).not.toContain('homeassistant/sensor/0x0017880104e45520/action/config');
 
@@ -1307,7 +1493,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            stringify({action: "single"}),
+            stringify({action: "single", "battery":null,"linkquality":null,"voltage":null}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -1323,9 +1509,9 @@ describe('HomeAssistant extension', () => {
 
         await MQTT.events.message('zigbee2mqtt/U202DST600ZB/l2/set', stringify({state: 'ON', brightness: 20}));
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', stringify({state_l2:"ON", brightness_l2:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', stringify({state_l2:"ON", brightness_l2:20, linkquality: null, state_l1: null}), {"qos": 0, "retain": false}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l2', stringify({state:"ON", brightness:20}), {"qos": 0, "retain": false}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', stringify({}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', stringify({state: null}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Shouldnt crash in onPublishEntityState on group publish', async () => {
@@ -1353,10 +1539,10 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', battery: null, linkquality: null, voltage: null});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: ''});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', battery: null, linkquality: null, voltage: null});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/action_single/config');
         expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/action');
@@ -1364,14 +1550,11 @@ describe('HomeAssistant extension', () => {
 
     it('Load Home Assistant mapping from external converters', async () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
-        const beforeCount = Object.entries((new HomeAssistant(null, null, null, null, {on: () => {}}))._getMapping()).length;
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         const ha = controller.extensions.find((e) => e.constructor.name === 'HomeAssistant');
         await controller.start();
         await flushPromises();
-        const afterCount = Object.entries(ha._getMapping()).length;
-        expect(beforeCount + 1).toStrictEqual(afterCount);
 
         const homeassistantSwitch = {
             type: 'switch',
@@ -1383,11 +1566,11 @@ describe('HomeAssistant extension', () => {
                 command_topic: true,
             },
         };
-        expect(ha._getMapping()['external_converters_device']).toEqual([homeassistantSwitch]);
+        expect(ha._getMapping()['external_converters_device_1']).toEqual([homeassistantSwitch]);
     });
 
     it('Should clear outdated configs', async () => {
-        controller = new Controller(false);
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
 
@@ -1452,4 +1635,78 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', null, {qos: 0, retain: true}, expect.any(Function));
     });
+
+    it('Should not have Home Assistant legacy entity attributes when disabled', async () => {
+        settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+        controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'weather_sensor_temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'weather_sensor',
+                'sw_version': this.version,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should rediscover group when device is added to it', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'ha_discovery_group', device: 'wall_switch_double/left'}));
+        await flushPromises();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":[
+                  "zigbee2mqtt_9"
+               ],
+               "name":"ha_discovery_group",
+               "sw_version":this.version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -15,13 +15,13 @@ describe('Bridge legacy', () => {
 
     beforeAll(async () => {
         this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
     })
 
     beforeEach(() => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeDefaultState();
         logger.info.mockClear();
         logger.warn.mockClear();
@@ -123,12 +123,12 @@ describe('Bridge legacy', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'true');
         await flushPromises();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
         zigbeeHerdsman.permitJoin.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'false');
         await flushPromises();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
     });
 
     it('Should allow to reset', async () => {
@@ -188,7 +188,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","optimistic":true,"retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","optimistic":true,"retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","optimistic":true,"retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","optimistic":true,"retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","optimistic":true,"retain":false},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","optimistic":true,"retain":false}],"type":"groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","retain":false},{"ID":9,"devices":["bulb_color_2","bulb_2","wall_switch_double/right"],"friendly_name":"ha_discovery_group"},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group"},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {
@@ -221,10 +221,10 @@ describe('Bridge legacy', () => {
 
     it('Should allow rename groups', async () => {
         MQTT.publish.mockClear();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", retain: false});
         MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
         await flushPromises();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", retain: false});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
             stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
@@ -251,7 +251,7 @@ describe('Bridge legacy', () => {
     });
 
     it('Shouldnt rename when no device has been joined', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
@@ -270,7 +270,7 @@ describe('Bridge legacy', () => {
             {qos: 0, retain: false},
             expect.any(Function)
         );
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
     });
@@ -279,7 +279,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -294,7 +294,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -309,7 +309,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
         await flushPromises();
-        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
     });

test/legacy/report.test.js

@@ -1,6 +1,6 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
@@ -8,16 +8,15 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45559');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45524');
 zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
-const MQTT = require('./stub/mqtt');
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
+const MQTT = require('../stub/mqtt');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-jest.mock('debounce', () => jest.fn(fn => fn));
-const debounce = require('debounce');
 
-const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Report', () => {
     let controller;
@@ -72,7 +71,7 @@ describe('Report', () => {
 
     beforeEach(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
         settings.set(['advanced', 'report'], true);
         for (const device of Object.values(zigbeeHerdsman.devices)) {
@@ -80,7 +79,7 @@ describe('Report', () => {
             delete device.meta.reporting;
         }
 
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
@@ -98,7 +97,7 @@ describe('Report', () => {
         mockClear(device);
         delete device.meta.report;
         settings.set(['advanced', 'report'], false);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(device.meta.reporting).toBe(undefined);
@@ -111,7 +110,7 @@ describe('Report', () => {
         const endpoint = device.getEndpoint(1);
         settings.set(['advanced', 'report'], false);
         mockClear(device);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expectOnOffBrightnessColorReportDisabled(endpoint, true);
@@ -240,36 +239,6 @@ describe('Report', () => {
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
     });
 
-    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
-        const remote = zigbeeHerdsman.devices.remote;
-        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
-        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(debounce).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
-    });
-
-    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote and should', async () => {
-        const remote = zigbeeHerdsman.devices.tradfri_remote;
-        const data = {"stepmode":0,"stepsize":43,"transtime":5};
-        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(debounce).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
-
-        // Should also only debounce once
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(debounce).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
-
-        // Should only call Hue bulb, not e.g. tradfri
-        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
-    });
-
     it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
         const device = zigbeeHerdsman.devices.ZNLDP12LM;
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);

test/logger.test.js

@@ -6,6 +6,7 @@ const path = require('path');
 const data = require('./stub/data');
 let stdOutWriteOriginal;
 const rimraf = require('rimraf');
+const Transport = require('winston-transport');
 
 describe('Logger', () => {
     beforeEach(async () => {
@@ -13,7 +14,7 @@ describe('Logger', () => {
         jest.resetModules();
         settings = require('../lib/util/settings');
         settings.set(['advanced', 'log_directory'], dir.name + '/%TIMESTAMP%');
-        settings._reRead();
+        settings.reRead();
         stdOutWriteOriginal = console._stdout.write;
         console._stdout.write = () => {};
     });
@@ -66,6 +67,26 @@ describe('Logger', () => {
         expect(logger.getLevel()).toBe('debug');
     });
 
+    it('Add transport', () => {
+        class DummyTransport extends Transport {
+            log(info, callback) {
+            }
+        }
+
+        const logger = require('../lib/util/logger.js');
+        expect(logger.transports.length).toBe(2);
+        logger.addTransport(new DummyTransport());
+        expect(logger.transports.length).toBe(3);
+    });
+
+    it('Set and get log level warn <-> warning', () => {
+        const logger = require('../lib/util/logger.js');
+        logger.logOutput();
+        logger.setLevel('warn');
+        expect(logger.transports[0].level).toBe('warning');
+        expect(logger.getLevel()).toBe('warn');
+    });
+
     it('Logger should be console and file by default', () => {
         const logger = require('../lib/util/logger.js');
         logger.logOutput();
@@ -117,4 +138,14 @@ describe('Logger', () => {
         expect(pipes[1].tailable).toBeFalsy();
         expect(pipes[1].maxsize).toBeNull();
     });
+
+    it('Should allow to symlink logs to current directory', () => {
+        settings.set(['advanced', 'log_symlink_current'], true);
+        let logger = require('../lib/util/logger.js');
+        logger.logOutput();
+        expect(fs.readdirSync(dir.name).includes('current')).toBeTruthy()
+
+        jest.resetModules();
+        logger = require('../lib/util/logger.js');
+    });
 });

test/networkMap.test.js

@@ -17,21 +17,21 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 Date.now = jest.fn()
 Date.now.mockReturnValue(10000);
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const setTimeoutNative = setTimeout;
 
 describe('Networkmap', () => {
     let controller;
 
     beforeAll(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
         settings.set(['external_converters'], ['mock-external-converter.js']);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
@@ -43,6 +43,11 @@ describe('Networkmap', () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         device.lastSeen = 1000;
         external_converter_device.lastSeen = 1000;
+        global.setTimeout = (r) => r();
+    });
+
+    afterEach(async () => {
+        global.setTimeout = setTimeoutNative;
     });
 
     function mock() {
@@ -92,7 +97,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"model":["external_converter_device"],"supports":"","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -132,18 +137,18 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|0 seconds ago}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
-              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
-              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0x0000)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (0x9db1)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
+              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 0x198c)"]
+              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (0x9dcf)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (0x198a)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (0x1988)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (0x198c)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
-              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (1114)|external undefined (external_converter_device)|9 seconds ago}"];
+              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (0x045a)|external external (external_converter_device)|9 seconds ago}"];
               "0x0017880104e45511" -> "0x00124b00120144ae" [penwidth=1, weight=0, color="#994444", label="92"]
             }`;
 
@@ -175,9 +180,9 @@ describe('Networkmap', () => {
         card 0x0017880104e45511 [
         0x0017880104e45511
         ---
-        0x0017880104e45511 (1114)
+        0x0017880104e45511 (0x045a)
         ---
-        external undefined (external_converter_device)
+        external external (external_converter_device)
         ---
         9 seconds ago
         ]
@@ -185,7 +190,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45525 [
         0x0017880104e45525
         ---
-        0x0017880104e45525 (6536) failed: lqi,routingTable
+        0x0017880104e45525 (0x1988) failed: lqi,routingTable
         ---
         Boef notSupportedModelID
         ---
@@ -195,7 +200,7 @@ describe('Networkmap', () => {
         card 0x000b57fffec6a5b2 [
         bulb
         ---
-        0x000b57fffec6a5b2 (40369)
+        0x000b57fffec6a5b2 (0x9db1)
         ---
         IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
         ---
@@ -205,7 +210,7 @@ describe('Networkmap', () => {
         card 0x000b57fffec6a5b3 [
         bulb_color
         ---
-        0x000b57fffec6a5b3 (40399)
+        0x000b57fffec6a5b3 (0x9dcf)
         ---
         Philips Hue Go (7146060PH)
         ---
@@ -215,7 +220,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45521 [
         button_double_key
         ---
-        0x0017880104e45521 (6538)
+        0x0017880104e45521 (0x198a)
         ---
         Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)
         ---
@@ -225,7 +230,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45559 [
         cc2530_router
         ---
-        0x0017880104e45559 (6540)
+        0x0017880104e45559 (0x198c)
         ---
         Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
         ---
@@ -235,7 +240,7 @@ describe('Networkmap', () => {
         card 0x00124b00120144ae [
         Coordinator
         ---
-        0x00124b00120144ae (0)
+        0x00124b00120144ae (0x0000)
         ---
         0 seconds ago
         ]
@@ -266,7 +271,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_xy), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"model":["external_converter_device"],"supports":"","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
         expect(actual).toStrictEqual(expected);
     });

test/onEvent.test.js

@@ -23,9 +23,9 @@ describe('On event', () => {
 
     beforeEach(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
         zigbeeHerdsmanConverters.onEvent.mockClear();

test/otaUpdate.test.js

@@ -18,10 +18,10 @@ describe('OTA update', () => {
 
     beforeEach(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         settings.set(['advanced', 'ikea_ota_use_test_url'], true);
         data.writeEmptyState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -51,7 +51,7 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');
@@ -166,11 +166,11 @@ describe('OTA update', () => {
     });
 
     it('Should not check for OTA when device does not support it', async () => {
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "ZNLDP12LM");
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "dimmer_wall_switch");
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
-            stringify({"data":{"id": "ZNLDP12LM"},"status":"error","error": `Device 'ZNLDP12LM' does not support OTA updates`}),
+            stringify({"data":{"id": "dimmer_wall_switch"},"status":"error","error": `Device 'dimmer_wall_switch' does not support OTA updates`}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -250,8 +250,22 @@ describe('OTA update', () => {
         );
     });
 
+    it('Should not check for update when device requests it and disable_automatic_update_check is set to true', async () => {
+        settings.set(['ota', 'disable_automatic_update_check'], true);
+        const device = zigbeeHerdsman.devices.bulb;
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+    });
+
     it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
-        const device = zigbeeHerdsman.devices.QBKG04LM;
+        const device = zigbeeHerdsman.devices.HGZB04D;
         const data = {imageType: 12382};
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
@@ -261,7 +275,7 @@ describe('OTA update', () => {
     });
 
     it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
-        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const device = zigbeeHerdsman.devices.SV01;
         const data = {imageType: 12382};
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
         logger.error.mockClear();
@@ -296,7 +310,7 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
         expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
@@ -360,9 +374,9 @@ describe('OTA update', () => {
     });
 
     it('Legacy api: Should not check for OTA when device does not support it', async () => {
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'dimmer_wall_switch');
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+        expect(logger.error).toHaveBeenCalledWith(`Device 'dimmer_wall_switch' does not support OTA updates`);
     });
 
     it('Legacy api: Shouldnt crash when read modelID after OTA update fails', async () => {

test/publish.test.js

@@ -28,7 +28,7 @@ describe('Publish', () => {
 
     beforeAll(async () => {
         data.writeEmptyState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
     });
@@ -38,7 +38,7 @@ describe('Publish', () => {
         await flushPromises();
         data.writeDefaultConfiguration();
         controller.state.state = {};
-        settings._reRead();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         Object.values(zigbeeHerdsman.devices).forEach((d) => {
             d.endpoints.forEach((e) => {
@@ -139,7 +139,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 222, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 222, color: {x: 0.360786471097048, y: 0.363543544699483}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 222});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -152,7 +152,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 500, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 500, color: {x: 0.526676280311873, y: 0.41329727450763}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 500});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -176,10 +176,10 @@ describe('Publish', () => {
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_right: 'OFF'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state_right: 'OFF'});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should publish messages to zigbee devices to non default-ep with state_[EP]', async () => {
@@ -219,7 +219,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 62, color: {x: 100, y: 50}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -231,13 +231,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON'}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy and brightness', async () => {
@@ -248,13 +247,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, state: 'ON', color_temp: 62, brightness: 20});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', brightness: 20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state on', async () => {
@@ -265,13 +263,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, state: 'ON', color_temp: 62, brightness: 20});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color: {x: 100, y: 50}, state: 'ON', brightness: 20, color_mode: 'xy'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state off', async () => {
@@ -282,40 +280,40 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'OFF'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with color rgb object', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y:  0.6585}, color_mode: 'xy'});
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with color rgb string', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y:  0.6585}, color_mode: 'xy'});
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with brightness', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON', brightness: '50'}));
@@ -343,10 +341,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({brightness_percent: 50}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
-        expect(group.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 127, transtime: 0}, {});
+        expect(group.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 128, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 127});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 128});
     });
 
     it('Should publish messages to groups when converter is not in the default list but device in it supports it', async () => {
@@ -354,7 +352,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/thermostat_group/set', stringify({child_lock: 'LOCK'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
-        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 7, length_hi: 0, length_lo: 1, status: 0, transid: 0}, {disableDefaultResponse: true});
+        expect(group.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 7, length_hi: 0, length_lo: 1, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
     });
 
     it('Should publish messages to groups with on and brightness', async () => {
@@ -387,7 +385,7 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 24248, colory: 18350, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.37, y: 0.28}, color_temp: 249});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.37, y: 0.28}, color_mode: 'xy'});
     });
 
     it('Should publish messages to groups color temperature', async () => {
@@ -398,15 +396,15 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 100, color_mode: 'color_temp'});
     });
 
     it('Should create and publish to group which is in configuration.yaml but not in zigbee-herdsman', async () => {
         delete zigbeeHerdsman.groups.group_2;
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(7);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(9);
         await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'ON'}));
         await flushPromises();
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(8);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(10);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
     });
@@ -422,6 +420,25 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Shouldnt publish new brightness state when filtered_optimistic is used', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        settings.set(['devices', device.ieeeAddr, 'filtered_optimistic'], ["brightness"]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {"level": 200, "transtime": 0}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+    });
+
+    it('Shouldnt publish new state when optimistic = false for group', async () => {
+        settings.set(['groups', '2', 'optimistic'], false);
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
     it('Should handle non-valid topics', async () => {
         await MQTT.events.message('zigbee2mqtt1/bulb_color/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -551,10 +568,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y: 0.6584}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y: 0.6585}, color_mode: "xy"});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -678,7 +695,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColor", {"colorx": 45940, "colory": 19595, "transtime": 0}, {}]);
         expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read.mock.calls[0]).toEqual(["genOnOff", ["onOff"]]);
-        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["currentX", "currentY"]]);
+        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["colorMode", "currentX", "currentY", "colorTemperature"]]);
     });
 
     it('Should also use on/off cluster when controlling group with switch', async () => {
@@ -910,10 +927,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891.475, "saturation": 199.21474, "transtime": 0,}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44877, "saturation": 200, "transtime": 0,}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}, "color_mode": "hs"});
     });
 
     it('ZNCLDJ11LM open', async () => {
@@ -1064,6 +1081,7 @@ describe('Publish', () => {
     it('Home Assistant: should not set state when color temperature is also set and device is already on', async () => {
         settings.set(['homeassistant'], true);
         const device = zigbeeHerdsman.devices.bulb_color;
+        controller.state.remove(device.ieeeAddr);
         controller.state.set(device.ieeeAddr, {state: 'ON'})
         const endpoint = device.getEndpoint(1);
         const payload = {state: 'ON', color_temp: 100};
@@ -1073,12 +1091,13 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', color_temp: 100, color_mode: 'color_temp'});
     });
 
     it('Home Assistant: should set state when color temperature is also set and device is off', async () => {
         settings.set(['homeassistant'], true);
         const device = zigbeeHerdsman.devices.bulb_color;
+        controller.state.remove(device.ieeeAddr);
         controller.state.set(device.ieeeAddr, {state: 'OFF'})
         const endpoint = device.getEndpoint(1);
         const payload = {state: 'ON', color_temp: 100};
@@ -1087,16 +1106,19 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON', color_temp: 100, color_mode: 'color_temp'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Home Assistant: should not set state when color is also set', async () => {
         settings.set(['homeassistant'], true);
         const device = zigbeeHerdsman.devices.bulb_color;
+        controller.state.remove(device.ieeeAddr);
         controller.state.set(device.ieeeAddr, {state: 'ON'})
         const endpoint = device.getEndpoint(1);
         const payload = {state: 'ON', color: {x: 0.41, y: 0.25}};
@@ -1106,7 +1128,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "moveToColor", {colorx: 26869, colory: 16384, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.41, y: 0.25}, color_temp: 150, state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.41, y: 0.25}, state: 'ON', color_mode: 'xy'});
     });
 
     it('Should publish correct state on toggle command to zigbee bulb', async () => {
@@ -1247,7 +1269,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 3, length_hi: 0, length_lo: 1, status: 0, transid: 1}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuya", "setData", {data: [1], datatype: 1, dp: 3, length_hi: 0, length_lo: 1, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});
@@ -1282,6 +1304,25 @@ describe('Publish', () => {
         expect(logger.error).toHaveBeenCalledWith("No converter available for 'get' 'brightness_move' (20)");
     });
 
+    it('Should restore brightness when its turned on with transition, Z2M is restarted and turned on again', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/7106
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.command.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON', brightness: 20, transition: 0.0}));
+        await flushPromises();
+
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({"state": "ON", "transition": 1.0}));
+        await flushPromises();
+
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command).toHaveBeenCalledWith('genLevelCtrl', 'moveToLevelWithOnOff', {level: 20, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith('genLevelCtrl', 'moveToLevelWithOnOff', {level: 20, transtime: 10}, {});
+    });
+
     it('Should restore brightness when its turned off without transition and is turned on with', async () => {
         // https://github.com/Koenkk/zigbee-herdsman-converters/issues/1097
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -1359,30 +1400,65 @@ describe('Publish', () => {
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith('genScenes', 'recall', { groupid: 15071, sceneid: 1 }, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledTimes(7);
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
             'zigbee2mqtt/group_tradfri_remote',
-            stringify({"brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"brightness":50,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(2,
             'zigbee2mqtt/bulb_color_2',
-            stringify({"brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"color_mode": "color_temp", "brightness":50,"color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(3,
             'zigbee2mqtt/group_tradfri_remote',
-            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"brightness":100,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(4,
             'zigbee2mqtt/bulb_2',
-            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(5,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":50,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(6,
             'zigbee2mqtt/group_with_tradfri',
-            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(7,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should sync colors', async () => {
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color_temp: 100}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {x: 0.1, y: 0.5}}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color_temp: 300}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color_mode":"color_temp","color_temp":100}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"x":0.1,"y":0.5},"color_mode":"xy","color_temp":79}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(3,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"x":0.4152,"y":0.3954},"color_mode":"color_temp","color_temp":300}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });

test/receive.test.js

@@ -15,9 +15,9 @@ describe('Receive', () => {
     beforeEach(async () => {
         jest.useRealTimers();
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
         delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
@@ -64,7 +64,7 @@ describe('Receive', () => {
         const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
@@ -314,7 +314,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'temperature': 19});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 

test/settings.test.js

@@ -22,7 +22,7 @@ describe('Settings', () => {
     const write = (file, json, reread=true) => {
         fs.writeFileSync(file, yaml.safeDump(json))
         if (reread) {
-            settings._reRead();
+            settings.reRead();
         }
     };
     const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
@@ -352,7 +352,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -378,7 +377,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -420,7 +418,6 @@ describe('Settings', () => {
 
         const group = settings.getGroup('1');
         const expectedGroup = {
-            optimistic: true,
             ID: 1,
             friendlyName: '123',
             friendly_name: '123',
@@ -616,12 +613,24 @@ describe('Settings', () => {
             advanced: {network_key: 'NOT_GENERATE'},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
+    it('Should not allow any string values for pan_id', () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            advanced: {pan_id: 'NOT_GENERATE'},
+        });
+
+        settings.reRead();
+
+        const error = `advanced.pan_id: should be number or 'GENERATE' (is 'NOT_GENERATE')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
     it('Should allow retention configuration with MQTT v5', () => {
         write(configurationFile, {
             ...minimalConfig,
@@ -629,7 +638,7 @@ describe('Settings', () => {
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
         });
 
-        settings._reRead();
+        settings.reRead();
         expect(settings.validate()).toEqual([]);
     });
 
@@ -640,7 +649,7 @@ describe('Settings', () => {
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = 'MQTT retention requires protocol version 5';
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
@@ -653,7 +662,7 @@ describe('Settings', () => {
             advanced: {availability_blocklist: ['0x0017880104e45519', 'non_existing']},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `Non-existing entity 'non_existing' specified in 'availability_blocklist'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
@@ -665,7 +674,7 @@ describe('Settings', () => {
             advanced: null,
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `advanced should be object`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
@@ -711,19 +720,55 @@ describe('Settings', () => {
             groups: {'1': {friendly_name: 'myname', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `Duplicate friendly_name 'myname' found`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
+    it('Configuration friendly name cannot be empty', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: '', retain: false}},
+        });
+
+        settings.reRead();
+
+        const error = `friendly_name must be at least 1 char long`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Configuration friendly name cannot end with /', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: 'blaa/', retain: false}},
+        });
+
+        settings.reRead();
+
+        const error = `friendly_name is not allowed to end or start with /`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Configuration friendly name cannot contain null char', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: 'blaa/blaa' + String.fromCharCode(0), retain: false}},
+        });
+
+        settings.reRead();
+
+        const error = `friendly_name is not allowed to contain null char`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
     it('Configuration shouldnt be valid when friendly_name ends with /DIGIT', async () => {
         write(configurationFile, {
             ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname/123', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `Friendly name cannot end with a "/DIGIT" ('myname/123')`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
@@ -735,7 +780,7 @@ describe('Settings', () => {
             devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
@@ -747,7 +792,7 @@ describe('Settings', () => {
             devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
         const error = `Following friendly_name are not allowed: '${utils.getEndpointNames()}'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
@@ -761,7 +806,7 @@ describe('Settings', () => {
             },
         });
 
-        settings._reRead();
+        settings.reRead();
 
         expect(() => {
             settings.changeFriendlyName('myname1', 'myname');
@@ -776,7 +821,7 @@ describe('Settings', () => {
             },
         });
 
-        settings._reRead();
+        settings.reRead();
 
         expect(() => {
             settings.removeDevice('myname33');

test/stub/data.js

@@ -165,6 +165,15 @@ function writeDefaultConfiguration() {
             '0x0017882194e45543': {
                 friendly_name: 'QS-Zigbee-D02-TRIAC-2C-LN',
             },
+            '0x0017880104e45724': {
+                friendly_name: 'GLEDOPTO_2ID',
+            },
+            '0x0017880104e45561': {
+                friendly_name: 'temperature_sensor',
+            },
+            '0x0017880104e45562': {
+                friendly_name: 'heating_actuator',
+            }
         },
         groups: {
             '1': {
@@ -194,7 +203,15 @@ function writeDefaultConfiguration() {
                 friendly_name: 'switch_group',
                 retain: false,
                 devices: ['power_plug'],
-            }
+            },
+            '21': {
+                friendly_name: 'gledopto_group',
+                devices: ['GLEDOPTO_2ID/cct'],
+            },
+            '9': {
+                friendly_name: 'ha_discovery_group',
+                devices: ['bulb_color_2', 'bulb_2', 'wall_switch_double/right']
+            },
         },
         external_converters: [],
     };

test/stub/logger.js

@@ -19,6 +19,7 @@ const mock = {
     cleanup: jest.fn(),
     logOutput: jest.fn(),
     add: (transport) => transports.push(transport),
+    addTransport: (transport) => transports.push(transport),
     setLevel: (newLevel) => {level = newLevel},
     getLevel: () => level,
     setTransportsEnabled: (value) => {transportsEnabled = value},

test/stub/zigbeeHerdsman.js

@@ -26,10 +26,12 @@ const clusters = {
     'genLevelCtrl': 8,
     'lightingColorCtrl': 768,
     'closuresWindowCovering': 258,
+    'hvacThermostat': 513,
+    'msTemperatureMeasurement': 1026,
 }
 
 class Endpoint {
-    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}, configuredReportings=[]) {
+    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}, configuredReportings=[], profileID=null, deviceID=null) {
         this.deviceIeeeAddress = deviceIeeeAddress;
         this.clusterValues = clusterValues;
         this.ID = ID;
@@ -45,6 +47,8 @@ class Endpoint {
         this.configureReporting = jest.fn();
         this.meta = {};
         this.binds = binds;
+        this.profileID = profileID;
+        this.deviceID = deviceID;
         this.configuredReportings = configuredReportings;
         this.getInputClusters = () => inputClusters.map((c) => {
             return {ID: c, name: getKeyByValue(clusters, c)};
@@ -85,9 +89,11 @@ class Endpoint {
             Object.values(groups).forEach((g) => this.removeFromGroup(g))
         }
 
-        this.getClusterAttributeValue = (cluster, value) => {
+        this.getClusterAttributeValue = jest.fn();
+        this.getClusterAttributeValue.mockImplementation((cluster, value) => {
+            if (!(cluster in this.clusterValues)) return undefined;
             return this.clusterValues[cluster][value];
-        }
+        });
     }
 }
 
@@ -124,29 +130,45 @@ const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new
 const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020", false, 'Philips', '2019.09', '5.127.1.26581');
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
+const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
+const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
+const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
+
+const groups = {
+    'group_1': new Group(1, []),
+    'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
+    'group/with/slashes': new Group(99, []),
+    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
+    'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
+    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0]]),
+    'gledopto_group': new Group(21, [GLEDOPTO_2ID.endpoints[3]]),
+    'default_bind_group': new Group(901, []),
+    'ha_discovery_group': new Group(9, [bulb_color_2.endpoints[0], bulb_2.endpoints[0], QBKG03LM.endpoints[1]]),
+}
 
 const devices = {
-    'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
+    'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [], '0x00124b00120144ae')], false),
     'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}}, [{cluster: {name: 'genOnOff'}, attribute: {name: 'onOff'}, minimumReportInterval: 1, maximumReportInterval: 10, reportableChange: 20}])], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
-    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: new Group(1, []), cluster: {ID: 6, name: 'genOnOff'}}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
+    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: bulb_color.endpoints[0], cluster: {ID: 6, name: 'genOnOff'}}, {target: bulb_color.endpoints[0], cluster: {ID: 768, name: 'lightingColorCtrl'}}, {target: groups.group_1, cluster: {ID: 6, name: 'genOnOff'}}, {target: groups.group_1, cluster: {ID: 6, name: 'genLevelCtrl'}}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
     'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID", false, "notSupportedMfg"),
     'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
     'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
     'notInSettings': new Device('EndDevice', '0x0017880104e45519', 6537, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
-    'WXKG11LM': new Device('EndDevice', '0x0017880104e45520', 6537,4151, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
+    'WXKG11LM': new Device('EndDevice', '0x0017880104e45520', 6537,4151, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45520', [], {}, [{cluster: {name: 'genOnOff'}, attribute: {name: undefined, ID: 1337}, minimumReportInterval: 1, maximumReportInterval: 10, reportableChange: 20}])], true, "Battery", "lumi.sensor_switch.aq2"),
     'WXKG02LM_rev1': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
     'WSDCGQ11LM': new Device('EndDevice', '0x0017880104e45522', 6539,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.weather"),
     'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
-    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug"),
+    'ZNCZ02LM': ZNCZ02LM,
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
     'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
-    'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
+    'QBKG03LM':QBKG03LM,
     'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
-    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45545')], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
+    'GLEDOPTO_2ID': GLEDOPTO_2ID,
+    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [25], '0x0017880104e45545')], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
     'ZNCLDJ11LM': new Device('Router', '0x0017880104e45547', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'lumi.curtain'),
     'HAMPTON99432':  new Device('Router', '0x0017880104e45548', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'HDC52EastwindFan'),
     'HS2WD': new Device('Router', '0x0017880104e45549', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'WarningDevice'),
@@ -169,18 +191,11 @@ const devices = {
     '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
     'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
     'TS0601_thermostat': TS0601_thermostat,
-    'external_converter_device': new Device('EndDevice', '0x0017880104e45511', 1114, 'external', [], false, null, 'external_converter_device' ),
+    'external_converter_device': new Device('EndDevice', '0x0017880104e45511', 1114, 'external', [new Endpoint(1, [], [], '0x0017880104e45511')], false, null, 'external_converter_device' ),
     'QS_Zigbee_D02_TRIAC_2C_LN':new Device('Router', '0x0017882194e45543', 6549,4151, [new Endpoint(1, [0], [], '0x0017882194e45543'), new Endpoint(2, [0, 6], [], '0x0017882194e45543')], true, "Mains (single phase)", 'TS110F', false, '_TYZB01_v8gtiaed'),
-}
-
-const groups = {
-    'group_1': new Group(1, []),
-    'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
-    'group/with/slashes': new Group(99, []),
-    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
-    'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
-    'group_with_switch': new Group(14, [devices.ZNCZ02LM.endpoints[0]]),
-    'default_bind_group': new Group(901, []),
+    'unknown': new Device('Router', '0x0017980134e45545', 6540,4151, [], true, "Mains (single phase)"),
+    'temperature_sensor': new Device('EndDevice', '0x0017880104e45561', 6544,4151, [new Endpoint(1, [0,3,4,1026], [])], true, "Battery", "temperature.sensor"),
+    'heating_actuator': new Device('Router', '0x0017880104e45562', 6545,4151, [new Endpoint(1, [0,3,4,513], [1026])], true, "Mains (single phase)", "heating.actuator"),
 }
 
 const mock = {
@@ -190,6 +205,7 @@ const mock = {
     touchlinkScan: jest.fn(),
     touchlinkIdentify: jest.fn(),
     start: jest.fn(),
+    isStopping: jest.fn(),
     permitJoin: jest.fn(),
     getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
     getNetworkParameters: jest.fn().mockReturnValue({panID: 0x162a, extendedPanID: [0, 11, 22], channel: 15}),
@@ -217,6 +233,7 @@ const mock = {
         return Object.values(groups).find((d) => d.groupID === groupID);
     }),
     getPermitJoin: jest.fn().mockReturnValue(false),
+    getPermitJoinTimeout: jest.fn().mockReturnValue(undefined),
     reset: jest.fn(),
     createGroup: jest.fn().mockImplementation((groupID) => {
         const group = new Group(groupID, []);

test/utils.test.js

@@ -46,7 +46,7 @@ describe('Utils', () => {
         var dependency = 'zigbee-herdsman';
         expect(await utils.getDependencyVersion(dependency)).toStrictEqual({"version": versionHerdsman});
     })
-    
+
     it('To local iso string', async () => {
         var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
         var getTimezoneOffset = Date.prototype.getTimezoneOffset;
@@ -61,4 +61,10 @@ describe('Utils', () => {
         var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
         expect(() => utils.formatDate(date, 'invalid', 1)).toThrowError("Unsupported type 'invalid'")
     })
+
+    it('Get key', async () => {
+        expect(utils.getKey({'1': '1'}, '1', 2, null)).toBe('1');
+        expect(utils.getKey({'1': '1'}, '2', 2, null)).toBe(2);
+        expect(utils.getKey({'1': '1'}, '1', null, () => '3')).toBe('3');
+    })
 });