1.21.0

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -1,5 +1,5 @@
 .eslintignore
-.eslintrc.json
+.eslintrc.js
 .git
 .github
 .gitignore
@@ -13,4 +13,5 @@ images
 node_modules
 scripts
 test
+coverage
 update.sh

.eslintrc.js

@@ -0,0 +1,45 @@
+
+module.exports = {
+    'env': {
+        'jest/globals': true,
+        'es6': true,
+        'node': true,
+    },
+    'extends': ['eslint:recommended', 'google', 'plugin:jest/recommended', 'plugin:jest/style'],
+    'parserOptions': {
+        'ecmaVersion': 2018,
+        'sourceType': 'module',
+    },
+    'rules': {
+        'require-jsdoc': 'off',
+        'indent': ['error', 4],
+        'max-len': ['error', {'code': 120}],
+        'no-prototype-builtins': 'off',
+    },
+    'plugins': [
+        'jest',
+    ],
+    'overrides': [{
+        files: ['*.ts'],
+        parser: '@typescript-eslint/parser',
+        plugins: ['@typescript-eslint'],
+        extends: ['plugin:@typescript-eslint/recommended'],
+        parserOptions: {
+            project: './tsconfig.json',
+        },
+        rules: {
+            '@typescript-eslint/await-thenable': 'error',
+            '@typescript-eslint/ban-ts-ignore': 'off',
+            '@typescript-eslint/explicit-function-return-type': 'error',
+            '@typescript-eslint/no-empty-function': 'off',
+            '@typescript-eslint/no-explicit-any': 'error',
+            '@typescript-eslint/no-unused-vars': 'error',
+            '@typescript-eslint/semi': ['error'],
+            'array-bracket-spacing': ['error', 'never'],
+            'indent': ['error', 4],
+            'max-len': ['error', {'code': 120}],
+            'no-return-await': 'error',
+            'object-curly-spacing': ['error', 'never'],
+        },
+    }],
+};

.eslintrc.json

@@ -1,21 +0,0 @@
-{
-    "env": {
-        "jest/globals": true,
-        "es6": true,
-        "node": true
-    },
-    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
-    "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module"
-    },
-    "rules": {
-        "require-jsdoc": "off",
-        "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }],
-        "no-prototype-builtins": "off"
-    },
-    "plugins": [
-        "jest"
-    ]
-}

.github/workflows/ci.yml

@@ -24,6 +24,8 @@ jobs:
     - name: Install dependencies
       if: steps.cache-node-modules.outputs.cache-hit != 'true'
       run: npm ci
+    - name: Build
+      run: npm run build
     - name: Test
       run: npm run test-with-coverage
     - name: Lint

.github/workflows/codeql-analysis.yml

@@ -1,67 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master, dev ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-  schedule:
-    - cron: '28 12 * * 3'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/update_deps.yml

@@ -15,9 +15,9 @@ jobs:
     - uses: actions/setup-node@v2-beta
       with:
         node-version: 12
-    - run: npm install
-    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
-    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+    - run: rm -rf npm-shrinkwrap.json
+    - run: npx npm-check-updates -u
+    - run: npm install && npm shrinkwrap --dev
     - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies

.gitignore

@@ -11,6 +11,9 @@ pids
 *.seed
 *.pid.lock
 
+# Compiled source
+dist/*
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 

.npmignore

@@ -7,6 +7,7 @@ coverage
 .jenkins.yml
 .codeclimate.yml
 .github
+babel.config.js
 
 #linters
 .jscsrc
@@ -14,11 +15,13 @@ coverage
 .eslintrc*
 .dockerignore
 .eslintignore
+.gitignore
 
 #editor settings
 .vscode
 
 #src
+lib
 docs
 docker
 images

README.md

@@ -69,6 +69,9 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 ### Internal Architecture
 Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
 
+### Developing
+Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

babel.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+};

docker/Dockerfile

@@ -5,20 +5,24 @@ RUN apk add --no-cache tzdata eudev tini
 
 COPY package.json ./
 
-# Dependencies
-FROM base as dependencies
+# Dependencies and build
+FROM base as dependencies_and_build
 
-COPY npm-shrinkwrap.json ./
+COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY lib ./lib
 
 RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
-    npm ci --production && \
+    npm ci --no-audit --no-optional --no-update-notifier && \
+    npm run build && \
+    rm -rf node_modules && \
+    npm ci --production --no-audit --no-optional --no-update-notifier && \
     apk del .buildtools
 
 # Release
 FROM base as release
 
-COPY --from=dependencies /app/node_modules ./node_modules
-COPY lib ./lib
+COPY --from=dependencies_and_build /app/node_modules ./node_modules
+COPY --from=dependencies_and_build /app/dist ./dist
 COPY LICENSE index.js data/configuration.yaml ./
 
 COPY docker/docker-entrypoint.sh /usr/local/bin/
@@ -27,7 +31,7 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 RUN mkdir /app/data
 
 ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+RUN echo "$COMMIT" > dist/.hash
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD [ "/sbin/tini", "--", "node", "index.js"]

docker/README.md

@@ -1 +0,0 @@
-qemu-aarch64-static and qemu-arm-static are taken from https://github.com/resin-io/qemu/releases/tag/v2.9.0%2Bresin1

index.js

@@ -1,10 +1,16 @@
 const semver = require('semver');
 const engines = require('./package.json').engines;
 const indexJsRestart = 'indexjs.restart';
+const fs = require('fs');
+const path = require('path');
+const {exec} = require('child_process');
+const rimraf = require('rimraf');
 
 let controller;
 let stopping = false;
 
+const hashFile = path.join('dist', '.hash');
+
 async function restart() {
     await stop(indexJsRestart);
     await start();
@@ -16,14 +22,59 @@ async function exit(code, reason) {
     }
 }
 
+async function currentHash() {
+    const git = require('git-last-commit');
+    return new Promise((resolve) => {
+        git.getLastCommit((err, commit) => {
+            if (err) resolve('unknown');
+            else resolve(commit.shortHash);
+        });
+    });
+}
+
+async function writeHash() {
+    const hash = await currentHash();
+    fs.writeFileSync(hashFile, hash);
+}
+
+async function build(reason) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
+        rimraf.sync('dist');
+        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+            if (err) {
+                process.stdout.write(', failed\n');
+                reject(err);
+            } else {
+                process.stdout.write(', finished\n');
+                resolve();
+            }
+        });
+    });
+}
+
+async function checkDist() {
+    if (!fs.existsSync(hashFile)) {
+        await build('initial build');
+    }
+
+    const distHash = fs.readFileSync(hashFile, 'utf-8');
+    const hash = await currentHash();
+    if (hash !== 'unknown' && distHash !== hash) {
+        await build('hash changed');
+    }
+}
+
 async function start() {
+    await checkDist();
+
     const version = engines.node;
     if (!semver.satisfies(process.version, version)) {
         console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
     }
 
     // Validate settings
-    const settings = require('./lib/util/settings');
+    const settings = require('./dist/util/settings');
     settings.reRead();
     const errors = settings.validate();
     if (errors.length > 0) {
@@ -38,7 +89,7 @@ async function start() {
         exit(1);
     }
 
-    const Controller = require('./lib/controller');
+    const Controller = require('./dist/controller');
     controller = new Controller(restart, exit);
     await controller.start();
 }
@@ -54,7 +105,10 @@ async function handleQuit() {
     }
 }
 
-process.on('SIGINT', handleQuit);
-process.on('SIGTERM', handleQuit);
-
-start();
+if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+    writeHash();
+} else {
+    process.on('SIGINT', handleQuit);
+    process.on('SIGTERM', handleQuit);
+    start();
+}

lib/controller.js

@@ -50,9 +50,10 @@ class Controller {
         this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
-        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+            this.enableDisableExtension, this.restartCallback, this.addExtension];
         this.extensions = [
-            new ExtensionBridge(...args, this.enableDisableExtension, this.restartCallback),
+            new ExtensionBridge(...args),
             new ExtensionPublish(...args),
             new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
@@ -89,7 +90,7 @@ class Controller {
         if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
-        this.extensions.push(new ExtensionExternalExtension(...args, this.addExtension, this.enableDisableExtension));
+        this.extensions.push(new ExtensionExternalExtension(...args));
     }
 
     async start() {
@@ -100,8 +101,9 @@ class Controller {
         logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
         // Start zigbee
+        let startResult;
         try {
-            await this.zigbee.start();
+            startResult = await this.zigbee.start();
             this.callExtensionMethod('onZigbeeStarted', []);
             this.zigbee.on('event', this.onZigbeeEvent.bind(this));
             this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
@@ -114,6 +116,14 @@ class Controller {
             this.exitCallback(1);
         }
 
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
+        }
+
         // Log zigbee clients on startup
         const devices = this.zigbee.getClients();
         logger.info(`Currently ${devices.length} devices are joined:`);
@@ -172,10 +182,12 @@ class Controller {
         } else {
             const Extension = AllExtensions.find((e) => e.name === name);
             assert(Extension, `Extension '${name}' does not exist`);
-            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+                this.enableDisableExtension, this.restartCallback, this.addExtension);
+
             this.extensions.push(extension);
-            this.callExtensionMethod('onZigbeeStarted', [], [extension]);
-            this.callExtensionMethod('onMQTTConnected', [], [extension]);
+            await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            await this.callExtensionMethod('onMQTTConnected', [], [extension]);
         }
     }
 
@@ -252,16 +264,27 @@ class Controller {
             }
         } else if (type === 'deviceAnnounce') {
             logger.debug(`Device '${name}' announced itself`);
+        } else if (type === 'deviceNetworkAddressChanged') {
+            logger.debug(`Device '${name}' changed network address`);
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
-                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                const settingsDevice = settings.getDevice(data.ieeeAddr);
+                logger.warn(`Device '${settingsDevice ? settingsDevice.friendlyName : data.ieeeAddr}' ` +
+                    `left the network`);
                 this.state.remove(data.ieeeAddr);
             }
         }
 
         // Call extensions
-        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+        const result = await this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+
+        // In case this message is not handled by the receive extension and last_seen is enabled, publish
+        // a message to update the last_seen.
+        if (name && !result[ExtensionReceive.prototype.constructor.name] &&
+            settings.get().advanced.last_seen !== 'disable' && type !== 'deviceLeave') {
+            this.publishEntityState(name, {});
+        }
     }
 
     onMQTTMessage(payload) {
@@ -378,10 +401,11 @@ class Controller {
     }
 
     async callExtensionMethod(method, parameters, extensions=null) {
+        const result = {};
         for (const extension of extensions || this.extensions) {
             if (extension[method]) {
                 try {
-                    await extension[method](...parameters);
+                    result[extension.constructor.name] = await extension[method](...parameters);
                 } catch (error) {
                     /* istanbul ignore next */
                     logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
@@ -392,6 +416,7 @@ class Controller {
                 }
             }
         }
+        return result;
     }
 }
 

lib/eventBus.js

@@ -23,13 +23,11 @@ class EventBus extends events.EventEmitter {
         super.emit(event, data);
     }
 
-    on(event, callback, extension=null) {
+    on(event, callback, extension) {
         assert(allowedEvents.includes(event), `Event '${event}' not supported`);
-        if (extension) {
-            if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
-            this.callbackByExtension[extension].push({event, callback});
-        }
-
+        assert(extension, `Extension cannot be null`);
+        if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
+        this.callbackByExtension[extension].push({event, callback});
         super.on(event, callback);
     }
 

lib/extension/availability.js

@@ -182,9 +182,8 @@ class Availability extends Extension {
             this.onReconnect(device);
         }
 
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
-        const topic = `${name}/availability`;
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        const topic = `${resolvedEntity.name}/availability`;
         const payload = available ? 'online' : 'offline';
         if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;

lib/extension/bind.js

@@ -118,7 +118,7 @@ class Bind extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.legacyApi = settings.get().advanced.legacy_api;
-        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d));
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d), this.constructor.name);
         this.pollDebouncers = {};
     }
 
@@ -127,6 +127,7 @@ class Bind extends Extension {
         let sourceKey = null;
         let targetKey = null;
         let clusters = null;
+        let skipDisableReporting = false;
 
         if (this.legacyApi && topic.match(legacyTopicRegex)) {
             topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
@@ -139,13 +140,14 @@ class Bind extends Extension {
             sourceKey = message.from;
             targetKey = message.to;
             clusters = message.clusters;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
         }
 
-        return {type, sourceKey, targetKey, clusters};
+        return {type, sourceKey, targetKey, clusters, skipDisableReporting};
     }
 
     async onMQTTMessage(topic, message) {
-        const {type, sourceKey, targetKey, clusters} = this.parseMQTTMessage(topic, message);
+        const {type, sourceKey, targetKey, clusters, skipDisableReporting} = this.parseMQTTMessage(topic, message);
         if (!type) return null;
         message = utils.parseJSON(message, message);
 
@@ -255,7 +257,7 @@ class Bind extends Extension {
                 if (type === 'bind') {
                     await this.setupReporting(source.endpoint.binds.filter((b) =>
                         successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
-                } else if (target.type !== 'group_number') {
+                } else if ((target.type !== 'group_number') && !skipDisableReporting) {
                     await this.disableUnnecessaryReportings(bindTarget);
                 }
             }
@@ -275,13 +277,15 @@ class Bind extends Extension {
     }
 
     async groupMembersChanged(data) {
-        const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
-            .flat().filter((b) => b.target === data.group.group);
-
         if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v)).map((e) => e.binds)
+                .reduce((a, v) => a.concat(v)).filter((b) => b.target === data.group.group);
             await this.setupReporting(bindsToGroup);
         } else { // action === remove/remove_all
-            await this.disableUnnecessaryReportings(data.endpoint);
+            if (!data.skipDisableReporting) {
+                await this.disableUnnecessaryReportings(data.endpoint);
+            }
         }
     }
 
@@ -330,8 +334,8 @@ class Bind extends Extension {
             const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
             const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
                 .map((b) => b.cluster.name);
-            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
-                .flat().filter((bind) => {
+            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).reduce((a, v) => a.concat(v))
+                .map((e) => e.binds).reduce((a, v) => a.concat(v)).filter((bind) => {
                     if (bind.target.constructor.name === 'Group') {
                         return bind.target.members.includes(endpoint);
                     } else {

lib/extension/bridge.js

@@ -47,23 +47,30 @@ class Bridge extends Extension {
         this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
-        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups(), this.constructor.name);
         this.eventBus.on(`devicesChanged`, () => {
             this.publishDevices();
             this.publishInfo();
-        });
+        }, this.constructor.name);
         this.eventBus.on(`deviceRenamed`, () => {
             this.publishInfo();
-        });
+        }, this.constructor.name);
         this.eventBus.on(`groupRenamed`, () => {
             this.publishInfo();
-        });
-        this.zigbee.on('permitJoinChanged', (data) => this.permitJoinChanged(data));
+        }, this.constructor.name);
+        this.zigbee.removeListener('permitJoinChanged', this.permitJoinChanged);
+        this.permitJoinChanged = this.permitJoinChanged.bind(this);
+        this.zigbee.on('permitJoinChanged', this.permitJoinChanged);
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
     }
 
+    async stop() {
+        super.stop();
+        this.zigbee.removeListener('permitJoinChanged', this.permitJoinChanged);
+    }
+
     setupMQTTLogging() {
         const mqtt = this.mqtt;
         class EventTransport extends Transport {
@@ -122,7 +129,12 @@ class Bridge extends Extension {
                 payload = {
                     friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
                 };
-            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
+            } else {
+                // deviceLeave
+                const settingsDevice = settings.getDevice(ieeeAddress);
+                payload = {ieee_address: ieeeAddress,
+                    friendly_name: settingsDevice ? settingsDevice.friendlyName : null};
+            }
 
             await this.mqtt.publish(
                 'bridge/event',
@@ -131,7 +143,8 @@ class Bridge extends Extension {
             );
         }
 
-        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started') ||
+            'deviceNetworkAddressChanged' === type) {
             await this.publishDevices();
         }
     }
@@ -625,6 +638,7 @@ class Bridge extends Extension {
                 model_id: device.modelID,
                 interviewing: device.interviewing,
                 interview_completed: device.interviewCompleted,
+                manufacturer: device.manufacturerName,
                 endpoints,
             };
         });

lib/extension/configure.js

@@ -3,6 +3,7 @@ const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 const stringify = require('json-stable-stringify-without-jsonify');
+const zhc = require('zigbee-herdsman-converters');
 
 /**
  * This extension calls the zigbee-herdsman-converters definition configure() method
@@ -17,7 +18,7 @@ class Configure extends Extension {
 
         this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled);
+        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled, this.constructor.name);
     }
 
     onReportingDisabled(data) {
@@ -42,7 +43,7 @@ class Configure extends Extension {
 
         if (resolvedEntity.device.meta &&
             resolvedEntity.device.meta.hasOwnProperty('configured') &&
-            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            resolvedEntity.device.meta.configured === zhc.getConfigureKey(resolvedEntity.definition)) {
             return false;
         }
 
@@ -135,7 +136,7 @@ class Configure extends Extension {
         try {
             await resolvedEntity.definition.configure(device, this.coordinatorEndpoint, logger);
             logger.info(`Successfully configured '${resolvedEntity.name}'`);
-            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.meta.configured = zhc.getConfigureKey(resolvedEntity.definition);
             device.save();
             this.eventBus.emit(`devicesChanged`);
         } catch (error) {

lib/extension/externalExtension.js

@@ -9,7 +9,8 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
 
 class ExternalExtension extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus, addExtension, enableDisableExtension) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus,
+        enableDisableExtension, restartCallback, addExtension) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
         this.addExtension = addExtension;

lib/extension/frontend.js

@@ -19,8 +19,8 @@ class Frontend extends Extension {
         this.onRequest = this.onRequest.bind(this);
         this.onUpgrade = this.onUpgrade.bind(this);
         this.mqttBaseTopic = settings.get().mqtt.base_topic;
-        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
-        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
+        this.onMQTTPublishMessage = this.onMQTTPublishMessage.bind(this);
+        this.mqtt.on('publishMessage', this.onMQTTPublishMessage);
         this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
         this.server = http.createServer(this.onRequest);
         this.server.on('upgrade', this.onUpgrade);
@@ -106,7 +106,7 @@ class Frontend extends Extension {
         }
     }
 
-    onMQTTPublishedMessage(data) {
+    onMQTTPublishMessage(data) {
         let {topic, payload, options} = data;
         if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
             // Send topic without base_topic

lib/extension/groups.js

@@ -11,6 +11,21 @@ const topicRegex =
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
+const stateProperties = {
+    'state': (value, definition) => true,
+    'brightness': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
 class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
@@ -20,7 +35,7 @@ class Groups extends Extension {
     }
 
     async onZigbeeStarted() {
-        this.eventBus.on('stateChange', this.onStateChange);
+        this.eventBus.on('stateChange', this.onStateChange, this.constructor.name);
         await this.syncGroupsWithSettings();
     }
 
@@ -38,6 +53,7 @@ class Groups extends Extension {
                 }
             } catch (error) {
                 logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+                logger.debug(error.stack);
             }
         };
 
@@ -82,7 +98,6 @@ class Groups extends Extension {
             return;
         }
 
-        const properties = ['state', 'brightness', 'color_temp', 'color', 'color_mode'];
         const payload = {};
 
         let endpointName;
@@ -93,7 +108,7 @@ class Groups extends Extension {
                 endpointName = endpointNameMatch;
             }
 
-            if (properties.includes(prop)) {
+            if (prop in stateProperties) {
                 payload[prop] = value;
             }
         }
@@ -102,7 +117,7 @@ class Groups extends Extension {
             const resolvedEntity = this.zigbee.resolveEntity(`${data.ID}${endpointName ? `/${endpointName}` : ''}`);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
-                return settingsGroup && settingsGroup.optimistic;
+                return settingsGroup && (!settingsGroup.hasOwnProperty('optimistic') || settingsGroup.optimistic);
             });
 
             if (resolvedEntity.type === 'device') {
@@ -121,8 +136,15 @@ class Groups extends Extension {
 
                 const groupIDsToPublish = new Set();
                 for (const member of resolvedEntity.group.members) {
-                    const memberPayload = {...payload};
-                    const endpointName = this.zigbee.resolveEntity(member).endpointName;
+                    const resolvedEntity = this.zigbee.resolveEntity(member);
+                    const memberPayload = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], resolvedEntity.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = resolvedEntity.endpointName;
                     if (endpointName) {
                         Object.keys(memberPayload).forEach((key) => {
                             memberPayload[`${key}_${endpointName}`] = memberPayload[key];
@@ -169,6 +191,7 @@ class Groups extends Extension {
         let groupKey = null;
         let deviceKey = null;
         let triggeredViaLegacyApi = false;
+        let skipDisableReporting = false;
 
         /* istanbul ignore else */
         const topicRegexMatch = topic.match(topicRegex);
@@ -224,6 +247,7 @@ class Groups extends Extension {
             type = topicRegexMatch[1];
             message = JSON.parse(message);
             deviceKey = message.device;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
 
             if (type !== 'remove_all') {
                 groupKey = message.group;
@@ -243,14 +267,14 @@ class Groups extends Extension {
 
         return {
             resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
-            triggeredViaLegacyApi,
+            triggeredViaLegacyApi, skipDisableReporting,
         };
     }
 
     async onMQTTMessage(topic, message) {
         let {
             resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
-            groupKey, deviceKey,
+            groupKey, deviceKey, skipDisableReporting,
         } = this.parseMQTTMessage(topic, message);
         if (!type) return;
         message = utils.parseJSON(message, message);
@@ -270,8 +294,11 @@ class Groups extends Extension {
                 const definition = resolvedEntityDevice.definition;
                 const endpoints = definition && definition.endpoint ?
                     definition.endpoint(resolvedEntityDevice.device) : null;
-                const endpointName = endpoints ?
-                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+                let endpointName = null;
+                if (endpoints) {
+                    const matches = Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID);
+                    endpointName = matches && matches[0];
+                }
 
                 if (endpointName) {
                     keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
@@ -327,6 +354,7 @@ class Groups extends Extension {
                 }
             } catch (e) {
                 error = `Failed to ${type} from group (${e.message})`;
+                logger.debug(e.stack);
             }
         }
 
@@ -339,7 +367,8 @@ class Groups extends Extension {
             logger.error(error);
         } else {
             this.eventBus.emit('groupMembersChanged',
-                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint});
+                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint,
+                    skipDisableReporting: skipDisableReporting});
         }
     }
 }

lib/extension/networkMap.js

@@ -84,7 +84,7 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
@@ -132,7 +132,7 @@ class NetworkMap extends Extension {
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
-                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const textRoutes = e.routes.map((r) => utils.toNetworkAddressHex(r.destinationAddress));
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
                 text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
@@ -160,7 +160,7 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             text.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
             );
 
@@ -214,8 +214,23 @@ class NetworkMap extends Extension {
             failed.set(device, []);
             await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
             const resolvedEntity = this.zigbee.resolveEntity(device);
+
+            const doRequest = async (request, firstAttempt = true) => {
+                try {
+                    return await request();
+                } catch (error) {
+                    if (!firstAttempt) {
+                        throw error;
+                    } else {
+                        // Network is possibly congested, sleep 5 seconds to let the network settle.
+                        await utils.sleep(5);
+                        return await doRequest(request, false);
+                    }
+                }
+            };
+
             try {
-                const result = await device.lqi();
+                const result = await doRequest(async () => await device.lqi());
                 lqis.set(device, result);
                 logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
@@ -225,7 +240,7 @@ class NetworkMap extends Extension {
 
             if (includeRoutes) {
                 try {
-                    const result = await device.routingTable();
+                    const result = await doRequest(async () => await device.routingTable());
                     routingTables.set(device, result);
                     logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {

lib/extension/otaUpdate.js

@@ -235,6 +235,7 @@ class OTAUpdate extends Extension {
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
                     }
                 } catch (e) {
+                    logger.debug(`Update of '${resolvedEntity.name}' failed (${e})`);
                     error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
 
                     this.removeProgressAndRemainingFromState(resolvedEntity.device);

lib/extension/publish.js

@@ -10,7 +10,7 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
 const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 
-const groupConverters = [
+const defaultGroupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
     zigbeeHerdsmanConverters.toZigbeeConverters.effect,
@@ -91,11 +91,9 @@ class EntityPublish extends Extension {
             options = resolvedEntity.settings;
             definition = resolvedEntity.group.members
                 .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
-            converters = new Set(groupConverters);
-            for (const d of definition) {
-                d.toZigbee.forEach(converters.add, converters);
-            }
-            converters = [...converters];
+            converters = new Set();
+            definition.forEach((d) => d.toZigbee.forEach(converters.add, converters));
+            converters = converters.size ? [...converters] : defaultGroupConverters;
             membersState = {};
             for (const member of resolvedEntity.group.members) {
                 const ieeeAddr = member.getDevice().ieeeAddr;
@@ -160,19 +158,28 @@ class EntityPublish extends Extension {
 
         // For each attribute call the corresponding converter
         const usedConverters = {};
+        const toPublish = {};
+        const addToToPublish = (ID, payload) => {
+            if (!(ID in toPublish)) toPublish[ID] = {};
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
         for (let [key, value] of entries) {
             let endpointName = topic.endpointName;
             let actualTarget = target;
+            let endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
                 const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
                 if (endpointNameMatch) {
                     endpointName = endpointNameMatch;
-                    key = key.replace(`_${endpointNameMatch}`, '');
+                    const regex = new RegExp(`(_${endpointNameMatch})$`, 'gm');
+                    key = key.replace(regex, ``);
 
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+                    endpointOrGroupID = endpointName;
                     if (!actualTarget) {
                         logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
                         continue;
@@ -180,7 +187,6 @@ class EntityPublish extends Extension {
                 }
             }
 
-            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
             if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
@@ -222,27 +228,28 @@ class EntityPublish extends Extension {
                 if (topic.type === 'set' && converter.convertSet) {
                     logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
-                    const isGroup = actualTarget.constructor.name == 'Group';
-                    // Optimistic behaviour of groups and devices is different.
-                    const publishOptimistic = isGroup || (!options.hasOwnProperty('optimistic') || options.optimistic);
-                    if (result && result.state && publishOptimistic) {
+                    const optimistic = !options.hasOwnProperty('optimistic') || options.optimistic;
+                    if (result && result.state && optimistic) {
                         const msg = result.state;
 
                         if (endpointName) {
-                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
-                                if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${endpointName}`] = msg[key];
-                                    delete msg[key];
-                                }
+                            for (const key of Object.keys(msg)) {
+                                msg[`${key}_${endpointName}`] = msg[key];
+                                delete msg[key];
                             }
                         }
 
-                        this.publishEntityState(resolvedEntity.settings.ID, msg);
+                        // filter out attribute listed in filtered_optimistic
+                        if (options.hasOwnProperty('filtered_optimistic')) {
+                            options.filtered_optimistic.forEach((a) => delete msg[a]);
+                        }
+
+                        addToToPublish(resolvedEntity.settings.ID, msg);
                     }
 
-                    if (result && result.membersState && publishOptimistic) {
+                    if (result && result.membersState && optimistic) {
                         for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
-                            this.publishEntityState(ieeeAddr, state);
+                            addToToPublish(ieeeAddr, state);
                         }
                     }
 
@@ -286,6 +293,10 @@ class EntityPublish extends Extension {
             usedConverters[endpointOrGroupID].push(converter);
         }
 
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            this.publishEntityState(ID, payload);
+        }
+
         return true;
     }
 }

lib/extension/receive.js

@@ -10,7 +10,7 @@ class Receive extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.elapsed = {};
         this.debouncers = {};
-        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data), this.constructor.name);
     }
 
     async onZigbeeStarted() {
@@ -166,6 +166,7 @@ class Receive extends Extension {
 
         if (Object.keys(payload).length) {
             publish(payload);
+            return true;
         }
     }
 }

lib/mqtt.js

@@ -116,6 +116,8 @@ class MQTT extends events.EventEmitter {
             this.publishedTopics.add(topic);
         }
 
+        this.emit('publishMessage', {topic, payload, options});
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -135,7 +137,6 @@ class MQTT extends events.EventEmitter {
 
         return new Promise((resolve) => {
             this.client.publish(topic, payload, actualOptions, () => {
-                this.emit('publishedMessage', {topic, payload, options});
                 resolve();
             });
         });

lib/state.js

@@ -7,9 +7,9 @@ const objectAssignDeep = require('object-assign-deep');
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
+    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
+    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
+    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
 ];
 
 class State {

lib/types/types.ts

@@ -0,0 +1,108 @@
+/* eslint-disable */
+
+interface KeyValue {
+    [s: string]: any,
+}
+
+interface Settings {
+    devices?: {[s: string]: {friendly_name: string, retention?: number}},
+    groups?: {[s: string]: {friendly_name: string, devices?: string[]}},
+    passlist: string[],
+    blocklist: string[],
+    whitelist: string[],
+    ban: string[],
+    permit_join: boolean,
+    frontend?: {
+        auth_token?: string,
+    },
+    mqtt: {
+        include_device_information: boolean,
+        force_disable_retain: boolean
+        version?: number,
+        user?: string,
+        password?: string,
+    },
+    serial: {
+        disable_led: boolean,
+    },
+    device_options: {[s: string]: unknown},
+    map_options: {
+        graphviz: {
+            colors: {
+                fill: {
+                    enddevice: string,
+                    coordinator: string,
+                    router: string,
+                },
+                font: {
+                    coordinator: string,
+                    router: string,
+                    enddevice: string,
+                },
+                line: {
+                    active: string,
+                    inactive: string,
+                },
+            },
+        },
+    },
+    experimental: {
+        output: 'json' | 'attribute' | 'attribute_and_json',
+    },
+    advanced: {
+        legacy_api: boolean,
+        log_rotation: boolean,
+        log_symlink_current: boolean,
+        log_output: ('console' | 'file')[],
+        log_directory: string,
+        log_file: string,
+        log_level: 'debug' | 'info' | 'error' | 'warn',
+        log_syslog: {},
+        soft_reset_timeout: number,
+        pan_id: number,
+        ext_pan_id: number[],
+        channel: number,
+        adapter_concurrent: number | null,
+        adapter_delay: number | null,
+        availability_timeout: number,
+        availability_blocklist: string[],
+        availability_passlist: string[],
+        availability_blacklist: string[],
+        availability_whitelist: string[],
+        cache_state: boolean,
+        cache_state_persistent: boolean,
+        cache_state_send_on_startup: boolean,
+        last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' |  'epoch',
+        elapsed: boolean,
+        network_key: number[],
+        report: boolean,
+        homeassistant_discovery_topic: string,
+        homeassistant_status_topic: string,
+        homeassistant_legacy_entity_attributes: boolean,
+        homeassistant_legacy_triggers: boolean,
+        timestamp_format: string,
+    },
+    ota: {
+        update_check_interval: number,
+        disable_automatic_update_check: boolean
+    },
+    external_converters: string[],
+}
+
+interface DeviceSettings {
+    friendlyName: string,
+    ID: string,
+    retention?: number,
+}
+
+interface GroupSettings {
+    friendlyName: string,
+    devices: string[],
+    ID: number,
+}
+
+type EntitySettings = {
+    type: 'device' | 'group'
+    ID: number | string,
+    friendlyName: string,
+}

lib/util/settings.schema.json

@@ -112,7 +112,7 @@
           "requiresRestart": true,
           "description": "MQTT protocol version",
           "default": 4,
-          "examples": [4, 5]
+          "examples": [5]
         },
         "force_disable_retain": {
           "type": "boolean",
@@ -143,9 +143,10 @@
           "default": false
         },
         "adapter": {
-          "type": ["string", "null"],
-          "enum": ["deconz", "zstack", "zigate", "ezsp"],
+          "type": ["string"],
+          "enum": ["deconz", "zstack", "zigate", "ezsp", "auto"],
           "title": "Adapter",
+          "default": "auto",
           "requiresRestart": true,
           "description": "Adapter type, not needed unless you are experiencing problems"
         }
@@ -247,7 +248,7 @@
           "title": "ZigBee channel",
           "requiresRestart": true,
           "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
-          "examples": [11, 15, 20, 25]
+          "examples": [15, 20, 25]
         },
         "cache_state": {
           "type": "boolean",
@@ -422,6 +423,12 @@
           "requiresRestart": true,
           "examples": ["homeassistant"]
         },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
         "homeassistant_status_topic": {
           "type": "string",
           "title": "Home Assistant status topic",
@@ -483,7 +490,6 @@
               "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
               "default": "tcp4",
               "examples": [
-                "tcp4",
                 "udp4",
                 "tls4",
                 "unix",
@@ -496,7 +502,6 @@
               "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
               "default": "/dev/log",
               "examples": [
-                "/dev/log",
                 "/var/run/syslog"
               ]
             },
@@ -680,7 +685,7 @@
         "qos": {
           "type": "number",
           "title": "QoS",
-          "descritption": "QoS level for MQTT messages of this device"
+          "description": "QoS level for MQTT messages of this device"
         },
         "debounce": {
           "type": "number",
@@ -699,7 +704,17 @@
         "optimistic": {
           "type": "boolean",
           "title": "Optimistic",
-          "description": "Publish optimistic state after set (default true)"
+          "description": "Publish optimistic state after set",
+          "default": true
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["color_mode", "color_temp", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
         },
         "filtered_attributes": {
           "type": "array",
@@ -707,8 +722,8 @@
             "type": "string"
           },
           "examples": ["temperature", "battery", "action"],
-          "title": "Filtered attributes",
-          "description": "Allows to prevent certain attributes from being published"
+          "title": "Filtered publish attributes",
+          "description": "Filter attributes from publish payload."
         },
         "icon": {
           "type": "string",

lib/util/settings.ts

@@ -1,17 +1,19 @@
-const data = require('./data');
-const utils = require('./utils');
-// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
-const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
-const objectAssignDeep = require(`object-assign-deep`);
-const path = require('path');
-const yaml = require('./yaml');
-const Ajv = require('ajv');
-const schema = require('./settings.schema.json');
-const ajvSetting = new Ajv({allErrors: true}).compile(schema);
-const ajvRestartRequired = new Ajv({allErrors: true})
-    .addKeyword('requiresRestart', {validate: (v) => !v}).compile(schema);
+import data from './data';
+import utils from './utils';
+import objectAssignDeep from 'object-assign-deep';
+import path from 'path';
+import * as yaml from './yaml';
+import Ajv from 'ajv';
+import schemaJson from './settings.schema.json';
+export const schema = schemaJson;
 
-const defaults = {
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+
+const defaults: Settings = {
     passlist: [],
     blocklist: [],
     // Deprecated: use block/passlist
@@ -125,6 +127,15 @@ const defaults = {
          */
         homeassistant_status_topic: 'hass/status',
 
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
         /**
          * Home Assistant legacy triggers, when enabled:
          * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
@@ -152,11 +163,12 @@ const defaults = {
     external_converters: [],
 };
 
-let _settings;
-let _settingsWithDefaults;
+let _settings: Partial<Settings>;
+let _settingsWithDefaults: Settings;
 
-function write() {
-    const settings = get();
+function write(): void {
+    const settings = getInternalSettings();
+    /* eslint-disable-line */ // @ts-ignore
     const toWrite = objectAssignDeep.noMutate(settings);
 
     // Read settings to check if we have to split devices/groups into separate file.
@@ -179,17 +191,18 @@ function write() {
     }
 
     // Write devices/groups to separate file if required.
-    const writeDevicesOrGroups = (type) => {
+    const writeDevicesOrGroups = (type: 'devices' | 'groups'): void => {
         if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
             const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            /* eslint-disable-line */ // @ts-ignore
             const content = objectAssignDeep.noMutate(settings[type]);
 
             // If an array, only write to first file and only devices which are not in the other files.
             if (Array.isArray(actual[type])) {
-                actual[type].filter((f, i) => i !== 0)
-                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
-                    .map((c) => Object.keys(c))
-                    .forEach((k) => delete content[k]);
+                actual[type].filter((f: string, i: number) => i !== 0)
+                    .map((f: string) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c: KeyValue) => Object.keys(c))
+                    .forEach((k: string) => delete content[k]);
             }
 
             yaml.writeIfChanged(data.joinPath(fileToWrite), content);
@@ -203,12 +216,13 @@ function write() {
     yaml.writeIfChanged(file, toWrite);
 
     _settings = read();
-    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    /* eslint-disable-line */ // @ts-ignore
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
 }
 
-function validate() {
+export function validate(): string[] {
     try {
-        get();
+        getInternalSettings();
     } catch (error) {
         if (error.name === 'YAMLException') {
             return [
@@ -221,7 +235,7 @@ function validate() {
     }
 
     if (!ajvSetting(_settings)) {
-        return ajvSetting.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+        return ajvSetting.errors.map((v) => `${v.instancePath.substring(1)} ${v.message}`);
     }
 
     const errors = [];
@@ -236,14 +250,14 @@ function validate() {
     }
 
     // Verify that all friendly names are unique
-    const names = [];
-    const check = (name) => {
+    const names: string[] = [];
+    const check = (name: string): void => {
         if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
         errors.push(...utils.validateFriendlyName(name));
         names.push(name);
     };
 
-    const settingsWithDefaults = getWithDefaults();
+    const settingsWithDefaults = get();
     Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
     Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
 
@@ -255,7 +269,7 @@ function validate() {
         }
     }
 
-    const checkAvailabilityList = (list, type) => {
+    const checkAvailabilityList = (list: string[], type: string): void => {
         list.forEach((e) => {
             if (!getEntity(e)) {
                 errors.push(`Non-existing entity '${e}' specified in '${type}'`);
@@ -271,11 +285,12 @@ function validate() {
     return errors;
 }
 
-function read() {
-    const s = yaml.read(file);
+function read(): Settings {
+    const s = yaml.read(file) as Settings;
 
     // Read !secret MQTT username and password if set
-    const interpetValue = (value) => {
+    // eslint-disable-next-line
+    const interpetValue = (value: any): any => {
         const re = /!(.*) (.*)/g;
         const match = re.exec(value);
         if (match) {
@@ -287,26 +302,28 @@ function read() {
         }
     };
 
-    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+    if (s.mqtt?.user && s.mqtt?.password) {
         s.mqtt.user = interpetValue(s.mqtt.user);
         s.mqtt.password = interpetValue(s.mqtt.password);
     }
 
-    if (s.advanced && s.advanced.network_key) {
+    if (s.advanced?.network_key) {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    if (s.frontend && s.frontend.auth_token) {
+    if (s.frontend?.auth_token) {
         s.frontend.auth_token = interpetValue(s.frontend.auth_token);
     }
 
     // Read devices/groups configuration from separate file if specified.
-    const readDevicesOrGroups = (type) => {
+    const readDevicesOrGroups = (type: 'devices' | 'groups'): void => {
         if (typeof s[type] === 'string' || Array.isArray(s[type])) {
-            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            /* eslint-disable-line */ // @ts-ignore
+            const files: string[] = Array.isArray(s[type]) ? s[type] : [s[type]];
             s[type] = {};
             for (const file of files) {
                 const content = yaml.readIfExists(data.joinPath(file), {});
+                /* eslint-disable-line */ // @ts-ignore
                 s[type] = objectAssignDeep.noMutate(s[type], content);
             }
         }
@@ -318,23 +335,26 @@ function read() {
     return s;
 }
 
-function applyEnvironmentVariables(settings) {
-    const iterate = (obj, path) => {
+function applyEnvironmentVariables(settings: Partial<Settings>): void {
+    const iterate = (obj: KeyValue, path: string[]): void => {
         Object.keys(obj).forEach((key) => {
             if (key !== 'type') {
-                if (key !== 'properties') {
+                if (key !== 'properties' && obj[key]) {
                     const type = (obj[key].type || 'object').toString();
                     const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
                     const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
                     if (process.env[envVariableName]) {
-                        const setting = path.reduce((acc, val, index) => {
+                        const setting = path.reduce((acc, val) => {
+                            /* eslint-disable-line */ // @ts-ignore
                             acc[val] = acc[val] || {};
+                            /* eslint-disable-line */ // @ts-ignore
                             return acc[val];
                         }, settings);
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
                             setting[key] = JSON.parse(process.env[envVariableName]);
                         } else if (type.indexOf('number') >= 0) {
+                            /* eslint-disable-line */ // @ts-ignore
                             setting[key] = process.env[envVariableName] * 1;
                         } else if (type.indexOf('boolean') >= 0) {
                             setting[key] = process.env[envVariableName].toLowerCase() === 'true';
@@ -347,7 +367,7 @@ function applyEnvironmentVariables(settings) {
                     }
                 }
 
-                if (typeof obj[key] === 'object') {
+                if (typeof obj[key] === 'object' && obj[key]) {
                     const newPath = [...path];
                     if (key !== 'properties') {
                         newPath.push(key);
@@ -360,7 +380,7 @@ function applyEnvironmentVariables(settings) {
     iterate(schema.properties, []);
 }
 
-function get() {
+function getInternalSettings(): Partial<Settings> {
     if (!_settings) {
         _settings = read();
         applyEnvironmentVariables(_settings);
@@ -369,9 +389,10 @@ function get() {
     return _settings;
 }
 
-function getWithDefaults() {
+export function get(): Settings {
     if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+        /* eslint-disable-line */ // @ts-ignore
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
     }
 
     if (!_settingsWithDefaults.devices) {
@@ -385,8 +406,9 @@ function getWithDefaults() {
     return _settingsWithDefaults;
 }
 
-function set(path, value) {
-    let settings = get();
+export function set(path: string[], value: string | number): void {
+    /* eslint-disable-next-line */
+    let settings: any = getInternalSettings();
 
     for (let i = 0; i < path.length; i++) {
         const key = path[i];
@@ -404,15 +426,16 @@ function set(path, value) {
     write();
 }
 
-function apply(newSettings) {
+export function apply(newSettings: Record<string, unknown>): boolean {
     ajvSetting(newSettings);
     const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
     if (errors.length) {
         const error = errors[0];
-        throw new Error(`${error.dataPath.substring(1)} ${error.message}`);
+        throw new Error(`${error.instancePath.substring(1)} ${error.message}`);
     }
 
-    get(); // Ensure _settings is intialized.
+    getInternalSettings(); // Ensure _settings is intialized.
+    /* eslint-disable-line */ // @ts-ignore
     _settings = objectAssignDeep.noMutate(_settings, newSettings);
     write();
 
@@ -422,30 +445,30 @@ function apply(newSettings) {
     return restartRequired;
 }
 
-function getGroup(IDorName) {
-    const settings = getWithDefaults();
+export function getGroup(IDorName: string): GroupSettings {
+    const settings = get();
     const byID = settings.groups[IDorName];
     if (byID) {
-        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+        return {devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
     }
 
     for (const [ID, group] of Object.entries(settings.groups)) {
         if (group.friendly_name === IDorName) {
-            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+            return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
         }
     }
 
     return null;
 }
 
-function getGroups() {
-    const settings = getWithDefaults();
+export function getGroups(): GroupSettings[] {
+    const settings = get();
     return Object.entries(settings.groups).map(([ID, group]) => {
-        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
     });
 }
 
-function getGroupThrowIfNotExists(IDorName) {
+function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
     const group = getGroup(IDorName);
     if (!group) {
         throw new Error(`Group '${IDorName}' does not exist`);
@@ -454,8 +477,8 @@ function getGroupThrowIfNotExists(IDorName) {
     return group;
 }
 
-function getDevice(IDorName) {
-    const settings = getWithDefaults();
+export function getDevice(IDorName: string): DeviceSettings {
+    const settings = get();
     const byID = settings.devices[IDorName];
     if (byID) {
         return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
@@ -470,7 +493,7 @@ function getDevice(IDorName) {
     return null;
 }
 
-function getDeviceThrowIfNotExists(IDorName) {
+function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
     const device = getDevice(IDorName);
     if (!device) {
         throw new Error(`Device '${IDorName}' does not exist`);
@@ -479,7 +502,7 @@ function getDeviceThrowIfNotExists(IDorName) {
     return device;
 }
 
-function getEntity(IDorName) {
+export function getEntity(IDorName: string): EntitySettings {
     const device = getDevice(IDorName);
     if (device) {
         return {...device, type: 'device'};
@@ -493,12 +516,12 @@ function getEntity(IDorName) {
     return null;
 }
 
-function addDevice(ID) {
+export function addDevice(ID: string): DeviceSettings {
     if (getDevice(ID)) {
         throw new Error(`Device '${ID}' already exists`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
 
     if (!settings.devices) {
         settings.devices = {};
@@ -509,9 +532,8 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
-// Legacy: can be removed after bridgeLegacy has been removed
-function whitelistDevice(ID) {
-    const settings = get();
+export function whitelistDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.whitelist) {
         settings.whitelist = [];
     }
@@ -524,8 +546,8 @@ function whitelistDevice(ID) {
     write();
 }
 
-function blockDevice(ID) {
-    const settings = get();
+export function blockDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.blocklist) {
         settings.blocklist = [];
     }
@@ -534,8 +556,8 @@ function blockDevice(ID) {
     write();
 }
 
-function banDevice(ID) {
-    const settings = get();
+export function banDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.ban) {
         settings.ban = [];
     }
@@ -544,20 +566,30 @@ function banDevice(ID) {
     write();
 }
 
-function removeDevice(IDorName) {
+export function removeDevice(IDorName: string): void {
     const device = getDeviceThrowIfNotExists(IDorName);
-    const settings = get();
+    const settings = getInternalSettings();
     delete settings.devices[device.ID];
+
+    // Remove device from groups
+    if (settings.groups) {
+        const regex =
+            new RegExp(`^(${device.friendlyName}|${device.ID})(/(\\d|${utils.getEndpointNames().join('|')}))?$`);
+        for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
+            group.devices = group.devices.filter((device) => !device.match(regex));
+        }
+    }
+
     write();
 }
 
-function addGroup(name, ID=null) {
+export function addGroup(name: string, ID?: string): GroupSettings {
     utils.validateFriendlyName(name, true);
     if (getGroup(name) || getDevice(name)) {
         throw new Error(`friendly_name '${name}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (!settings.groups) {
         settings.groups = {};
     }
@@ -572,7 +604,7 @@ function addGroup(name, ID=null) {
         // ensure provided ID is not in use
         ID = ID.toString();
         if (settings.groups.hasOwnProperty(ID)) {
-            throw new Error(`group id '${ID}' is already in use`);
+            throw new Error(`Group ID '${ID}' is already in use`);
         }
     }
 
@@ -582,56 +614,50 @@ function addGroup(name, ID=null) {
     return getGroup(ID);
 }
 
-function groupHasDevice(group, keys) {
-    for (const device of group.devices) {
-        const index = keys.indexOf(device);
-        if (index != -1) {
-            return keys[index];
-        }
+function groupGetDevice(group: {devices?: string[]}, keys: string[]): string {
+    for (const device of group.devices ?? []) {
+        if (keys.includes(device)) return device;
     }
 
-    return false;
+    return null;
 }
 
-function addDeviceToGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function addDeviceToGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
 
     const group = settings.groups[groupID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    if (!groupHasDevice(group, keys)) {
+    if (!groupGetDevice(group, keys)) {
+        if (!group.devices) group.devices = [];
         group.devices.push(keys[0]);
         write();
     }
 }
 
-function removeDeviceFromGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeDeviceFromGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
     const group = settings.groups[groupID];
     if (!group.devices) {
-        group.devices = [];
+        return;
     }
 
-    const key = groupHasDevice(group, keys);
+    const key = groupGetDevice(group, keys);
     if (key) {
         group.devices = group.devices.filter((d) => d != key);
         write();
     }
 }
 
-function removeGroup(groupIDorName) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeGroup(IDorName: string): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
     delete settings.groups[groupID];
     write();
 }
 
-function changeEntityOptions(IDorName, newOptions) {
-    const settings = get();
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+    const settings = getInternalSettings();
     delete newOptions.friendly_name;
     delete newOptions.devices;
     if (getDevice(IDorName)) {
@@ -647,13 +673,13 @@ function changeEntityOptions(IDorName, newOptions) {
     write();
 }
 
-function changeFriendlyName(IDorName, newName) {
+export function changeFriendlyName(IDorName: string, newName: string): void {
     utils.validateFriendlyName(newName, true);
     if (getGroup(newName) || getDevice(newName)) {
         throw new Error(`friendly_name '${newName}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (getDevice(IDorName)) {
         settings.devices[getDevice(IDorName).ID].friendly_name = newName;
     } else if (getGroup(IDorName)) {
@@ -665,39 +691,18 @@ function changeFriendlyName(IDorName, newName) {
     write();
 }
 
-module.exports = {
-    validate,
-    get: getWithDefaults,
-    set,
-    apply,
-    getDevice,
-    getGroup,
-    getGroups,
-    getEntity,
-    whitelistDevice,
-    banDevice,
-    blockDevice,
-    addDevice,
-    removeDevice,
-    addGroup,
-    removeGroup,
-    addDeviceToGroup,
-    removeDeviceFromGroup,
-    changeEntityOptions,
-    changeFriendlyName,
-    schema,
-    reRead: () => {
-        _settings = null;
-        get();
-        _settingsWithDefaults = null;
-        getWithDefaults();
-    },
+export function reRead(): void {
+    _settings = null;
+    getInternalSettings();
+    _settingsWithDefaults = null;
+    get();
+}
 
-    // For tests only
-    _write: write,
-    _clear: () => {
+export const testing = {
+    write,
+    clear: (): void => {
         _settings = null;
         _settingsWithDefaults = null;
     },
-    _getDefaults: () => defaults,
+    defaults,
 };

lib/util/utils.js

@@ -52,7 +52,7 @@ function capitalize(s) {
 async function getZigbee2mqttVersion() {
     return new Promise((resolve, reject) => {
         const git = require('git-last-commit');
-        const packageJSON = require('../../package.json');
+        const packageJSON = require('../..' + '/package.json');
         const version = packageJSON.version;
 
         git.getLastCommit((err, commit) => {
@@ -60,8 +60,9 @@ async function getZigbee2mqttVersion() {
 
             if (err) {
                 try {
-                    commitHash = require('../../.hash.json').hash;
+                    commitHash = fs.readFileSync(path.join(__dirname, '..', '..', 'dist', '.hash'), 'utf-8');
                 } catch (error) {
+                    /* istanbul ignore next */
                     commitHash = 'unknown';
                 }
             } else {
@@ -75,7 +76,7 @@ async function getZigbee2mqttVersion() {
 
 async function getDependencyVersion(depend) {
     return new Promise((resolve, reject) => {
-        const packageJSON = require('../../node_modules/'+depend+'/package.json');
+        const packageJSON = require(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
         const version = packageJSON.version;
         resolve({version});
     });
@@ -211,6 +212,11 @@ function getKey(object, value, fallback, convertTo) {
     return fallback;
 }
 
+function toNetworkAddressHex(value) {
+    const hex = value.toString(16);
+    return `0x${'0'.repeat(4 - hex.length)}${hex}`;
+}
+
 function toSnakeCase(value) {
     if (typeof value === 'object') {
         value = {...value};
@@ -280,6 +286,7 @@ module.exports = {
     equalsPartial,
     getResponse,
     capitalize,
+    toNetworkAddressHex,
     parseJSON,
     getExternalConvertersDefinitions,
     validateFriendlyName,

lib/util/yaml.js

@@ -1,36 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-const equals = require('fast-deep-equal/es6');
-
-function read(file) {
-    try {
-        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-    } catch (error) {
-        if (error.name === 'YAMLException') {
-            error.file = file;
-        }
-
-        throw error;
-    }
-}
-
-function readIfExists(file, default_=null) {
-    return fs.existsSync(file) ? read(file) : default_;
-}
-
-function writeIfChanged(file, content) {
-    const before = readIfExists(file);
-    if (!equals(before, content)) {
-        fs.writeFileSync(file, yaml.safeDump(content));
-    }
-}
-
-function updateIfChanged(file, key, value) {
-    const content = read(file);
-    if (content[key] !== value) {
-        content[key] = value;
-        writeIfChanged(file, content);
-    }
-}
-
-module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/util/yaml.ts

@@ -0,0 +1,36 @@
+import yaml from 'js-yaml';
+import fs from 'fs';
+import equals from 'fast-deep-equal/es6';
+import 'source-map-support/register';
+
+export function read(file: string): KeyValue {
+    try {
+        /* eslint-disable-line */ // @ts-ignore
+        return yaml.load(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+export function readIfExists(file: string, default_?: KeyValue): KeyValue {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+export function writeIfChanged(file: string, content: KeyValue): void {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.dump(content));
+    }
+}
+
+export function updateIfChanged(file: string, key: string, value: KeyValue): void {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}

lib/zigbee.js

@@ -40,6 +40,7 @@ class Zigbee extends events.EventEmitter {
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
                 delay: settings.get().advanced.adapter_delay,
+                disableLED: settings.get().serial.disable_led,
             },
         };
 
@@ -59,16 +60,19 @@ class Zigbee extends events.EventEmitter {
             herdsmanSettings.network.panID = newPanID;
         }
 
+        let startResult;
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
             this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, logger);
-            await this.herdsman.start();
+            startResult = await this.herdsman.start();
         } catch (error) {
             logger.error(`Error while starting zigbee-herdsman`);
             throw error;
         }
 
         this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
+        this.herdsman.on('deviceNetworkAddressChanged', (data) =>
+            this.emit('event', 'deviceNetworkAddressChanged', data));
         this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
         this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
         this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
@@ -76,7 +80,7 @@ class Zigbee extends events.EventEmitter {
         this.herdsman.on('message', (data) => this.emit('event', 'message', data));
         this.herdsman.on('permitJoinChanged', (data) => this.emit('permitJoinChanged', data));
 
-        logger.info('zigbee-herdsman started');
+        logger.info(`zigbee-herdsman started (${startResult})`);
         logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
         logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
@@ -106,6 +110,8 @@ class Zigbee extends events.EventEmitter {
             await this.herdsman.setTransmitPower(transmitPower);
             logger.info(`Set transmit power to '${transmitPower}'`);
         }
+
+        return startResult;
     }
 
     async getCoordinatorVersion() {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.18.3",
+  "version": "1.21.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^14 || ^15"
+    "node": "^10 || ^12 || ^14 || ^15 || ^16"
   },
   "keywords": [
     "xiaomi",
@@ -20,9 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "test-with-coverage": "jest test --coverage",
-    "eslint": "node_modules/.bin/eslint lib/",
+    "build": "node_modules/.bin/tsc && node index.js writehash",
+    "build-watch": "node_modules/.bin/tsc --watch",
+    "eslint": "node_modules/.bin/eslint lib/ --max-warnings=0",
     "start": "node index.js",
+    "test-with-coverage": "jest test --coverage",
     "test": "jest test",
     "test-watch": "jest test --watch"
   },
@@ -33,35 +35,47 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^6.12.4",
-    "debounce": "^1.2.0",
+    "ajv": "^8.6.1",
+    "debounce": "^1.2.1",
     "deep-object-diff": "^1.1.0",
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.1.2",
     "git-last-commit": "^1.0.0",
-    "humanize-duration": "^3.23.1",
-    "js-yaml": "^3.14.0",
-    "json-stable-stringify-without-jsonify": "=1.0.1",
+    "humanize-duration": "^3.27.0",
+    "js-yaml": "^4.1.0",
+    "json-stable-stringify-without-jsonify": "^1.0.1",
     "mkdir-recursive": "^0.4.0",
-    "moment": "^2.27.0",
-    "mqtt": "4.2.1",
+    "moment": "^2.29.1",
+    "mqtt": "4.2.8",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
-    "semver": "^7.3.2",
+    "semver": "^7.3.5",
     "serve-static": "^1.14.1",
+    "source-map-support": "^0.5.19",
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
-    "ws": "^7.3.1",
-    "zigbee-herdsman": "0.13.92",
-    "zigbee-herdsman-converters": "14.0.131",
-    "zigbee2mqtt-frontend": "0.4.0"
+    "ws": "^7.5.2",
+    "zigbee-herdsman": "0.13.124",
+    "zigbee-herdsman-converters": "14.0.210",
+    "zigbee2mqtt-frontend": "0.5.18"
   },
   "devDependencies": {
-    "eslint": "*",
-    "eslint-config-google": "*",
-    "eslint-plugin-jest": "*",
-    "jest": "*",
-    "tmp": "*"
+    "@babel/core": "^7.14.6",
+    "@babel/preset-env": "^7.14.7",
+    "@babel/preset-typescript": "^7.14.5",
+    "@types/jest": "^26.0.24",
+    "@types/js-yaml": "^4.0.2",
+    "@types/object-assign-deep": "^0.4.0",
+    "@types/ws": "^7.4.6",
+    "@typescript-eslint/eslint-plugin": "^4.28.3",
+    "@typescript-eslint/parser": "^4.28.3",
+    "babel-jest": "^27.0.6",
+    "eslint": "^7.30.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-jest": "^24.3.6",
+    "jest": "^27.0.6",
+    "tmp": "^0.2.1",
+    "typescript": "^4.3.5"
   },
   "jest": {
     "coverageThreshold": {

test/availability.test.js

@@ -6,38 +6,41 @@ zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
-
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-
-const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const mocks = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Availability', () => {
     let controller;
+    let extension;
 
-    function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Availability');
+        await controller.enableDisableExtension(true, 'Availability');
+        extension = controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
         jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10);
         controller = new Controller(jest.fn(), jest.fn());
-        mocksClear.forEach((m) => m.mockClear());
         await controller.start();
         await flushPromises();
     });
 
-    afterEach(async () => {
-        await controller.stop();
-        await flushPromises();
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['advanced', 'availability_timeout'], 10);
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     })
 
     it('Should publish availabilty on startup', async () => {
@@ -153,7 +156,7 @@ describe('Availability', () => {
 
     it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
         MQTT.publish.mockClear();
-        getExtension().state = {};
+        extension.state = {};
         const payload = {device: zigbeeHerdsman.devices.bulb_color};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
@@ -199,11 +202,7 @@ describe('Availability', () => {
     it('Should not ping devices on blocklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -213,11 +212,7 @@ describe('Availability', () => {
     it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -227,11 +222,7 @@ describe('Availability', () => {
     it('Should not ping devices on blocklist by IEEE address', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -254,11 +245,7 @@ describe('Availability', () => {
     it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -268,11 +255,7 @@ describe('Availability', () => {
     it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -282,11 +265,7 @@ describe('Availability', () => {
     it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -295,13 +274,9 @@ describe('Availability', () => {
 
     it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        getExtension().state[device.ieeeAddr] = false;
+        extension.state[device.ieeeAddr] = false;
         settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(0);
@@ -311,7 +286,7 @@ describe('Availability', () => {
 
     it('Should not read when device has no modelID and reconnects', async () => {
         const device = zigbeeHerdsman.devices.nomodel;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -320,7 +295,7 @@ describe('Availability', () => {
 
     it('Should not read when device has is unsupported', async () => {
         const device = zigbeeHerdsman.devices.unsupported_router;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -382,13 +357,9 @@ describe('Availability', () => {
 
     it('Should republish existing state on MQTT connected', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        await controller.stop();
-        await flushPromises();
         MQTT.publish.mockClear();
-        controller = new Controller(jest.fn(), jest.fn());
-        getExtension().state[device.ieeeAddr] = false;
-        await controller.start();
-        await flushPromises();
+        extension.state[device.ieeeAddr] = false;
+        await extension.onMQTTConnected();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color/availability',
           'offline',

test/bind.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const stringify = require('json-stable-stringify-without-jsonify');
 jest.mock('debounce', () => jest.fn(fn => fn));
 const debounce = require('debounce');
@@ -12,7 +12,7 @@ const debounce = require('debounce');
 describe('Bind', () => {
     let controller;
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -23,22 +23,34 @@ describe('Bind', () => {
         }
     }
 
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Bind');
+        await controller.enableDisableExtension(true, 'Bind');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
         zigbeeHerdsman.groups.group_1.members = [];
         zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
         zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
         zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
         debounce.mockClear();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
         MQTT.publish.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Should bind to device and configure reporting', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
@@ -229,6 +241,52 @@ describe('Bind', () => {
         );
     });
 
+    it('Should unbind from group with skip_disable_reporting=true', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set to false, we don't expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(0);
+        endpoint.binds = originalBinds;
+    });
+
+    it('Should unbind from group with skip_disable_reporting=false', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: false}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set, we expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 65535, "minimumReportInterval": 5, "reportableChange": 1}])
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 65535, "minimumReportInterval": 0, "reportableChange": 0}])
+        endpoint.binds = originalBinds;
+    });
+
     it('Should bind to group by number', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;

test/configure.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const stringify = require('json-stable-stringify-without-jsonify');
 
@@ -12,13 +12,14 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Configure', () => {
     let controller;
+    let coordinatorEndpoint;
 
-    expectRemoteConfigured = () => {
+    const expectRemoteConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(2);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
 
         const endpoint2 = device.getEndpoint(2);
         expect(endpoint2.write).toHaveBeenCalledTimes(1);
@@ -26,29 +27,27 @@ describe('Configure', () => {
         expect(device.meta.configured).toBe(1);
     }
 
-    expectBulbConfigured = () => {
+    const expectBulbConfigured = () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint1 = device.getEndpoint(1);
-        console.log(endpoint1.read.mock.calls);
         expect(endpoint1.read).toHaveBeenCalledTimes(2);
         expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities']);
         expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorTempPhysicalMin', 'colorTempPhysicalMax' ]);
     }
 
-    expectBulbNotConfigured = () => {
+    const expectBulbNotConfigured = () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint1 = device.getEndpoint(1);
-        console.log(endpoint1.read.mock.calls);
         expect(endpoint1.read).toHaveBeenCalledTimes(0);
     }
 
-    expectRemoteNotConfigured = () => {
+    const expectRemoteNotConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(0);
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -57,18 +56,30 @@ describe('Configure', () => {
         }
     }
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Configure');
+        await controller.enableDisableExtension(true, 'Configure');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
-        mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
     });
 
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+     coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Should configure Router on startup', async () => {
         expectBulbConfigured();
     });
@@ -156,11 +167,11 @@ describe('Configure', () => {
     });
 
     it('Fail to configure via MQTT when device has no configure', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb_enddevice", transaction: 20}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "0x90fd9ffffe4b64ax", transaction: 20}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            stringify({"data":{"id": "bulb_enddevice"},"status":"error","error": "Device 'bulb_enddevice' cannot be configured","transaction":20}),
+            stringify({"data":{"id": "0x90fd9ffffe4b64ax"},"status":"error","error": "Device 'ZNLDP12LM' cannot be configured","transaction":20}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -180,9 +191,9 @@ describe('Configure', () => {
     });
 
     it('Legacy api: Should skip reconfigure when device does not require this', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x0017880104e45553');
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x90fd9ffffe4b64ax');
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb_enddevice', device does not require this.`)
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
     it('Should not configure when interviewing', async () => {

test/controller.test.js

@@ -6,7 +6,7 @@ const path = require('path');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const tmp = require('tmp');
 const mocksClear = [
     zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
@@ -20,6 +20,10 @@ describe('Controller', () => {
     let controller;
     let mockExit;
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
         mockExit = jest.fn();
@@ -30,9 +34,13 @@ describe('Controller', () => {
         data.writeDefaultState();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -132,14 +140,12 @@ describe('Controller', () => {
     });
 
     it('Log when MQTT client is unavailable', async () => {
-        jest.useFakeTimers();
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
         controller.mqtt.client.reconnecting = true;
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
         controller.mqtt.client.reconnecting = false;
     });
@@ -623,4 +629,13 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
     });
+
+    it('Should disable legacy options on new network start', async () => {
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        expect(settings.get().advanced.legacy_api).toBeTruthy();
+        zigbeeHerdsman.start.mockReturnValueOnce('reset');
+        await controller.start();
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().advanced.legacy_api).toBeFalsy();
+    });
 });

test/externalConverters.test.js

@@ -6,7 +6,7 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const path = require('path');
 const fs = require('fs');
 
@@ -41,20 +41,32 @@ jest.mock(
 describe('Loads external converters', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'ExternalConverters');
+        await controller.enableDisableExtension(true, 'ExternalConverters');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
         await flushPromises();
-        mocksClear.forEach((m) => m.mockClear());
+    });
+
+    beforeEach(async () => {
         data.writeDefaultConfiguration();
-        data.writeEmptyState();
         settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Does not load external converters', async () => {
         settings.set(['external_converters'], []);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
     });
 
@@ -62,9 +74,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter.js']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
             mock: true,
@@ -82,9 +92,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1,
@@ -110,9 +118,7 @@ describe('Loads external converters', () => {
 
     it('Loads external converters from package', async () => {
         settings.set(['external_converters'], ['mock-external-converter-module']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
             mock: true
@@ -121,9 +127,7 @@ describe('Loads external converters', () => {
 
     it('Loads multiple external converters from package', async () => {
         settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1

test/externalExtension.test.js

@@ -6,8 +6,7 @@ const path = require('path');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
-const tmp = require('tmp');
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [
     zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
     MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
@@ -20,12 +19,24 @@ const unlinkSyncSpy = jest.spyOn(fs, 'unlinkSync');
 
 describe('User extensions', () => {
     let controller;
-    let mockExit;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
 
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
-        mockExit = jest.fn();
-        controller = new Controller(jest.fn(), mockExit);
+        controller = new Controller(jest.fn(), jest.fn());
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
         settings.reRead();

test/frontend.test.js

@@ -5,7 +5,7 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 jest.spyOn(process, 'exit').mockImplementation(() => {});
 
@@ -70,6 +70,10 @@ jest.mock('ws', () => ({
 describe('Frontend', () => {
     let controller;
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
     beforeEach(async () => {
         mockWS.implementation.clients = [];
         data.writeDefaultConfiguration();
@@ -80,6 +84,10 @@ describe('Frontend', () => {
         zigbeeHerdsman.devices.bulb.linkquality = 10;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     afterEach(async() => {
         delete zigbeeHerdsman.devices.bulb.linkquality;
     });
@@ -120,14 +128,14 @@ describe('Frontend', () => {
         await mockWS.events.connection(mockWSClient.implementation);
 
         expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bridge/state', payload: 'online'}));
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255}}));
 
         // Message
         MQTT.publish.mockClear();
         mockWSClient.implementation.send.mockClear();
         mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
             stringify({state: 'ON', linkquality: null}),
@@ -140,7 +148,7 @@ describe('Frontend', () => {
         await flushPromises();
 
         // Received message on socket
-        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(4);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(1);
         expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', linkquality: null}}));
 
         // Shouldnt set when not ready
@@ -155,7 +163,7 @@ describe('Frontend', () => {
         settings.set(['advanced'], {last_seen: 'ISO_8601'});
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z"}}));
     });
 
     it('onReques/onUpgrade', async () => {

test/group.test.js

@@ -13,28 +13,42 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45724');
 
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const settings = require('../lib/util/settings');
 
 describe('Groups', () => {
     let controller;
 
-    beforeEach(() => {
-        data.writeEmptyState();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Groups');
+        await controller.enableDisableExtension(true, 'Groups');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(() => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
         settings.reRead();
         MQTT.publish.mockClear();
         zigbeeHerdsman.groups.gledopto_group.command.mockClear();
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+        controller.state.state = {};
     })
 
     it('Apply group updates add', async () => {
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
         zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -46,8 +60,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
@@ -58,8 +71,7 @@ describe('Groups', () => {
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
         logger.error.mockClear();
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(logger.error).toHaveBeenCalledWith(`Failed to remove 'bulb_color' from 'group_1'`);
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([endpoint]);
     });
@@ -70,32 +82,28 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
     it('Add non standard endpoint to group with name', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
     });
 
     it('Add non standard endpoint to group with number', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
     });
 
     it('Shouldnt crash on non-existing devices', async () => {
         logger.error.mockClear();
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
@@ -103,8 +111,7 @@ describe('Groups', () => {
     it('Should resolve device friendly names', async () => {
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'friendly_name'], 'bulb_friendly_name');
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb_friendly_name', 'bulb_color']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -117,8 +124,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -132,8 +138,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -146,8 +151,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -159,8 +163,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
@@ -175,8 +178,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -190,8 +192,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -204,8 +205,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -218,8 +218,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -232,8 +231,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -246,8 +244,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -261,8 +258,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -270,8 +266,7 @@ describe('Groups', () => {
     });
 
     it('Legacy api: Log when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
         await flushPromises();
@@ -279,8 +274,7 @@ describe('Groups', () => {
     });
 
     it('Legacy api: Log when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
         await flushPromises();
@@ -293,8 +287,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -310,18 +303,19 @@ describe('Groups', () => {
         const device1 = zigbeeHerdsman.devices.bulb_2;
         const device2 = zigbeeHerdsman.devices.bulb_color_2;
         const group = zigbeeHerdsman.groups.group_tradfri_remote;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(6);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/ha_discovery_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/switch_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -330,8 +324,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
@@ -348,8 +341,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
@@ -375,8 +367,7 @@ describe('Groups', () => {
 
     it('Should publish state of device with endpoint name', async () => {
         const group = zigbeeHerdsman.groups.gledopto_group;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/gledopto_group/set', stringify({state: 'ON'}));
@@ -390,8 +381,7 @@ describe('Groups', () => {
 
     it('Should publish state of group when specific state of specific endpoint is changed', async () => {
         const group = zigbeeHerdsman.groups.gledopto_group;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/GLEDOPTO_2ID/set', stringify({state_cct: 'ON'}));
@@ -408,8 +398,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
@@ -425,8 +414,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -447,8 +435,7 @@ describe('Groups', () => {
             '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
             '3': {friendly_name: 'group_3', retain: false, devices: []}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
@@ -470,8 +457,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -495,8 +481,7 @@ describe('Groups', () => {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
             '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -520,8 +505,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -547,8 +531,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
 
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF', color_temp: 200}));
@@ -559,9 +542,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 300}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode": "color_temp", "color":{"x":0.415211980162654,"y":0.395434886759171},"color_temp":300,"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode": "color_temp", "color":{"x":0.415211980162654,"y":0.395434886759171},"color_temp":300,"state":"ON"}), {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode": "color_temp", "color":{"x":0.415211980162654,"y":0.395434886759171},"color_temp":300,"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode": "color_temp","color_temp":300,"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -575,8 +558,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -597,8 +579,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({transaction: "123", group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -618,7 +599,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
+        await resetExtension();
         endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
         await flushPromises();
         MQTT.publish.mockClear();
@@ -640,8 +621,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
         await flushPromises();
@@ -660,8 +640,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
@@ -680,8 +659,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
@@ -703,8 +681,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -718,14 +695,33 @@ describe('Groups', () => {
         );
     });
 
+    it('Remove from group via MQTT keeping device reporting', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -745,8 +741,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
         await flushPromises();
@@ -766,8 +761,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
         await flushPromises();
@@ -787,8 +781,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
         await flushPromises();
@@ -808,8 +801,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
         await flushPromises();
@@ -824,8 +816,7 @@ describe('Groups', () => {
     });
 
     it('Error when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
@@ -839,8 +830,7 @@ describe('Groups', () => {
     });
 
     it('Error when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
@@ -852,4 +842,30 @@ describe('Groups', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
+
+    it('Should only include relevant properties when publishing member states', async () => {
+        const bulbColor = zigbeeHerdsman.devices.bulb_color;
+        const bulbColorTemp = zigbeeHerdsman.devices.bulb;
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(bulbColor.getEndpoint(1));
+        group.members.push(bulbColorTemp.getEndpoint(1));
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [bulbColor.ieeeAddr, bulbColorTemp.ieeeAddr]}});
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 50}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color: {x: 0.5, y: 0.3}}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":548}), {"retain": true, qos: 0}, expect.any(Function));
+    });
 });

test/homeassistant.test.js

@@ -3,7 +3,7 @@ const settings = require('../lib/util/settings');
 const stringify = require('json-stable-stringify-without-jsonify');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
 const fs = require('fs');
@@ -13,10 +13,11 @@ const HomeAssistant = require('../lib/extension/homeassistant');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 describe('HomeAssistant extension', () => {
+    let version;
+
     beforeEach(async () => {
-        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
-        this.version = `Zigbee2MQTT ${this.version.version}`;
-        jest.useRealTimers();
+        version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        version = `Zigbee2MQTT ${version.version}`;
         data.writeDefaultConfiguration();
         settings.reRead();
         data.writeEmptyState();
@@ -24,6 +25,14 @@ describe('HomeAssistant extension', () => {
         settings.set(['homeassistant'], true);
     });
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
@@ -44,16 +53,72 @@ describe('HomeAssistant extension', () => {
         expect(duplicated).toHaveLength(0);
     });
 
-    it('Should discover devices', async () => {
-        controller = new Controller(false);
+    it('Should discover devices and groups', async () => {
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
         await flushPromises();
 
+        payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version":version,
+            },
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version":version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "unique_id":"9_switch_zigbee2mqtt",
+            "value_template":"{{ value_json.state }}"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/1221051039810110150109113116116_9/switch/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -62,7 +127,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -79,6 +144,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -87,7 +153,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -104,6 +170,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -112,7 +179,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -129,6 +196,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'battery',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.battery }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -137,7 +205,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -153,7 +221,9 @@ describe('HomeAssistant extension', () => {
 
         payload = {
             'icon': 'mdi:signal',
+            'enabled_by_default': false,
             'unit_of_measurement': 'lqi',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.linkquality }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -162,7 +232,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -190,7 +260,7 @@ describe('HomeAssistant extension', () => {
                 "manufacturer":"Xiaomi",
                 "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
                 "name":"wall_switch_double",
-                "sw_version":this.version
+                "sw_version":version
             },
             "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
             "name":"wall_switch_double_left",
@@ -222,7 +292,7 @@ describe('HomeAssistant extension', () => {
                 "manufacturer":"Xiaomi",
                 "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
                 "name":"wall_switch_double",
-                "sw_version":this.version
+                "sw_version":version
             },
             "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
             "name":"wall_switch_double_right",
@@ -260,7 +330,7 @@ describe('HomeAssistant extension', () => {
                 "manufacturer":"IKEA",
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "name":"bulb",
-                "sw_version":this.version,
+                "sw_version":version,
             },
             "effect":true,
             "effect_list":[
@@ -295,7 +365,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -304,6 +374,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -312,7 +383,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -329,6 +400,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -337,7 +409,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -354,6 +426,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -362,7 +435,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -401,7 +474,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -410,6 +483,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -437,6 +511,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -444,7 +519,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'custom model',
                 'manufacturer': 'Not from Xiaomi',
             },
@@ -477,7 +552,7 @@ describe('HomeAssistant extension', () => {
             },
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -493,7 +568,7 @@ describe('HomeAssistant extension', () => {
               "manufacturer": "Xiaomi",
               "model": "Aqara single key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG04LM)",
               "name": "my_switch",
-              "sw_version": this.version
+              "sw_version": version
             },
             "json_attributes_topic": "zigbee2mqtt/my_switch",
             "name": "my_light_name_override",
@@ -520,7 +595,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         await flushPromises();
@@ -538,7 +613,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         await flushPromises();
@@ -549,7 +624,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover devices with fan', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -559,18 +634,18 @@ describe('HomeAssistant extension', () => {
             "state_topic":"zigbee2mqtt/fan",
             "state_value_template":"{{ value_json.fan_state }}",
             "command_topic":"zigbee2mqtt/fan/set/fan_state",
-            "speed_state_topic":"zigbee2mqtt/fan",
-            "speed_command_topic":"zigbee2mqtt/fan/set/fan_mode",
-            "speed_value_template":"{{ value_json.fan_mode }}",
-            "speeds":[
-               "off",
-               "low",
-               "medium",
-               "high",
-               "on",
-               "auto",
+            "percentage_state_topic":"zigbee2mqtt/fan",
+            "percentage_command_topic":"zigbee2mqtt/fan/set/fan_mode",
+            "percentage_value_template":"{{ {'off':0, 'low':1, 'medium':2, 'high':3, 'on':4}[value_json.fan_mode] | default('None') }}",
+            "percentage_command_template":"{{ {0:'off', 1:'low', 2:'medium', 3:'high', 4:'on'}[value] | default('') }}",
+            "preset_mode_state_topic":"zigbee2mqtt/fan",
+            "preset_mode_command_topic":"zigbee2mqtt/fan/set/fan_mode",
+            "preset_mode_value_template":"{{ value_json.fan_mode if value_json.fan_mode in ['smart'] else 'None' | default('None') }}",
+            "preset_modes":[
                "smart"
             ],
+            "speed_range_min":1,
+            "speed_range_max":4,
             "json_attributes_topic":"zigbee2mqtt/fan",
             "name":"fan",
             "unique_id":"0x0017880104e45548_fan_zigbee2mqtt",
@@ -579,7 +654,7 @@ describe('HomeAssistant extension', () => {
                   "zigbee2mqtt_0x0017880104e45548"
                ],
                "name":"fan",
-               "sw_version":this.version,
+               "sw_version":version,
                "model":"Universal wink enabled white ceiling fan premier remote control (99432)",
                "manufacturer":"Hampton Bay"
             },
@@ -595,7 +670,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover thermostat devices', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -615,7 +690,7 @@ describe('HomeAssistant extension', () => {
                "manufacturer":"TuYa",
                "model":"Radiator valve with thermostat (TS0601_thermostat)",
                "name":"TS0601_thermostat",
-               "sw_version":  this.version,
+               "sw_version":  version,
             },
             "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
             "hold_modes":[
@@ -655,7 +730,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover devices with cover_position', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -674,7 +749,7 @@ describe('HomeAssistant extension', () => {
             {
                 identifiers: [ 'zigbee2mqtt_0x0017880104e45551' ],
                 name: 'smart vent',
-                sw_version: this.version,
+                sw_version: version,
                 model: 'Smart vent (SV01)',
                 manufacturer: 'Keen Home'
             },
@@ -691,7 +766,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should discover devices with custom homeassistant_discovery_topic', async () => {
         settings.set(['advanced', 'homeassistant_discovery_topic'], 'my_custom_discovery_topic')
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -700,6 +775,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -708,7 +784,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -726,20 +802,20 @@ describe('HomeAssistant extension', () => {
     it('Should throw error when starting with attributes output', async () => {
         settings.set(['experimental', 'output'], 'attribute')
         expect(() => {
-            controller = new Controller(false);
-        }).toThrowError('Home Assitant integration is not possible with attribute output!');
+            const controller = new Controller(false);
+        }).toThrowError('Home Assistant integration is not possible with attribute output!');
     });
 
     it('Should warn when starting with cache_state false', async () => {
         settings.set(['advanced', 'cache_state'], false);
         logger.warn.mockClear();
-        controller = new Controller(false);
-        expect(logger.warn).toHaveBeenCalledWith("In order for HomeAssistant integration to work properly set `cache_state: true");
+        const controller = new Controller(false);
+        expect(logger.warn).toHaveBeenCalledWith("In order for Home Assistant integration to work properly set `cache_state: true");
     });
 
     it('Should set missing values to null', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/6987
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
@@ -757,8 +833,65 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should copy hue/saturtion to h/s if present', async () => {
+        const controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const data = {currentHue: 0, currentSaturation: 254}
+        const payload = {data, cluster: 'lightingColorCtrl', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"hue": 0, "saturation": 100, "h": 0, "s": 100}, "color_mode": "hs", "linkquality": null, "state": null}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should not copy hue/saturtion if properties are missing', async () => {
+        const controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const data = {currentX: 29991, currentY: 26872};
+        const payload = {data, cluster: 'lightingColorCtrl', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({"color": {"x": 0.4576,"y": 0.41}, "color_mode": "xy", "linkquality": null,"state": null}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should not copy hue/saturtion if color is missing', async () => {
+        const controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({"linkquality": null,"state": "ON"}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
     it('Shouldt discover when already discovered', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
@@ -772,7 +905,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover when not discovered yet', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         controller.extensions.find((e) => e.constructor.name === 'HomeAssistant').discovered = {};
@@ -785,6 +918,7 @@ describe('HomeAssistant extension', () => {
         const payloadHA = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -793,7 +927,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -809,7 +943,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt discover when device leaves', async () => {
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         controller.extensions.find((e) => e.constructor.name === 'HomeAssistant').discovered = {};
@@ -821,9 +955,8 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should send all status when home assistant comes online (default topic)', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         expect(MQTT.subscribe).toHaveBeenCalledWith('homeassistant/status');
         await flushPromises();
@@ -834,22 +967,21 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update":{"state":"idle"},"update_available":false}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null,"update":{"state":"idle"},"update_available":false}),
+            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
     });
 
     it('Should send all status when home assistant comes online', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(MQTT.subscribe).toHaveBeenCalledWith('hass/status');
@@ -860,22 +992,21 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false,"update":{"state":"idle"}}),
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"power_on_behavior":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null,"update":{"state":"idle"},"update_available":false}),
+            stringify({"action":null,"battery":null,"brightness":255,"linkquality":null}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
     });
 
     it('Shouldnt send all status when home assistant comes offline', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -887,9 +1018,8 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt send all status when home assistant comes online with different topic', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -902,7 +1032,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should discover devices with availability', async () => {
         settings.set(['advanced', 'availability_timeout'], 1)
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -911,6 +1041,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -919,7 +1050,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -935,7 +1066,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should clear discovery when device is removed', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -975,7 +1106,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should not clear discovery when unsupported device is removed', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -985,7 +1116,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should refresh discovery when device is renamed', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         await MQTT.events.message('homeassistant/device_automation/0x0017880104e45522/action_double/config', stringify({topic: 'zigbee2mqtt/weather_sensor/action'}));
@@ -997,6 +1128,7 @@ describe('HomeAssistant extension', () => {
         const payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
@@ -1005,7 +1137,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor_renamed',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -1039,7 +1171,7 @@ describe('HomeAssistant extension', () => {
                         "zigbee2mqtt_0x0017880104e45522"
                     ],
                     "name":"weather_sensor_renamed",
-                    "sw_version": this.version,
+                    "sw_version": version,
                     "model":"Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)",
                     "manufacturer":"Xiaomi"
                 }
@@ -1050,7 +1182,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1067,6 +1199,7 @@ describe('HomeAssistant extension', () => {
         const payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
@@ -1075,7 +1208,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor_renamed',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -1091,13 +1224,14 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover update_available sensor when device supports it', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const payload = {
             "payload_on":true,
             "payload_off":false,
             "value_template":"{{ value_json.update_available}}",
+            "enabled_by_default": false,
             "state_topic":"zigbee2mqtt/bulb",
             "json_attributes_topic":"zigbee2mqtt/bulb",
             "name":"bulb update available",
@@ -1107,7 +1241,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x000b57fffec6a5b2"
                 ],
                 "name":"bulb",
-                'sw_version': this.version,
+                'sw_version': version,
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "manufacturer":"IKEA"
             },
@@ -1123,7 +1257,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover trigger when click is published', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
 
@@ -1150,7 +1284,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x0017880104e45520"
                 ],
                 "name":"button",
-                "sw_version": this.version,
+                "sw_version": version,
                 "model":"Aqara wireless switch (WXKG11LM)",
                 "manufacturer":"Xiaomi"
             }
@@ -1174,7 +1308,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x0017880104e45520"
                 ],
                 "name":"button",
-                "sw_version": this.version,
+                "sw_version": version,
                 "model":"Aqara wireless switch (WXKG11LM)",
                 "manufacturer":"Xiaomi"
             }
@@ -1275,11 +1409,11 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
     });
 
-    it('Should not discover device_automtation when disabled', async () => {
+    it('Should not discover device_automation when disabled', async () => {
         settings.set(['device_options'], {
             homeassistant: {device_automation: null},
         })
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1303,7 +1437,7 @@ describe('HomeAssistant extension', () => {
             friendly_name: 'weather_sensor',
             retain: false,
         })
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
 
@@ -1316,7 +1450,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should disable Home Assistant legacy triggers', async () => {
         settings.set(['advanced', 'homeassistant_legacy_triggers'], false);
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
 
@@ -1344,7 +1478,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x0017880104e45520"
                 ],
                 "name":"button",
-                "sw_version": this.version,
+                "sw_version": version,
                 "model":"Aqara wireless switch (WXKG11LM)",
                 "manufacturer":"Xiaomi"
             }
@@ -1375,7 +1509,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should republish payload to postfix topic with lightWithPostfix config', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1388,7 +1522,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt crash in onPublishEntityState on group publish', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -1400,7 +1534,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should counter an action payload with an empty payload', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1424,7 +1558,7 @@ describe('HomeAssistant extension', () => {
     it('Load Home Assistant mapping from external converters', async () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         const ha = controller.extensions.find((e) => e.constructor.name === 'HomeAssistant');
         await controller.start();
         await flushPromises();
@@ -1443,10 +1577,37 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should clear outdated configs', async () => {
-        controller = new Controller(jest.fn(), jest.fn());
+        let controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
 
+        // Non-existing group -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/1221051039810110150109113116116_91231/light/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/light/1221051039810110150109113116116_91231/light/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Existing group -> dont clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/1221051039810110150109113116116_9/light/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Existing group with old topic structure (1.20.0) -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/9/light/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/light/9/light/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Existing group, non existing config ->  clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/1221051039810110150109113116116_9/switch/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/light/1221051039810110150109113116116_9/switch/config', null, {qos: 0, retain: true}, expect.any(Function));
+
         // Non-existing device -> clear
         MQTT.publish.mockClear();
         await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
@@ -1508,4 +1669,78 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', null, {qos: 0, retain: true}, expect.any(Function));
     });
+
+    it('Should not have Home Assistant legacy entity attributes when disabled', async () => {
+        settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+        const controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'weather_sensor_temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'weather_sensor',
+                'sw_version': version,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should rediscover group when device is added to it', async () => {
+        const controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'ha_discovery_group', device: 'wall_switch_double/left'}));
+        await flushPromises();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version":version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -7,14 +7,16 @@ const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 
 
 describe('Bridge legacy', () => {
     let controller;
+    let version;
 
     beforeAll(async () => {
-        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
+        jest.useFakeTimers();
+        version = await require('../../lib/util/utils').getZigbee2mqttVersion();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
     })
@@ -22,15 +24,18 @@ describe('Bridge legacy', () => {
     beforeEach(() => {
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeDefaultState();
         logger.info.mockClear();
         logger.warn.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should publish bridge configuration on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/config',
-          stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
+          stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -188,7 +193,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","optimistic":true,"retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","optimistic":true,"retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","optimistic":true,"retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","optimistic":true,"retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","optimistic":true,"retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group","optimistic":true},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","optimistic":true,"retain":false}],"type":"groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","retain":false},{"ID":9,"devices":["bulb_color_2","bulb_2","wall_switch_double/right"],"friendly_name":"ha_discovery_group"},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},{"ID":14,"devices":["power_plug","bulb_2"],"friendly_name":"switch_group","retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group"},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {
@@ -221,10 +226,10 @@ describe('Bridge legacy', () => {
 
     it('Should allow rename groups', async () => {
         MQTT.publish.mockClear();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", retain: false});
         MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
         await flushPromises();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", retain: false});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
             stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
@@ -270,7 +275,7 @@ describe('Bridge legacy', () => {
             {qos: 0, retain: false},
             expect.any(Function)
         );
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
     });
@@ -279,7 +284,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -294,7 +299,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -309,7 +314,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
         await flushPromises();
-        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
     });

test/legacy/report.test.js

@@ -13,13 +13,14 @@ zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
 const MQTT = require('../stub/mqtt');
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Report', () => {
     let controller;
+    let extension;
 
     function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
@@ -59,7 +60,7 @@ describe('Report', () => {
         }
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -69,23 +70,34 @@ describe('Report', () => {
         }
     }
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        settings.set(['advanced', 'report'], true);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        extension = controller.extensions.find((e) => e.constructor.name === 'Report');
+    });
+
     beforeEach(async () => {
+        extension.enabled = true;
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
-        settings.set(['advanced', 'report'], true);
         for (const device of Object.values(zigbeeHerdsman.devices)) {
             mockClear(device);
             delete device.meta.reporting;
         }
-
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
         mocksClear.forEach((m) => m.mockClear());
-        await flushPromises();
+        extension.queue = new Set();
+        extension.failed = new Set();
+        await extension.onZigbeeStarted();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should configure reporting on startup', async () => {
+        await extension.onZigbeeStarted();
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         expectOnOffBrightnessColorReport(endpoint, true);
@@ -96,10 +108,8 @@ describe('Report', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         delete device.meta.report;
-        settings.set(['advanced', 'report'], false);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        extension.enabled = false;
+        await extension.onZigbeeStarted();
         expect(device.meta.reporting).toBe(undefined);
         expect(endpoint.bind).toHaveBeenCalledTimes(0);
     });
@@ -108,11 +118,9 @@ describe('Report', () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         device.meta.reporting = 1;
         const endpoint = device.getEndpoint(1);
-        settings.set(['advanced', 'report'], false);
+        extension.enabled = false;
         mockClear(device);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await extension.onZigbeeStarted();
         expectOnOffBrightnessColorReportDisabled(endpoint, true);
     });
 
@@ -131,7 +139,6 @@ describe('Report', () => {
     });
 
     it('Should not configure reporting when still configuring', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         endpoint.bind.mockImplementationOnce(async () => await wait(1000));
@@ -143,8 +150,6 @@ describe('Report', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
-        jest.useRealTimers();
     });
 
     it('Should not mark as configured when reporting setup fails', async () => {

test/lib/flushPromises.js

@@ -0,0 +1,2 @@
+const globalSetImmediate = setImmediate;
+module.exports = () => new Promise(globalSetImmediate);

test/networkMap.test.js

@@ -16,9 +16,7 @@ zigbeeHerdsman.returnDevices.push(external_converter_device.ieeeAddr)
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
-Date.now = jest.fn()
-Date.now.mockReturnValue(10000);
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 const setTimeoutNative = setTimeout;
 
@@ -26,6 +24,9 @@ describe('Networkmap', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
+        Date.now = jest.fn()
+        Date.now.mockReturnValue(10000);
         data.writeDefaultConfiguration();
         settings.reRead();
         data.writeEmptyState();
@@ -50,6 +51,10 @@ describe('Networkmap', () => {
         global.setTimeout = setTimeoutNative;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     function mock() {
         /**
          * Topology
@@ -97,7 +102,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -137,18 +142,18 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|0 seconds ago}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
-              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
-              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0x0000)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (0x9db1)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
+              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 0x198c)"]
+              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (0x9dcf)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (0x198a)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (0x1988)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (0x198c)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
-              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (1114)|external external (external_converter_device)|9 seconds ago}"];
+              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (0x045a)|external external (external_converter_device)|9 seconds ago}"];
               "0x0017880104e45511" -> "0x00124b00120144ae" [penwidth=1, weight=0, color="#994444", label="92"]
             }`;
 
@@ -180,7 +185,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45511 [
         0x0017880104e45511
         ---
-        0x0017880104e45511 (1114)
+        0x0017880104e45511 (0x045a)
         ---
         external external (external_converter_device)
         ---
@@ -190,7 +195,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45525 [
         0x0017880104e45525
         ---
-        0x0017880104e45525 (6536) failed: lqi,routingTable
+        0x0017880104e45525 (0x1988) failed: lqi,routingTable
         ---
         Boef notSupportedModelID
         ---
@@ -200,7 +205,7 @@ describe('Networkmap', () => {
         card 0x000b57fffec6a5b2 [
         bulb
         ---
-        0x000b57fffec6a5b2 (40369)
+        0x000b57fffec6a5b2 (0x9db1)
         ---
         IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
         ---
@@ -210,7 +215,7 @@ describe('Networkmap', () => {
         card 0x000b57fffec6a5b3 [
         bulb_color
         ---
-        0x000b57fffec6a5b3 (40399)
+        0x000b57fffec6a5b3 (0x9dcf)
         ---
         Philips Hue Go (7146060PH)
         ---
@@ -220,7 +225,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45521 [
         button_double_key
         ---
-        0x0017880104e45521 (6538)
+        0x0017880104e45521 (0x198a)
         ---
         Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)
         ---
@@ -230,7 +235,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45559 [
         cc2530_router
         ---
-        0x0017880104e45559 (6540)
+        0x0017880104e45559 (0x198c)
         ---
         Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
         ---
@@ -240,7 +245,7 @@ describe('Networkmap', () => {
         card 0x00124b00120144ae [
         Coordinator
         ---
-        0x00124b00120144ae (0)
+        0x00124b00120144ae (0x0000)
         ---
         0 seconds ago
         ]
@@ -271,7 +276,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
         expect(actual).toStrictEqual(expected);
     });

test/onEvent.test.js

@@ -6,7 +6,7 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45560');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
@@ -22,14 +22,24 @@ describe('On event', () => {
     const device = zigbeeHerdsman.devices.LIVOLO;
 
     beforeEach(async () => {
+        jest.useFakeTimers();
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         zigbeeHerdsmanConverters.onEvent.mockClear();
-        await flushPromises();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should call with start event', async () => {

test/otaUpdate.test.js

@@ -4,26 +4,44 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('OTA update', () => {
     let controller;
 
-    mockClear = (mapped) => {
+    const mockClear = (mapped) => {
         mapped.ota.updateToLatest = jest.fn();
         mapped.ota.isUpdateAvailable = jest.fn();
     }
 
-    beforeEach(async () => {
+    beforeAll(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
+        data.writeDefaultConfiguration();
         settings.set(['advanced', 'ikea_ota_use_test_url'], true);
-        data.writeEmptyState();
+        settings.reRead();
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
+    });
+
+    afterEach(async () => {
+        settings.set(['ota', 'disable_automatic_update_check'], false);
+        jest.runOnlyPendingTimers();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(async () => {
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        extension.inProgress = new Set();
+        controller.state.state = {};
         MQTT.publish.mockClear();
     });
 
@@ -176,7 +194,6 @@ describe('OTA update', () => {
     });
 
     it('Should refuse to check/update when already in progress', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -189,7 +206,7 @@ describe('OTA update', () => {
         MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
@@ -201,7 +218,6 @@ describe('OTA update', () => {
     it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
-        let count = 0;
         endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
 
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
@@ -217,6 +233,7 @@ describe('OTA update', () => {
 
     it('Should check for update when device requests it', async () => {
         const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
         const data = {imageType: 12382};
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -236,8 +253,6 @@ describe('OTA update', () => {
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
 
-        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
-        extension.lastChecked = {};
         logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
         await zigbeeHerdsman.events.message(payload);
@@ -250,6 +265,28 @@ describe('OTA update', () => {
         );
     });
 
+    it('Should check for update when device requests it and it is not available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Should not check for update when device requests it and disable_automatic_update_check is set to true', async () => {
         settings.set(['ota', 'disable_automatic_update_check'], true);
         const device = zigbeeHerdsman.devices.bulb;

test/publish.test.js

@@ -6,7 +6,7 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
@@ -27,6 +27,7 @@ describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
         data.writeEmptyState();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
@@ -34,8 +35,6 @@ describe('Publish', () => {
     });
 
     beforeEach(async () => {
-        jest.useRealTimers();
-        await flushPromises();
         data.writeDefaultConfiguration();
         controller.state.state = {};
         settings.reRead();
@@ -54,6 +53,11 @@ describe('Publish', () => {
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
+    afterAll(async () => {
+        jest.runOnlyPendingTimers();
+        jest.useRealTimers();
+    });
+
     it('Should publish messages to zigbee devices', async () => {
         const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
@@ -139,7 +143,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 222, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 222, color: {x: 0.360786471097048, y: 0.363543544699483}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 222});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -152,7 +156,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 500, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 500, color: {x: 0.526676280311873, y: 0.41329727450763}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 500});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -176,10 +180,10 @@ describe('Publish', () => {
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_right: 'OFF'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state_right: 'OFF'});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should publish messages to zigbee devices to non default-ep with state_[EP]', async () => {
@@ -219,7 +223,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color_temp: 62, color: {x: 100, y: 50}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -231,13 +235,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, color_temp: 62, state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON'}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy and brightness', async () => {
@@ -248,13 +251,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', color_temp: 62, brightness: 20});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', brightness: 20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state on', async () => {
@@ -265,13 +267,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, state: 'ON', color_temp: 62, brightness: 20, color_mode: 'xy'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color: {x: 100, y: 50}, state: 'ON', brightness: 20, color_mode: 'xy'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state off', async () => {
@@ -282,40 +284,40 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, color_temp: 62});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'OFF'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with color rgb object', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}, color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y:  0.6585}, color_mode: 'xy'});
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with color rgb string', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}, color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y:  0.6585}, color_mode: 'xy'});
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with brightness', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON', brightness: '50'}));
@@ -387,7 +389,7 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 24248, colory: 18350, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.37, y: 0.28}, color_temp: 249, color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.37, y: 0.28}, color_mode: 'xy'});
     });
 
     it('Should publish messages to groups color temperature', async () => {
@@ -398,15 +400,15 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100, color_mode: 'color_temp'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 100, color_mode: 'color_temp'});
     });
 
     it('Should create and publish to group which is in configuration.yaml but not in zigbee-herdsman', async () => {
         delete zigbeeHerdsman.groups.group_2;
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(8);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(9);
         await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'ON'}));
         await flushPromises();
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(9);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(10);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
     });
@@ -422,6 +424,25 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Shouldnt publish new brightness state when filtered_optimistic is used', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        settings.set(['devices', device.ieeeAddr, 'filtered_optimistic'], ["brightness"]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {"level": 200, "transtime": 0}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+    });
+
+    it('Shouldnt publish new state when optimistic = false for group', async () => {
+        settings.set(['groups', '2', 'optimistic'], false);
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
     it('Should handle non-valid topics', async () => {
         await MQTT.events.message('zigbee2mqtt1/bulb_color/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -551,10 +572,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y: 0.6584}, color_mode: "xy"});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y: 0.6585}, color_mode: "xy"});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -665,20 +686,19 @@ describe('Publish', () => {
     });
 
     it('Should read after write when enabled', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'retrieve_state'], true);
         const endpoint = device.getEndpoint(1);
         const payload = {'state': 'ON', 'color': {'x': 0.701, 'y': 0.299}};
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify(payload));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColor", {"colorx": 45940, "colory": 19595, "transtime": 0}, {}]);
         expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read.mock.calls[0]).toEqual(["genOnOff", ["onOff"]]);
-        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["currentX", "currentY"]]);
+        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["colorMode", "currentX", "currentY", "colorTemperature"]]);
     });
 
     it('Should also use on/off cluster when controlling group with switch', async () => {
@@ -699,7 +719,7 @@ describe('Publish', () => {
 
         MQTT.publish.mockClear();
         group.command.mockClear();
-        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF', brightness: 100}));
+        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
@@ -910,7 +930,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891, "saturation": 199, "transtime": 0,}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44877, "saturation": 200, "transtime": 0,}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}, "color_mode": "hs"});
@@ -1074,7 +1094,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100, color_mode: 'color_temp'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', color_temp: 100, color_mode: 'color_temp'});
     });
 
     it('Home Assistant: should set state when color temperature is also set and device is off', async () => {
@@ -1089,11 +1109,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100, color_mode: 'color_temp'});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON', color_temp: 100, color_mode: 'color_temp'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Home Assistant: should not set state when color is also set', async () => {
@@ -1109,7 +1131,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "moveToColor", {colorx: 26869, colory: 16384, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.41, y: 0.25}, color_temp: 150, state: 'ON', color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.41, y: 0.25}, state: 'ON', color_mode: 'xy'});
     });
 
     it('Should publish correct state on toggle command to zigbee bulb', async () => {
@@ -1259,19 +1281,17 @@ describe('Publish', () => {
 
     it('Should publish separate genOnOff to GL-S-007ZS when setting state and brightness as bulb doesnt turn on with moveToLevelWithOnOff', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/2757
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices['GL-S-007ZS'];
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/GL-S-007ZS/set', stringify({state: 'ON', brightness: 20}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual([ 'genOnOff', 'on', {}, {} ]);
         expect(endpoint.command.mock.calls[1]).toEqual([ 'genLevelCtrl', 'moveToLevelWithOnOff', { level: 20, transtime: 0 }, {} ]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/GL-S-007ZS', stringify({"state":"ON","brightness":20}), { qos: 0, retain: false }, expect.any(Function)]);
-        jest.useRealTimers();
     });
 
     it('Should log as error when setting property with no defined converter', async () => {
@@ -1381,30 +1401,70 @@ describe('Publish', () => {
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith('genScenes', 'recall', { groupid: 15071, sceneid: 1 }, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledTimes(8);
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
             'zigbee2mqtt/group_tradfri_remote',
-            stringify({"brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON","color_mode": "color_temp"}),
+            stringify({"brightness":50,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(2,
             'zigbee2mqtt/bulb_color_2',
-            stringify({"color_mode": "color_temp", "brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"color_mode": "color_temp", "brightness":50,"color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(3,
             'zigbee2mqtt/group_tradfri_remote',
-            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON","color_mode": "color_temp"}),
+            stringify({"brightness":100,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(4,
             'zigbee2mqtt/bulb_2',
-            stringify({"brightness":100,"state":"ON"}),
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(5,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":50,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(6,
             'zigbee2mqtt/group_with_tradfri',
-            stringify({"brightness":100,"state":"ON"}),
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(7,
+            'zigbee2mqtt/switch_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(8,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should sync colors', async () => {
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color_temp: 100}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {x: 0.1, y: 0.5}}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color_temp: 300}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color_mode":"color_temp","color_temp":100}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"x":0.1,"y":0.5},"color_mode":"xy","color_temp":79}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(3,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"x":0.4152,"y":0.3954},"color_mode":"color_temp","color_temp":300}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });

test/receive.test.js

@@ -5,24 +5,31 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Receive', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should handle a zigbee message', async () => {
         const device = zigbeeHerdsman.devices.WXKG11LM;
         device.linkquality = 10;
@@ -100,7 +107,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce messages', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         const data1 = {measuredValue: 8}
@@ -115,7 +121,7 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
@@ -124,7 +130,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce and retain messages when set via device_options', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['device_options', 'debounce'], 0.1);
         settings.set(['device_options', 'retain'], true);
@@ -141,7 +146,7 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
@@ -150,7 +155,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         settings.set(['devices', device.ieeeAddr, 'debounce_ignore'], ['temperature']);
@@ -166,7 +170,7 @@ describe('Receive', () => {
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2});
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03});
@@ -174,13 +178,12 @@ describe('Receive', () => {
 
     it('Shouldnt republish old state', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/3572
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON'}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
@@ -327,6 +330,19 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should publish last_seen for unhandled messages', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genRssiLocation', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('number')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
     it('Should publish last_seen epoch', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'epoch');

test/settings.test.js

@@ -2,7 +2,7 @@ require('./stub/logger');
 require('./stub/data');
 const data = require('../lib/util/data.js');
 const utils = require('../lib/util/utils.js');
-const settings = require('../lib/util/settings.js');
+const settings = require('../lib/util/settings.ts');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
@@ -20,12 +20,12 @@ const minimalConfig = {
 
 describe('Settings', () => {
     const write = (file, json, reread=true) => {
-        fs.writeFileSync(file, yaml.safeDump(json))
+        fs.writeFileSync(file, yaml.dump(json))
         if (reread) {
             settings.reRead();
         }
     };
-    const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const read = (file) => yaml.load(fs.readFileSync(file, 'utf8'));
     const remove = (file) => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
@@ -47,7 +47,7 @@ describe('Settings', () => {
     it('Should return default settings', () => {
         write(configurationFile, {});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expect(s).toStrictEqual(expected);
@@ -56,7 +56,7 @@ describe('Settings', () => {
     it('Should return settings', () => {
         write(configurationFile, {permit_join: true});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.permit_join = true;
@@ -73,7 +73,7 @@ describe('Settings', () => {
 
         write(configurationFile, {});
         const s = settings.get();
-        const expected = objectAssignDeep.noMutate({}, settings._getDefaults());
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.serial.disable_led = true;
@@ -163,7 +163,7 @@ describe('Settings', () => {
         expect(settings.get().mqtt).toStrictEqual(expected);
         expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
 
-        settings._write();
+        settings.testing.write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
         expect(read(secretFile)).toStrictEqual(contentSecret);
 
@@ -352,7 +352,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -378,7 +377,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -420,7 +418,6 @@ describe('Settings', () => {
 
         const group = settings.getGroup('1');
         const expectedGroup = {
-            optimistic: true,
             ID: 1,
             friendlyName: '123',
             friendly_name: '123',
@@ -496,7 +493,7 @@ describe('Settings', () => {
         settings.addGroup('test123', 123);
         expect(() => {
             settings.addGroup('test_id_123', 123);
-        }).toThrow(new Error("group id '123' is already in use"));
+        }).toThrow(new Error("Group ID '123' is already in use"));
         const expected = {
             '123': {
                 friendly_name: 'test123',
@@ -679,7 +676,7 @@ describe('Settings', () => {
 
         settings.reRead();
 
-        const error = `advanced should be object`;
+        const error = `advanced must be object`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
@@ -705,13 +702,13 @@ describe('Settings', () => {
              \t wrong
         `)
 
-        settings._clear();
+        settings.testing.clear();
         const error = `Your YAML file: '${configurationFile}' is invalid (use https://jsonformatter.org/yaml-validator to find and fix the issue)`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should throw error when yaml file does not exist', () => {
-        settings._clear();
+        settings.testing.clear();
         const error = `ENOENT: no such file or directory, open '${configurationFile}'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });

test/stub/data.js

@@ -202,12 +202,16 @@ function writeDefaultConfiguration() {
             '14': {
                 friendly_name: 'switch_group',
                 retain: false,
-                devices: ['power_plug'],
+                devices: ['power_plug', 'bulb_2'],
             },
             '21': {
                 friendly_name: 'gledopto_group',
                 devices: ['GLEDOPTO_2ID/cct'],
-            }
+            },
+            '9': {
+                friendly_name: 'ha_discovery_group',
+                devices: ['bulb_color_2', 'bulb_2', 'wall_switch_double/right']
+            },
         },
         external_converters: [],
     };
@@ -229,20 +233,24 @@ function stateExists() {
     return fs.existsSync(stateFile);
 }
 
-function writeDefaultState() {
-    const state = {
-        "0x000b57fffec6a5b2": {
-            "state": "ON",
-            "brightness": 50,
-            "color_temp": 370,
-            "linkquality": 99,
-        },
-        "0x0017880104e45517": {
-            "brightness": 255
-        },
-    }
+const defaultState = {
+    "0x000b57fffec6a5b2": {
+        "state": "ON",
+        "brightness": 50,
+        "color_temp": 370,
+        "linkquality": 99,
+    },
+    "0x0017880104e45517": {
+        "brightness": 255
+    },
+}
 
-    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(state));
+function getDefaultState() {
+    return defaultState;
+}
+
+function writeDefaultState() {
+    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(defaultState));
 }
 
 jest.mock('../../lib/util/data', () => ({
@@ -261,4 +269,5 @@ module.exports = {
     removeState,
     writeEmptyState,
     stateExists,
+    getDefaultState,
 };

test/stub/zigbeeHerdsman.js

@@ -132,6 +132,7 @@ const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new End
 const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
 const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
+const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
 
 const groups = {
     'group_1': new Group(1, []),
@@ -139,9 +140,10 @@ const groups = {
     'group/with/slashes': new Group(99, []),
     'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
     'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
-    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0]]),
+    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0], bulb_2.endpoints[0]]),
     'gledopto_group': new Group(21, [GLEDOPTO_2ID.endpoints[3]]),
     'default_bind_group': new Group(901, []),
+    'ha_discovery_group': new Group(9, [bulb_color_2.endpoints[0], bulb_2.endpoints[0], QBKG03LM.endpoints[1]]),
 }
 
 const devices = {
@@ -162,7 +164,7 @@ const devices = {
     'ZNCZ02LM': ZNCZ02LM,
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
     'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
-    'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
+    'QBKG03LM':QBKG03LM,
     'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO_2ID': GLEDOPTO_2ID,

test/utils.test.js

@@ -39,7 +39,7 @@ describe('Utils', () => {
         expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "123", "version": version});
 
         mockReturnValue = [true, null]
-        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
+        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": expect.any(String), "version": version});
     })
 
     it('Check dependency version', async () => {

tsconfig.json

@@ -0,0 +1,28 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "esModuleInterop": true,
+        "target": "es2018",
+        "lib": ["es2018"],
+        "noImplicitAny": true,
+        "noImplicitThis": true,
+        "moduleResolution": "node",
+        "declaration": true,
+        "declarationMap": true,
+        "outDir": "dist",
+        "baseUrl": ".",
+        "allowJs": true,
+        "rootDir": "lib",
+        "inlineSourceMap": true,
+        "resolveJsonModule": true,
+        "paths": {
+            "*": [
+                "node_modules/*",
+                "lib/types/*"
+            ]
+        },
+    },
+    "include": [
+        "lib/**/*"
+    ],
+}