1.21.1

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -1,5 +1,5 @@
 .eslintignore
-.eslintrc.json
+.eslintrc.js
 .git
 .github
 .gitignore
@@ -13,4 +13,5 @@ images
 node_modules
 scripts
 test
+coverage
 update.sh

.eslintrc.js

@@ -0,0 +1,45 @@
+
+module.exports = {
+    'env': {
+        'jest/globals': true,
+        'es6': true,
+        'node': true,
+    },
+    'extends': ['eslint:recommended', 'google', 'plugin:jest/recommended', 'plugin:jest/style'],
+    'parserOptions': {
+        'ecmaVersion': 2018,
+        'sourceType': 'module',
+    },
+    'rules': {
+        'require-jsdoc': 'off',
+        'indent': ['error', 4],
+        'max-len': ['error', {'code': 120}],
+        'no-prototype-builtins': 'off',
+    },
+    'plugins': [
+        'jest',
+    ],
+    'overrides': [{
+        files: ['*.ts'],
+        parser: '@typescript-eslint/parser',
+        plugins: ['@typescript-eslint'],
+        extends: ['plugin:@typescript-eslint/recommended'],
+        parserOptions: {
+            project: './tsconfig.json',
+        },
+        rules: {
+            '@typescript-eslint/await-thenable': 'error',
+            '@typescript-eslint/ban-ts-ignore': 'off',
+            '@typescript-eslint/explicit-function-return-type': 'error',
+            '@typescript-eslint/no-empty-function': 'off',
+            '@typescript-eslint/no-explicit-any': 'error',
+            '@typescript-eslint/no-unused-vars': 'error',
+            '@typescript-eslint/semi': ['error'],
+            'array-bracket-spacing': ['error', 'never'],
+            'indent': ['error', 4],
+            'max-len': ['error', {'code': 120}],
+            'no-return-await': 'error',
+            'object-curly-spacing': ['error', 'never'],
+        },
+    }],
+};

.eslintrc.json

@@ -1,21 +0,0 @@
-{
-    "env": {
-        "jest/globals": true,
-        "es6": true,
-        "node": true
-    },
-    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
-    "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module"
-    },
-    "rules": {
-        "require-jsdoc": "off",
-        "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }],
-        "no-prototype-builtins": "off"
-    },
-    "plugins": [
-        "jest"
-    ]
-}

.github/workflows/ci.yml

@@ -24,6 +24,8 @@ jobs:
     - name: Install dependencies
       if: steps.cache-node-modules.outputs.cache-hit != 'true'
       run: npm ci
+    - name: Build
+      run: npm run build
     - name: Test
       run: npm run test-with-coverage
     - name: Lint

.github/workflows/update_deps.yml

@@ -15,9 +15,9 @@ jobs:
     - uses: actions/setup-node@v2-beta
       with:
         node-version: 12
-    - run: npm install
-    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
-    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+    - run: rm -rf npm-shrinkwrap.json
+    - run: npx npm-check-updates -u
+    - run: npm install && npm shrinkwrap --dev
     - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies

.gitignore

@@ -11,6 +11,9 @@ pids
 *.seed
 *.pid.lock
 
+# Compiled source
+dist/*
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 

.npmignore

@@ -7,6 +7,7 @@ coverage
 .jenkins.yml
 .codeclimate.yml
 .github
+babel.config.js
 
 #linters
 .jscsrc
@@ -14,11 +15,13 @@ coverage
 .eslintrc*
 .dockerignore
 .eslintignore
+.gitignore
 
 #editor settings
 .vscode
 
 #src
+lib
 docs
 docker
 images

README.md

@@ -69,6 +69,9 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 ### Internal Architecture
 Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
 
+### Developing
+Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

babel.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+};

docker/Dockerfile

@@ -5,20 +5,24 @@ RUN apk add --no-cache tzdata eudev tini
 
 COPY package.json ./
 
-# Dependencies
-FROM base as dependencies
+# Dependencies and build
+FROM base as dependencies_and_build
 
-COPY npm-shrinkwrap.json ./
+COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY lib ./lib
 
 RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
-    npm ci --production && \
+    npm ci --no-audit --no-optional --no-update-notifier && \
+    npm run build && \
+    rm -rf node_modules && \
+    npm ci --production --no-audit --no-optional --no-update-notifier && \
     apk del .buildtools
 
 # Release
 FROM base as release
 
-COPY --from=dependencies /app/node_modules ./node_modules
-COPY lib ./lib
+COPY --from=dependencies_and_build /app/node_modules ./node_modules
+COPY --from=dependencies_and_build /app/dist ./dist
 COPY LICENSE index.js data/configuration.yaml ./
 
 COPY docker/docker-entrypoint.sh /usr/local/bin/
@@ -27,7 +31,7 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 RUN mkdir /app/data
 
 ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+RUN echo "$COMMIT" > dist/.hash
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD [ "/sbin/tini", "--", "node", "index.js"]

docker/README.md

@@ -1 +0,0 @@
-qemu-aarch64-static and qemu-arm-static are taken from https://github.com/resin-io/qemu/releases/tag/v2.9.0%2Bresin1

index.js

@@ -1,10 +1,17 @@
 const semver = require('semver');
 const engines = require('./package.json').engines;
 const indexJsRestart = 'indexjs.restart';
+const fs = require('fs');
+const path = require('path');
+const {exec} = require('child_process');
+const rimraf = require('rimraf');
+require('source-map-support').install();
 
 let controller;
 let stopping = false;
 
+const hashFile = path.join(__dirname, 'dist', '.hash');
+
 async function restart() {
     await stop(indexJsRestart);
     await start();
@@ -16,14 +23,59 @@ async function exit(code, reason) {
     }
 }
 
+async function currentHash() {
+    const git = require('git-last-commit');
+    return new Promise((resolve) => {
+        git.getLastCommit((err, commit) => {
+            if (err) resolve('unknown');
+            else resolve(commit.shortHash);
+        });
+    });
+}
+
+async function writeHash() {
+    const hash = await currentHash();
+    fs.writeFileSync(hashFile, hash);
+}
+
+async function build(reason) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
+        rimraf.sync('dist');
+        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+            if (err) {
+                process.stdout.write(', failed\n');
+                reject(err);
+            } else {
+                process.stdout.write(', finished\n');
+                resolve();
+            }
+        });
+    });
+}
+
+async function checkDist() {
+    if (!fs.existsSync(hashFile)) {
+        await build('initial build');
+    }
+
+    const distHash = fs.readFileSync(hashFile, 'utf-8');
+    const hash = await currentHash();
+    if (hash !== 'unknown' && distHash !== hash) {
+        await build('hash changed');
+    }
+}
+
 async function start() {
+    await checkDist();
+
     const version = engines.node;
     if (!semver.satisfies(process.version, version)) {
         console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
     }
 
     // Validate settings
-    const settings = require('./lib/util/settings');
+    const settings = require('./dist/util/settings');
     settings.reRead();
     const errors = settings.validate();
     if (errors.length > 0) {
@@ -38,7 +90,7 @@ async function start() {
         exit(1);
     }
 
-    const Controller = require('./lib/controller');
+    const Controller = require('./dist/controller');
     controller = new Controller(restart, exit);
     await controller.start();
 }
@@ -54,7 +106,10 @@ async function handleQuit() {
     }
 }
 
-process.on('SIGINT', handleQuit);
-process.on('SIGTERM', handleQuit);
-
-start();
+if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+    writeHash();
+} else {
+    process.on('SIGINT', handleQuit);
+    process.on('SIGTERM', handleQuit);
+    start();
+}

lib/controller.js

@@ -21,6 +21,7 @@ const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
 const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
 const ExtensionAvailability = require('./extension/availability');
+const ExtensionAvailabilityNew = require('./extension/availabilityNew');
 const ExtensionBind = require('./extension/bind');
 const ExtensionReport = require('./extension/legacy/report');
 const ExtensionOnEvent = require('./extension/onEvent');
@@ -32,7 +33,7 @@ const AllExtensions = [
     ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
     ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
     ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
-    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension, ExtensionAvailabilityNew,
 ];
 
 class Controller {
@@ -50,9 +51,10 @@ class Controller {
         this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
-        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+            this.enableDisableExtension, this.restartCallback, this.addExtension];
         this.extensions = [
-            new ExtensionBridge(...args, this.enableDisableExtension, this.restartCallback),
+            new ExtensionBridge(...args),
             new ExtensionPublish(...args),
             new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
@@ -86,22 +88,26 @@ class Controller {
             this.extensions.push(new ExtensionSoftReset(...args));
         }
 
-        if (settings.get().advanced.availability_timeout) {
+        if (settings.get().experimental.availability_new) {
+            this.extensions.push(new ExtensionAvailabilityNew(...args));
+        } else if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
-        this.extensions.push(new ExtensionExternalExtension(...args, this.addExtension, this.enableDisableExtension));
+
+        this.extensions.push(new ExtensionExternalExtension(...args));
     }
 
     async start() {
         this.state.start();
         logger.logOutput();
 
-        const info = await utils.getZigbee2mqttVersion();
+        const info = await utils.getZigbee2MQTTVersion();
         logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
         // Start zigbee
+        let startResult;
         try {
-            await this.zigbee.start();
+            startResult = await this.zigbee.start();
             this.callExtensionMethod('onZigbeeStarted', []);
             this.zigbee.on('event', this.onZigbeeEvent.bind(this));
             this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
@@ -114,6 +120,14 @@ class Controller {
             this.exitCallback(1);
         }
 
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
+        }
+
         // Log zigbee clients on startup
         const devices = this.zigbee.getClients();
         logger.info(`Currently ${devices.length} devices are joined:`);
@@ -172,10 +186,12 @@ class Controller {
         } else {
             const Extension = AllExtensions.find((e) => e.name === name);
             assert(Extension, `Extension '${name}' does not exist`);
-            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+                this.enableDisableExtension, this.restartCallback, this.addExtension);
+
             this.extensions.push(extension);
-            this.callExtensionMethod('onZigbeeStarted', [], [extension]);
-            this.callExtensionMethod('onMQTTConnected', [], [extension]);
+            await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            await this.callExtensionMethod('onMQTTConnected', [], [extension]);
         }
     }
 
@@ -257,13 +273,22 @@ class Controller {
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
-                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                const settingsDevice = settings.getDevice(data.ieeeAddr);
+                logger.warn(`Device '${settingsDevice ? settingsDevice.friendlyName : data.ieeeAddr}' ` +
+                    `left the network`);
                 this.state.remove(data.ieeeAddr);
             }
         }
 
         // Call extensions
-        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+        const result = await this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+
+        // In case this message is not handled by the receive extension and last_seen is enabled, publish
+        // a message to update the last_seen.
+        if (name && !result[ExtensionReceive.prototype.constructor.name] &&
+            settings.get().advanced.last_seen !== 'disable' && type !== 'deviceLeave') {
+            this.publishEntityState(name, {});
+        }
     }
 
     onMQTTMessage(payload) {
@@ -380,10 +405,11 @@ class Controller {
     }
 
     async callExtensionMethod(method, parameters, extensions=null) {
+        const result = {};
         for (const extension of extensions || this.extensions) {
             if (extension[method]) {
                 try {
-                    await extension[method](...parameters);
+                    result[extension.constructor.name] = await extension[method](...parameters);
                 } catch (error) {
                     /* istanbul ignore next */
                     logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
@@ -394,6 +420,7 @@ class Controller {
                 }
             }
         }
+        return result;
     }
 }
 

lib/eventBus.js

@@ -23,13 +23,11 @@ class EventBus extends events.EventEmitter {
         super.emit(event, data);
     }
 
-    on(event, callback, extension=null) {
+    on(event, callback, extension) {
         assert(allowedEvents.includes(event), `Event '${event}' not supported`);
-        if (extension) {
-            if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
-            this.callbackByExtension[extension].push({event, callback});
-        }
-
+        assert(extension, `Extension cannot be null`);
+        if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
+        this.callbackByExtension[extension].push({event, callback});
         super.on(event, callback);
     }
 

lib/extension/availability.js

@@ -94,7 +94,7 @@ class Availability extends Extension {
                 } else {
                     this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
-                    }, utils.secondsToMilliseconds(300));
+                    }, utils.seconds(300));
                 }
             }
         }
@@ -148,7 +148,7 @@ class Availability extends Extension {
         clearTimeout(this.timers[device.ieeeAddr]);
         this.timers[device.ieeeAddr] = setTimeout(async () => {
             await this.handleIntervalPingable(device);
-        }, utils.secondsToMilliseconds(timeout));
+        }, utils.seconds(timeout));
     }
 
     async stop() {
@@ -182,9 +182,8 @@ class Availability extends Extension {
             this.onReconnect(device);
         }
 
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
-        const topic = `${name}/availability`;
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        const topic = `${resolvedEntity.name}/availability`;
         const payload = available ? 'online' : 'offline';
         if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;

lib/extension/availabilityNew.ts

@@ -0,0 +1,206 @@
+import ExtensionTS from './extensionts';
+import logger from '../util/logger';
+import {sleep, isAvailabilityEnabledForDevice, hours, minutes, seconds} from '../util/utils';
+import * as settings from '../util/settings';
+import debounce from 'debounce';
+
+// TODO
+// - Enable for HA addon
+// - Add to setting schema (when old availability is removed)
+class AvailabilityNew extends ExtensionTS {
+    private timers: {[s: string]: NodeJS.Timeout} = {};
+    private availabilityCache: {[s: string]: boolean} = {};
+    private retrieveStateDebouncers: {[s: string]: () => void} = {};
+    private pingQueue: ResolvedDevice[] = [];
+    private pingQueueExecuting = false;
+
+    constructor(zigbee: TempZigbee, mqtt: TempMQTT, state: TempState,
+        publishEntityState: TempPublishEntityState, eventBus: TempEventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.lastSeenChanged = this.lastSeenChanged.bind(this);
+        logger.warn('Using experimental new availability feature');
+    }
+
+    private getTimeout(rd: ResolvedDevice): number {
+        if (typeof rd.settings.availability === 'object' && rd.settings.availability?.timeout != null) {
+            return minutes(rd.settings.availability.timeout);
+        }
+
+        const key = this.isActiveDevice(rd) ? 'active' : 'passive';
+        const availabilitySettings = settings.get().availability;
+        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
+            return minutes(availabilitySettings[key]?.timeout);
+        }
+
+        return key === 'active' ? minutes(10) : hours(25);
+    }
+
+    private isActiveDevice(rd: ResolvedDevice): boolean {
+        return (rd.device.type === 'Router' && rd.device.powerSource !== 'Battery') ||
+            rd.device.powerSource === 'Mains (single phase)';
+    }
+
+    private isAvailable(rd: ResolvedDevice): boolean {
+        const ago = Date.now() - rd.device.lastSeen;
+        return ago < this.getTimeout(rd);
+    }
+
+    private resetTimer(rd: ResolvedDevice): void {
+        clearTimeout(this.timers[rd.device.ieeeAddr]);
+
+        // If the timer triggers, the device is not avaiable anymore otherwise resetTimer already have been called
+        if (this.isActiveDevice(rd)) {
+            // If device did not check in, ping it, if that fails it will be marked as offline
+            this.timers[rd.device.ieeeAddr] = setTimeout(
+                () => this.addToPingQueue(rd), this.getTimeout(rd) + seconds(1));
+        } else {
+            this.timers[rd.device.ieeeAddr] = setTimeout(
+                () => this.publishAvailability(rd, true), this.getTimeout(rd) + seconds(1));
+        }
+    }
+
+    private addToPingQueue(rd: ResolvedDevice): void {
+        this.pingQueue.push(rd);
+        this.pingQueueExecuteNext();
+    }
+
+    private removeFromPingQueue(rd: ResolvedDevice): void {
+        const index = this.pingQueue.findIndex((r) => r.device.ieeeAddr === rd.device.ieeeAddr);
+        index != -1 && this.pingQueue.splice(index, 1);
+    }
+
+    private async pingQueueExecuteNext(): Promise<void> {
+        if (this.pingQueue.length === 0 || this.pingQueueExecuting) return;
+        this.pingQueueExecuting = true;
+
+        const rd = this.pingQueue[0];
+        let pingedSuccessfully = false;
+        const available = this.availabilityCache[rd.device.ieeeAddr] || this.isAvailable(rd);
+        const attempts = available ? 2 : 1;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                // Enable recovery if device is marked as available and first ping fails.
+                const disableRecovery = !(i == 1 && available);
+                await rd.device.ping(disableRecovery);
+                pingedSuccessfully = true;
+                logger.debug(`Succesfully pinged '${rd.name}' (attempt ${i + 1}/${attempts})`);
+                break;
+            } catch (error) {
+                logger.error(`Failed to ping '${rd.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
+                // Try again in 3 seconds.
+                const lastAttempt = i - 1 === attempts;
+                !lastAttempt && await sleep(3);
+            }
+        }
+
+        this.publishAvailability(rd, !pingedSuccessfully);
+        this.resetTimer(rd);
+        this.removeFromPingQueue(rd);
+
+        // Sleep 2 seconds before executing next ping
+        await sleep(2);
+        this.pingQueueExecuting = false;
+        this.pingQueueExecuteNext();
+    }
+
+    override onMQTTConnected(): void {
+        for (const device of this.zigbee.getClients()) {
+            const rd = this.zigbee.resolveEntity(device) as ResolvedDevice;
+            if (isAvailabilityEnabledForDevice(rd, settings.get())) {
+                // Publish initial availablility
+                this.publishAvailability(rd, true);
+
+                this.resetTimer(rd);
+
+                // If an active device is initially unavailable, ping it.
+                if (this.isActiveDevice(rd) && !this.isAvailable(rd)) {
+                    this.addToPingQueue(rd);
+                }
+            }
+        }
+    }
+
+    override onZigbeeStarted(): void {
+        this.zigbee.on('lastSeenChanged', this.lastSeenChanged);
+    }
+
+    override onZigbeeEvent(type: ZigbeeEventType, data: ZigbeeEventData, re: ResolvedEntity): void {
+        /* istanbul ignore else */
+        if (type === 'deviceLeave') {
+            clearTimeout(this.timers[data.ieeeAddr]);
+        } else if (type === 'deviceAnnounce') {
+            this.retrieveState(re as ResolvedDevice);
+        }
+    }
+
+    private publishAvailability(rd: ResolvedDevice, logLastSeen: boolean): void {
+        if (logLastSeen) {
+            const ago = Date.now() - rd.device.lastSeen;
+            if (this.isActiveDevice(rd)) {
+                logger.debug(
+                    `Active device '${rd.name}' was last seen '${(ago / minutes(1)).toFixed(2)}' minutes ago.`);
+            } else {
+                logger.debug(`Passive device '${rd.name}' was last seen '${(ago / hours(1)).toFixed(2)}' hours ago.`);
+            }
+        }
+
+        const available = this.isAvailable(rd);
+        if (this.availabilityCache[rd.device.ieeeAddr] == available) {
+            return;
+        }
+
+        if (rd.device.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[rd.device.ieeeAddr] === false) {
+            logger.debug(`Device '${rd.name}' reconnected`);
+            this.retrieveState(rd);
+        }
+
+        const topic = `${rd.name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        this.availabilityCache[rd.device.ieeeAddr] = available;
+        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+    }
+
+    private lastSeenChanged(data: {device: Device}): void {
+        const rd = this.zigbee.resolveEntity(data.device) as ResolvedDevice;
+        if (isAvailabilityEnabledForDevice(rd, settings.get())) {
+            // Remove from ping queue, not necessary anymore since we know the device is online.
+            this.removeFromPingQueue(rd);
+            this.resetTimer(rd);
+            this.publishAvailability(rd, false);
+        }
+    }
+
+    override stop(): void {
+        Object.values(this.timers).forEach((t) => clearTimeout(t));
+        this.zigbee.removeListener('lastSeenChanged', this.lastSeenChanged);
+        super.stop();
+    }
+
+    private retrieveState(rd: ResolvedDevice): void {
+        /**
+         * Retrieve state of a device in a debounced manner, this function is called on a 'deviceAnnounce' which a
+         * device can send multiple times after each other.
+         */
+        if (rd.definition && !rd.device.interviewing && !this.retrieveStateDebouncers[rd.device.ieeeAddr]) {
+            this.retrieveStateDebouncers[rd.device.ieeeAddr] = debounce(async () => {
+                try {
+                    logger.debug(`Retrieving state of '${rd.name}' after reconnect`);
+                    // Color and color temperature converters do both, only needs to be called once.
+                    const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
+                    for (const keys of keySet) {
+                        const converter = rd.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
+                        await converter?.convertGet?.(rd.endpoint, keys[0],
+                            {message: this.state.get(rd.device.ieeeAddr) || {}, mapped: rd.definition});
+                    }
+                } catch (error) {
+                    logger.error(`Failed to read state of '${rd.name}' after reconnect (${error.message})`);
+                }
+            }, seconds(2));
+        }
+
+        this.retrieveStateDebouncers[rd.device.ieeeAddr]?.();
+    }
+}
+
+module.exports = AvailabilityNew;

lib/extension/bind.js

@@ -118,7 +118,7 @@ class Bind extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.legacyApi = settings.get().advanced.legacy_api;
-        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d));
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d), this.constructor.name);
         this.pollDebouncers = {};
     }
 
@@ -277,10 +277,10 @@ class Bind extends Extension {
     }
 
     async groupMembersChanged(data) {
-        const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
-            .flat().filter((b) => b.target === data.group.group);
-
         if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v)).map((e) => e.binds)
+                .reduce((a, v) => a.concat(v)).filter((b) => b.target === data.group.group);
             await this.setupReporting(bindsToGroup);
         } else { // action === remove/remove_all
             if (!data.skipDisableReporting) {
@@ -334,8 +334,8 @@ class Bind extends Extension {
             const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
             const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
                 .map((b) => b.cluster.name);
-            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
-                .flat().filter((bind) => {
+            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).reduce((a, v) => a.concat(v))
+                .map((e) => e.binds).reduce((a, v) => a.concat(v)).filter((bind) => {
                     if (bind.target.constructor.name === 'Group') {
                         return bind.target.members.includes(endpoint);
                     } else {

lib/extension/bridge.js

@@ -6,7 +6,7 @@ const settings = require('../util/settings');
 const Transport = require('winston-transport');
 const stringify = require('json-stable-stringify-without-jsonify');
 const objectAssignDeep = require(`object-assign-deep`);
-const {updatedDiff, addedDiff} = require('deep-object-diff');
+const {detailedDiff} = require('deep-object-diff');
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
@@ -44,26 +44,33 @@ class Bridge extends Extension {
     }
 
     async onMQTTConnected() {
-        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
-        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups(), this.constructor.name);
         this.eventBus.on(`devicesChanged`, () => {
             this.publishDevices();
             this.publishInfo();
-        });
+        }, this.constructor.name);
         this.eventBus.on(`deviceRenamed`, () => {
             this.publishInfo();
-        });
+        }, this.constructor.name);
         this.eventBus.on(`groupRenamed`, () => {
             this.publishInfo();
-        });
-        this.zigbee.on('permitJoinChanged', (data) => this.permitJoinChanged(data));
+        }, this.constructor.name);
+        this.zigbee.removeListener('permitJoinChanged', this.permitJoinChanged);
+        this.permitJoinChanged = this.permitJoinChanged.bind(this);
+        this.zigbee.on('permitJoinChanged', this.permitJoinChanged);
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
     }
 
+    async stop() {
+        super.stop();
+        this.zigbee.removeListener('permitJoinChanged', this.permitJoinChanged);
+    }
+
     setupMQTTLogging() {
         const mqtt = this.mqtt;
         class EventTransport extends Transport {
@@ -122,7 +129,12 @@ class Bridge extends Extension {
                 payload = {
                     friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
                 };
-            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
+            } else {
+                // deviceLeave
+                const settingsDevice = settings.getDevice(ieeeAddress);
+                payload = {ieee_address: ieeeAddress,
+                    friendly_name: settingsDevice ? settingsDevice.friendlyName : null};
+            }
 
             await this.mqtt.publish(
                 'bridge/event',
@@ -131,8 +143,8 @@ class Bridge extends Extension {
             );
         }
 
-        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started') ||
-            'deviceNetworkAddressChanged' === type) {
+        if (['deviceLeave', 'deviceInterview', 'deviceNetworkAddressChanged',
+            'deviceJoined', 'deviceAnnounce'].includes(type)) {
             await this.publishDevices();
         }
     }
@@ -154,9 +166,21 @@ class Bridge extends Extension {
             throw new Error(`Invalid payload`);
         }
 
-        const diffUpdated = updatedDiff(settings.get(), message.options);
-        const diffAdded = addedDiff(settings.get(), message.options);
-        const newSettings = objectAssignDeep.noMutate(diffUpdated, diffAdded);
+        const diff = detailedDiff(settings.get(), message.options);
+
+        // Remove any settings that are in the deleted.diff but not in the passed options
+        const cleanupDeleted = (options, deleted) => {
+            for (const key of Object.keys(deleted)) {
+                if (!(key in options)) {
+                    delete deleted[key];
+                } else if (!Array.isArray(options[key])) {
+                    cleanupDeleted(options[key], deleted[key]);
+                }
+            }
+        };
+        cleanupDeleted(message.options, diff.deleted);
+
+        const newSettings = objectAssignDeep.noMutate(diff.added, diff.updated, diff.deleted);
 
         // deep-object-diff converts arrays to objects, set original array back here
         const convertBackArray = (before, after) => {
@@ -626,6 +650,7 @@ class Bridge extends Extension {
                 model_id: device.modelID,
                 interviewing: device.interviewing,
                 interview_completed: device.interviewCompleted,
+                manufacturer: device.manufacturerName,
                 endpoints,
             };
         });

lib/extension/configure.js

@@ -18,7 +18,7 @@ class Configure extends Extension {
 
         this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled);
+        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled, this.constructor.name);
     }
 
     onReportingDisabled(data) {

lib/extension/extensionts.ts

@@ -0,0 +1,69 @@
+abstract class ExtensionTS {
+    protected zigbee: TempZigbee;
+    protected mqtt: TempMQTT;
+    protected state: TempState;
+    protected publishEntityState: TempPublishEntityState;
+    protected eventBus: TempEventBus;
+
+    /**
+     * Besides intializing variables, the constructor should do nothing!
+     *
+     * @param {Zigbee} zigbee Zigbee controller
+     * @param {MQTT} mqtt MQTT controller
+     * @param {State} state State controller
+     * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
+     */
+    constructor(zigbee: TempZigbee, mqtt: TempMQTT, state: TempState,
+        publishEntityState: TempPublishEntityState, eventBus: TempEventBus) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
+    }
+
+    /**
+     * This method is called by the controller once Zigbee has been started.
+     */
+    /* istanbul ignore next */
+    onZigbeeStarted(): void {}
+
+    /**
+     * This method is called by the controller once connected to the MQTT server.
+     */
+    /* istanbul ignore next */
+    onMQTTConnected(): void {}
+
+    /**
+     * Is called when a Zigbee message from a device is received.
+     * @param {string} type Type of the message
+     * @param {Object} data Data of the message
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
+     * @param {Object?} settingsDevice Device settings
+     */
+    /* istanbul ignore next */
+    /* eslint-disable-next-line @typescript-eslint/no-unused-vars */
+    onZigbeeEvent(type: ZigbeeEventType, data: ZigbeeEventData, resolvedEntity: ResolvedEntity): void {}
+
+    /**
+     * Is called when a MQTT message is received
+     * @param {string} topic Topic on which the message was received
+     * @param {Object} message The received message
+     * @return {boolean} if the message was handled
+     */
+    /* istanbul ignore next */
+    /* eslint-disable-next-line @typescript-eslint/no-unused-vars */
+    onMQTTMessage(topic: string, message: string): boolean {
+        return false;
+    }
+
+    /**
+     * Is called once the extension has to stop
+     */
+    stop(): void {
+        this.eventBus.removeListenersExtension(this.constructor.name);
+    }
+}
+
+export default ExtensionTS;

lib/extension/externalExtension.js

@@ -9,7 +9,8 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
 
 class ExternalExtension extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus, addExtension, enableDisableExtension) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus,
+        enableDisableExtension, restartCallback, addExtension) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
         this.addExtension = addExtension;

lib/extension/frontend.js

@@ -19,8 +19,8 @@ class Frontend extends Extension {
         this.onRequest = this.onRequest.bind(this);
         this.onUpgrade = this.onUpgrade.bind(this);
         this.mqttBaseTopic = settings.get().mqtt.base_topic;
-        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
-        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
+        this.onMQTTPublishMessage = this.onMQTTPublishMessage.bind(this);
+        this.mqtt.on('publishMessage', this.onMQTTPublishMessage);
         this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
         this.server = http.createServer(this.onRequest);
         this.server.on('upgrade', this.onUpgrade);
@@ -47,9 +47,9 @@ class Frontend extends Extension {
     async stop() {
         for (const client of this.wss.clients) {
             client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
-            client.close();
+            client.terminate();
         }
-
+        this.wss.close();
         return new Promise((resolve) => {
             this.server.close(resolve);
         });
@@ -75,8 +75,9 @@ class Frontend extends Extension {
     }
 
     onWebSocketConnection(ws) {
-        ws.on('message', (message) => {
-            if (message) {
+        ws.on('message', (data, isBinary) => {
+            if (!isBinary && data) {
+                const message = data.toString();
                 const {topic, payload} = utils.parseJSON(message, message);
                 this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
             }
@@ -106,7 +107,7 @@ class Frontend extends Extension {
         }
     }
 
-    onMQTTPublishedMessage(data) {
+    onMQTTPublishMessage(data) {
         let {topic, payload, options} = data;
         if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
             // Send topic without base_topic

lib/extension/groups.js

@@ -2,7 +2,6 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 const utils = require('../util/utils');
-const postfixes = utils.getEndpointNames();
 const stringify = require('json-stable-stringify-without-jsonify');
 const equals = require('fast-deep-equal/es6');
 
@@ -11,6 +10,21 @@ const topicRegex =
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
+const stateProperties = {
+    'state': (value, definition) => true,
+    'brightness': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
 class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
@@ -20,7 +34,7 @@ class Groups extends Extension {
     }
 
     async onZigbeeStarted() {
-        this.eventBus.on('stateChange', this.onStateChange);
+        this.eventBus.on('stateChange', this.onStateChange, this.constructor.name);
         await this.syncGroupsWithSettings();
     }
 
@@ -38,6 +52,7 @@ class Groups extends Extension {
                 }
             } catch (error) {
                 logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+                logger.debug(error.stack);
             }
         };
 
@@ -82,18 +97,17 @@ class Groups extends Extension {
             return;
         }
 
-        const properties = ['state', 'brightness', 'color_temp', 'color', 'color_mode'];
         const payload = {};
 
         let endpointName;
         for (let [prop, value] of Object.entries(data.update)) {
-            const endpointNameMatch = utils.getEndpointNames().find((n) => prop.endsWith(`_${n}`));
+            const endpointNameMatch = utils.endpointNames.find((n) => prop.endsWith(`_${n}`));
             if (endpointNameMatch) {
                 prop = prop.substring(0, prop.length - endpointNameMatch.length - 1);
                 endpointName = endpointNameMatch;
             }
 
-            if (properties.includes(prop)) {
+            if (prop in stateProperties) {
                 payload[prop] = value;
             }
         }
@@ -121,8 +135,15 @@ class Groups extends Extension {
 
                 const groupIDsToPublish = new Set();
                 for (const member of resolvedEntity.group.members) {
-                    const memberPayload = {...payload};
-                    const endpointName = this.zigbee.resolveEntity(member).endpointName;
+                    const resolvedEntity = this.zigbee.resolveEntity(member);
+                    const memberPayload = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], resolvedEntity.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = resolvedEntity.endpointName;
                     if (endpointName) {
                         Object.keys(memberPayload).forEach((key) => {
                             memberPayload[`${key}_${endpointName}`] = memberPayload[key];
@@ -220,7 +241,7 @@ class Groups extends Extension {
                 return {};
             }
 
-            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+            hasEndpointName = utils.endpointNames.find((p) => message.endsWith(`/${p}`));
         } else if (topicRegexMatch) {
             type = topicRegexMatch[1];
             message = JSON.parse(message);
@@ -240,7 +261,7 @@ class Groups extends Extension {
                 error = `Device '${message.device}' does not exist`;
             }
 
-            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
+            hasEndpointName = utils.endpointNames.find((p) => message.device.endsWith(`/${p}`));
         }
 
         return {
@@ -272,8 +293,11 @@ class Groups extends Extension {
                 const definition = resolvedEntityDevice.definition;
                 const endpoints = definition && definition.endpoint ?
                     definition.endpoint(resolvedEntityDevice.device) : null;
-                const endpointName = endpoints ?
-                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+                let endpointName = null;
+                if (endpoints) {
+                    const matches = Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID);
+                    endpointName = matches && matches[0];
+                }
 
                 if (endpointName) {
                     keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
@@ -329,6 +353,7 @@ class Groups extends Extension {
                 }
             } catch (e) {
                 error = `Failed to ${type} from group (${e.message})`;
+                logger.debug(e.stack);
             }
         }
 

lib/extension/legacy/bridgeLegacy.js

@@ -373,7 +373,7 @@ class BridgeLegacy extends Extension {
     }
 
     async publish() {
-        const info = await utils.getZigbee2mqttVersion();
+        const info = await utils.getZigbee2MQTTVersion();
         const coordinator = await this.zigbee.getCoordinatorVersion();
         const topic = `bridge/config`;
         const payload = {

lib/extension/legacy/softReset.js

@@ -12,11 +12,11 @@ class SoftReset extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;
-        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
+        this.timeout = utils.seconds(settings.get().advanced.soft_reset_timeout);
     }
 
     onZigbeeStarted() {
-        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+        logger.debug(`Soft reset timeout set to ${this.timeout / 1000} seconds`);
         this.resetTimer();
     }
 

lib/extension/otaUpdate.js

@@ -37,6 +37,7 @@ class OTAUpdate extends Extension {
     async onZigbeeEvent(type, data, resolvedEntity) {
         if (settings.get().ota.disable_automatic_update_check) return;
         if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+        logger.debug(`Device '${resolvedEntity.name}' requested OTA`);
 
         const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
         if (supportsOTA) {
@@ -119,6 +120,7 @@ class OTAUpdate extends Extension {
         const type = topic.substring(topic.lastIndexOf('/') + 1);
         const responseData = {id: ID};
         let error = null;
+        let errorStack = null;
 
         if (!resolvedEntity || resolvedEntity.type !== 'device') {
             error = `Device '${ID}' does not exist`;
@@ -173,6 +175,7 @@ class OTAUpdate extends Extension {
                     responseData.updateAvailable = available;
                 } catch (e) {
                     error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
+                    errorStack = e.stack;
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
@@ -235,7 +238,9 @@ class OTAUpdate extends Extension {
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
                     }
                 } catch (e) {
+                    logger.debug(`Update of '${resolvedEntity.name}' failed (${e})`);
                     error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+                    errorStack = e.stack;
 
                     this.removeProgressAndRemainingFromState(resolvedEntity.device);
                     const payload = this.getEntityPublishPayload('available');
@@ -260,6 +265,7 @@ class OTAUpdate extends Extension {
 
         if (error) {
             logger.error(error);
+            errorStack && logger.debug(errorStack);
         }
     }
 }

lib/extension/publish.js

@@ -7,10 +7,10 @@ const assert = require('assert');
 const Extension = require('./extension');
 const stringify = require('json-stable-stringify-without-jsonify');
 
-const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}))?/(get|set)(?:/(.+))?`);
 const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 
-const groupConverters = [
+const defaultGroupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
     zigbeeHerdsmanConverters.toZigbeeConverters.effect,
@@ -91,11 +91,9 @@ class EntityPublish extends Extension {
             options = resolvedEntity.settings;
             definition = resolvedEntity.group.members
                 .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
-            converters = new Set(groupConverters);
-            for (const d of definition) {
-                d.toZigbee.forEach(converters.add, converters);
-            }
-            converters = [...converters];
+            converters = new Set();
+            definition.forEach((d) => d.toZigbee.forEach(converters.add, converters));
+            converters = converters.size ? [...converters] : defaultGroupConverters;
             membersState = {};
             for (const member of resolvedEntity.group.members) {
                 const ieeeAddr = member.getDevice().ieeeAddr;
@@ -160,6 +158,12 @@ class EntityPublish extends Extension {
 
         // For each attribute call the corresponding converter
         const usedConverters = {};
+        const toPublish = {};
+        const addToToPublish = (ID, payload) => {
+            if (!(ID in toPublish)) toPublish[ID] = {};
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
         for (let [key, value] of entries) {
             let endpointName = topic.endpointName;
             let actualTarget = target;
@@ -167,7 +171,7 @@ class EntityPublish extends Extension {
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
+                const endpointNameMatch = utils.endpointNames.find((n) => key.endsWith(`_${n}`));
                 if (endpointNameMatch) {
                     endpointName = endpointNameMatch;
                     const regex = new RegExp(`(_${endpointNameMatch})$`, 'gm');
@@ -229,11 +233,9 @@ class EntityPublish extends Extension {
                         const msg = result.state;
 
                         if (endpointName) {
-                            for (const key of ['state', 'brightness', 'color', 'color_temp', 'color_mode']) {
-                                if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${endpointName}`] = msg[key];
-                                    delete msg[key];
-                                }
+                            for (const key of Object.keys(msg)) {
+                                msg[`${key}_${endpointName}`] = msg[key];
+                                delete msg[key];
                             }
                         }
 
@@ -241,13 +243,14 @@ class EntityPublish extends Extension {
                         if (options.hasOwnProperty('filtered_optimistic')) {
                             options.filtered_optimistic.forEach((a) => delete msg[a]);
                         }
-
-                        this.publishEntityState(resolvedEntity.settings.ID, msg);
+                        if (Object.keys(msg).length != 0) {
+                            addToToPublish(resolvedEntity.settings.ID, msg);
+                        }
                     }
 
                     if (result && result.membersState && optimistic) {
                         for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
-                            this.publishEntityState(ieeeAddr, state);
+                            addToToPublish(ieeeAddr, state);
                         }
                     }
 
@@ -291,6 +294,10 @@ class EntityPublish extends Extension {
             usedConverters[endpointOrGroupID].push(converter);
         }
 
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            this.publishEntityState(ID, payload);
+        }
+
         return true;
     }
 }

lib/extension/receive.js

@@ -10,7 +10,7 @@ class Receive extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.elapsed = {};
         this.debouncers = {};
-        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data), this.constructor.name);
     }
 
     async onZigbeeStarted() {
@@ -166,6 +166,7 @@ class Receive extends Extension {
 
         if (Object.keys(payload).length) {
             publish(payload);
+            return true;
         }
     }
 }

lib/mqtt.js

@@ -116,6 +116,8 @@ class MQTT extends events.EventEmitter {
             this.publishedTopics.add(topic);
         }
 
+        this.emit('publishMessage', {topic, payload, options});
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -135,7 +137,6 @@ class MQTT extends events.EventEmitter {
 
         return new Promise((resolve) => {
             this.client.publish(topic, payload, actualOptions, () => {
-                this.emit('publishedMessage', {topic, payload, options});
                 resolve();
             });
         });

lib/state.js

@@ -7,9 +7,9 @@ const objectAssignDeep = require('object-assign-deep');
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
+    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
+    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
+    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
 ];
 
 class State {

lib/types/types.ts

@@ -0,0 +1,169 @@
+import type {Device as ZHDevice, Endpoint} from 'zigbee-herdsman/dist/controller/model';
+
+declare global {
+    type Device = ZHDevice;
+
+    type ZigbeeEventType = 'deviceLeave' | 'deviceAnnounce';
+
+    interface ZigbeeEventData {
+        ieeeAddr: string;
+    }
+
+    /* eslint-disable */
+    // Controller
+    interface KeyValue {
+        [s: string]: any,
+    }
+
+    interface Settings {
+        devices?: {[s: string]: {friendly_name: string, retention?: number}},
+        groups?: {[s: string]: {friendly_name: string, devices?: string[]}},
+        passlist: string[],
+        blocklist: string[],
+        whitelist: string[],
+        ban: string[],
+        availability?: boolean | {
+            active?: {timeout?: number},
+            passive?: {timeout?: number}
+        },
+        permit_join: boolean,
+        frontend?: {
+            auth_token?: string,
+        },
+        mqtt: {
+            include_device_information: boolean,
+            force_disable_retain: boolean
+            version?: number,
+            user?: string,
+            password?: string,
+        },
+        serial: {
+            disable_led: boolean,
+        },
+        device_options: {[s: string]: unknown},
+        map_options: {
+            graphviz: {
+                colors: {
+                    fill: {
+                        enddevice: string,
+                        coordinator: string,
+                        router: string,
+                    },
+                    font: {
+                        coordinator: string,
+                        router: string,
+                        enddevice: string,
+                    },
+                    line: {
+                        active: string,
+                        inactive: string,
+                    },
+                },
+            },
+        },
+        experimental: {
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            availability_new?: boolean,
+        },
+        advanced: {
+            legacy_api: boolean,
+            log_rotation: boolean,
+            log_symlink_current: boolean,
+            log_output: ('console' | 'file')[],
+            log_directory: string,
+            log_file: string,
+            log_level: 'debug' | 'info' | 'error' | 'warn',
+            log_syslog: {},
+            soft_reset_timeout: number,
+            pan_id: number,
+            ext_pan_id: number[],
+            channel: number,
+            adapter_concurrent: number | null,
+            adapter_delay: number | null,
+            availability_timeout: number,
+            availability_blocklist: string[],
+            availability_passlist: string[],
+            availability_blacklist: string[],
+            availability_whitelist: string[],
+            cache_state: boolean,
+            cache_state_persistent: boolean,
+            cache_state_send_on_startup: boolean,
+            last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' |  'epoch',
+            elapsed: boolean,
+            network_key: number[],
+            report: boolean,
+            homeassistant_discovery_topic: string,
+            homeassistant_status_topic: string,
+            homeassistant_legacy_entity_attributes: boolean,
+            homeassistant_legacy_triggers: boolean,
+            timestamp_format: string,
+        },
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean
+        },
+        external_converters: string[],
+    }
+
+    interface DeviceSettings {
+        friendlyName: string,
+        ID: string,
+        retention?: number,
+    }
+
+    interface GroupSettings {
+        friendlyName: string,
+        devices: string[],
+        ID: number,
+    }
+
+    type EntitySettings = {
+        type: 'device' | 'group'
+        ID: number | string,
+        friendlyName: string,
+    }
+
+    interface ResolvedEntity {
+        type: 'device' | 'group',
+    }
+
+    interface Definition  {
+        model: string
+        toZigbee: {key: string[], convertGet?: (entity: Endpoint, key: string, meta: {message: {}, mapped: Definition}) => Promise<void>}[]
+    }
+
+    interface ResolvedDevice {
+        type: 'device',
+        definition?: Definition,
+        name: string,
+        endpoint: Endpoint,
+        device: Device,
+        settings: {
+            friendlyName: string,
+            availability?: {timeout?: number} | boolean,
+        }
+    }
+
+    type lastSeenChangedHandler = (data: {device: Device}) => void;
+
+    interface TempZigbee {
+        getClients: () => Device[];
+        on: (event: 'lastSeenChanged', handler: lastSeenChangedHandler) => void;
+        removeListener: (event: 'lastSeenChanged', handler: lastSeenChangedHandler) => void;
+        resolveEntity: (device: Device) => ResolvedEntity;
+    }
+
+    interface TempMQTT {
+        publish: (topic: string, payload: string, options: {}, base?: string, skipLog?: boolean, skipReceive?: boolean) => Promise<void>;
+    }
+
+    interface TempState {
+        get: (ID: string) => {} | null;
+    }
+
+    interface TempEventBus {
+        removeListenersExtension: (extension: string) => void;
+    }
+
+    type TempPublishEntityState = () => void;
+}

lib/util/settings.schema.json

@@ -423,6 +423,12 @@
           "requiresRestart": true,
           "examples": ["homeassistant"]
         },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
         "homeassistant_status_topic": {
           "type": "string",
           "title": "Home Assistant status topic",
@@ -679,7 +685,7 @@
         "qos": {
           "type": "number",
           "title": "QoS",
-          "descritption": "QoS level for MQTT messages of this device"
+          "description": "QoS level for MQTT messages of this device"
         },
         "debounce": {
           "type": "number",

lib/util/settings.ts

@@ -1,17 +1,19 @@
-const data = require('./data');
-const utils = require('./utils');
-// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
-const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
-const objectAssignDeep = require(`object-assign-deep`);
-const path = require('path');
-const yaml = require('./yaml');
-const Ajv = require('ajv');
-const schema = require('./settings.schema.json');
-const ajvSetting = new Ajv({allErrors: true}).compile(schema);
-const ajvRestartRequired = new Ajv({allErrors: true})
-    .addKeyword('requiresRestart', {validate: (v) => !v}).compile(schema);
+import data from './data';
+import * as utils from './utils';
+import objectAssignDeep from 'object-assign-deep';
+import path from 'path';
+import * as yaml from './yaml';
+import Ajv from 'ajv';
+import schemaJson from './settings.schema.json';
+export const schema = schemaJson;
 
-const defaults = {
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+
+const defaults: Settings = {
     passlist: [],
     blocklist: [],
     // Deprecated: use block/passlist
@@ -125,6 +127,15 @@ const defaults = {
          */
         homeassistant_status_topic: 'hass/status',
 
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
         /**
          * Home Assistant legacy triggers, when enabled:
          * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
@@ -152,11 +163,12 @@ const defaults = {
     external_converters: [],
 };
 
-let _settings;
-let _settingsWithDefaults;
+let _settings: Partial<Settings>;
+let _settingsWithDefaults: Settings;
 
-function write() {
-    const settings = get();
+function write(): void {
+    const settings = getInternalSettings();
+    /* eslint-disable-line */ // @ts-ignore
     const toWrite = objectAssignDeep.noMutate(settings);
 
     // Read settings to check if we have to split devices/groups into separate file.
@@ -179,17 +191,18 @@ function write() {
     }
 
     // Write devices/groups to separate file if required.
-    const writeDevicesOrGroups = (type) => {
+    const writeDevicesOrGroups = (type: 'devices' | 'groups'): void => {
         if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
             const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            /* eslint-disable-line */ // @ts-ignore
             const content = objectAssignDeep.noMutate(settings[type]);
 
             // If an array, only write to first file and only devices which are not in the other files.
             if (Array.isArray(actual[type])) {
-                actual[type].filter((f, i) => i !== 0)
-                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
-                    .map((c) => Object.keys(c))
-                    .forEach((k) => delete content[k]);
+                actual[type].filter((f: string, i: number) => i !== 0)
+                    .map((f: string) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c: KeyValue) => Object.keys(c))
+                    .forEach((k: string) => delete content[k]);
             }
 
             yaml.writeIfChanged(data.joinPath(fileToWrite), content);
@@ -203,12 +216,13 @@ function write() {
     yaml.writeIfChanged(file, toWrite);
 
     _settings = read();
-    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    /* eslint-disable-line */ // @ts-ignore
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
 }
 
-function validate() {
+export function validate(): string[] {
     try {
-        get();
+        getInternalSettings();
     } catch (error) {
         if (error.name === 'YAMLException') {
             return [
@@ -221,7 +235,7 @@ function validate() {
     }
 
     if (!ajvSetting(_settings)) {
-        return ajvSetting.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+        return ajvSetting.errors.map((v) => `${v.instancePath.substring(1)} ${v.message}`);
     }
 
     const errors = [];
@@ -236,14 +250,14 @@ function validate() {
     }
 
     // Verify that all friendly names are unique
-    const names = [];
-    const check = (name) => {
+    const names: string[] = [];
+    const check = (name: string): void => {
         if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
         errors.push(...utils.validateFriendlyName(name));
         names.push(name);
     };
 
-    const settingsWithDefaults = getWithDefaults();
+    const settingsWithDefaults = get();
     Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
     Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
 
@@ -255,7 +269,7 @@ function validate() {
         }
     }
 
-    const checkAvailabilityList = (list, type) => {
+    const checkAvailabilityList = (list: string[], type: string): void => {
         list.forEach((e) => {
             if (!getEntity(e)) {
                 errors.push(`Non-existing entity '${e}' specified in '${type}'`);
@@ -271,11 +285,12 @@ function validate() {
     return errors;
 }
 
-function read() {
-    const s = yaml.read(file);
+function read(): Settings {
+    const s = yaml.read(file) as Settings;
 
     // Read !secret MQTT username and password if set
-    const interpetValue = (value) => {
+    // eslint-disable-next-line
+    const interpetValue = (value: any): any => {
         const re = /!(.*) (.*)/g;
         const match = re.exec(value);
         if (match) {
@@ -287,26 +302,28 @@ function read() {
         }
     };
 
-    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+    if (s.mqtt?.user && s.mqtt?.password) {
         s.mqtt.user = interpetValue(s.mqtt.user);
         s.mqtt.password = interpetValue(s.mqtt.password);
     }
 
-    if (s.advanced && s.advanced.network_key) {
+    if (s.advanced?.network_key) {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    if (s.frontend && s.frontend.auth_token) {
+    if (s.frontend?.auth_token) {
         s.frontend.auth_token = interpetValue(s.frontend.auth_token);
     }
 
     // Read devices/groups configuration from separate file if specified.
-    const readDevicesOrGroups = (type) => {
+    const readDevicesOrGroups = (type: 'devices' | 'groups'): void => {
         if (typeof s[type] === 'string' || Array.isArray(s[type])) {
-            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            /* eslint-disable-line */ // @ts-ignore
+            const files: string[] = Array.isArray(s[type]) ? s[type] : [s[type]];
             s[type] = {};
             for (const file of files) {
                 const content = yaml.readIfExists(data.joinPath(file), {});
+                /* eslint-disable-line */ // @ts-ignore
                 s[type] = objectAssignDeep.noMutate(s[type], content);
             }
         }
@@ -318,8 +335,8 @@ function read() {
     return s;
 }
 
-function applyEnvironmentVariables(settings) {
-    const iterate = (obj, path) => {
+function applyEnvironmentVariables(settings: Partial<Settings>): void {
+    const iterate = (obj: KeyValue, path: string[]): void => {
         Object.keys(obj).forEach((key) => {
             if (key !== 'type') {
                 if (key !== 'properties' && obj[key]) {
@@ -327,14 +344,17 @@ function applyEnvironmentVariables(settings) {
                     const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
                     const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
                     if (process.env[envVariableName]) {
-                        const setting = path.reduce((acc, val, index) => {
+                        const setting = path.reduce((acc, val) => {
+                            /* eslint-disable-line */ // @ts-ignore
                             acc[val] = acc[val] || {};
+                            /* eslint-disable-line */ // @ts-ignore
                             return acc[val];
                         }, settings);
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
                             setting[key] = JSON.parse(process.env[envVariableName]);
                         } else if (type.indexOf('number') >= 0) {
+                            /* eslint-disable-line */ // @ts-ignore
                             setting[key] = process.env[envVariableName] * 1;
                         } else if (type.indexOf('boolean') >= 0) {
                             setting[key] = process.env[envVariableName].toLowerCase() === 'true';
@@ -360,7 +380,7 @@ function applyEnvironmentVariables(settings) {
     iterate(schema.properties, []);
 }
 
-function get() {
+function getInternalSettings(): Partial<Settings> {
     if (!_settings) {
         _settings = read();
         applyEnvironmentVariables(_settings);
@@ -369,9 +389,10 @@ function get() {
     return _settings;
 }
 
-function getWithDefaults() {
+export function get(): Settings {
     if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+        /* eslint-disable-line */ // @ts-ignore
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
     }
 
     if (!_settingsWithDefaults.devices) {
@@ -385,8 +406,9 @@ function getWithDefaults() {
     return _settingsWithDefaults;
 }
 
-function set(path, value) {
-    let settings = get();
+export function set(path: string[], value: string | number | boolean | KeyValue): void {
+    /* eslint-disable-next-line */
+    let settings: any = getInternalSettings();
 
     for (let i = 0; i < path.length; i++) {
         const key = path[i];
@@ -404,15 +426,16 @@ function set(path, value) {
     write();
 }
 
-function apply(newSettings) {
+export function apply(newSettings: Record<string, unknown>): boolean {
     ajvSetting(newSettings);
     const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
     if (errors.length) {
         const error = errors[0];
-        throw new Error(`${error.dataPath.substring(1)} ${error.message}`);
+        throw new Error(`${error.instancePath.substring(1)} ${error.message}`);
     }
 
-    get(); // Ensure _settings is intialized.
+    getInternalSettings(); // Ensure _settings is intialized.
+    /* eslint-disable-line */ // @ts-ignore
     _settings = objectAssignDeep.noMutate(_settings, newSettings);
     write();
 
@@ -422,8 +445,8 @@ function apply(newSettings) {
     return restartRequired;
 }
 
-function getGroup(IDorName) {
-    const settings = getWithDefaults();
+export function getGroup(IDorName: string): GroupSettings {
+    const settings = get();
     const byID = settings.groups[IDorName];
     if (byID) {
         return {devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
@@ -438,14 +461,14 @@ function getGroup(IDorName) {
     return null;
 }
 
-function getGroups() {
-    const settings = getWithDefaults();
+export function getGroups(): GroupSettings[] {
+    const settings = get();
     return Object.entries(settings.groups).map(([ID, group]) => {
         return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
     });
 }
 
-function getGroupThrowIfNotExists(IDorName) {
+function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
     const group = getGroup(IDorName);
     if (!group) {
         throw new Error(`Group '${IDorName}' does not exist`);
@@ -454,8 +477,8 @@ function getGroupThrowIfNotExists(IDorName) {
     return group;
 }
 
-function getDevice(IDorName) {
-    const settings = getWithDefaults();
+export function getDevice(IDorName: string): DeviceSettings {
+    const settings = get();
     const byID = settings.devices[IDorName];
     if (byID) {
         return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
@@ -470,7 +493,7 @@ function getDevice(IDorName) {
     return null;
 }
 
-function getDeviceThrowIfNotExists(IDorName) {
+function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
     const device = getDevice(IDorName);
     if (!device) {
         throw new Error(`Device '${IDorName}' does not exist`);
@@ -479,7 +502,7 @@ function getDeviceThrowIfNotExists(IDorName) {
     return device;
 }
 
-function getEntity(IDorName) {
+export function getEntity(IDorName: string): EntitySettings {
     const device = getDevice(IDorName);
     if (device) {
         return {...device, type: 'device'};
@@ -493,12 +516,12 @@ function getEntity(IDorName) {
     return null;
 }
 
-function addDevice(ID) {
+export function addDevice(ID: string): DeviceSettings {
     if (getDevice(ID)) {
         throw new Error(`Device '${ID}' already exists`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
 
     if (!settings.devices) {
         settings.devices = {};
@@ -509,9 +532,8 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
-// Legacy: can be removed after bridgeLegacy has been removed
-function whitelistDevice(ID) {
-    const settings = get();
+export function whitelistDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.whitelist) {
         settings.whitelist = [];
     }
@@ -524,8 +546,8 @@ function whitelistDevice(ID) {
     write();
 }
 
-function blockDevice(ID) {
-    const settings = get();
+export function blockDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.blocklist) {
         settings.blocklist = [];
     }
@@ -534,8 +556,8 @@ function blockDevice(ID) {
     write();
 }
 
-function banDevice(ID) {
-    const settings = get();
+export function banDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.ban) {
         settings.ban = [];
     }
@@ -544,15 +566,15 @@ function banDevice(ID) {
     write();
 }
 
-function removeDevice(IDorName) {
+export function removeDevice(IDorName: string): void {
     const device = getDeviceThrowIfNotExists(IDorName);
-    const settings = get();
+    const settings = getInternalSettings();
     delete settings.devices[device.ID];
 
     // Remove device from groups
     if (settings.groups) {
         const regex =
-            new RegExp(`^(${device.friendly_name}|${device.ID})(/(\\d|${utils.getEndpointNames().join('|')}))?$`);
+            new RegExp(`^(${device.friendlyName}|${device.ID})(/(\\d|${utils.endpointNames.join('|')}))?$`);
         for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
             group.devices = group.devices.filter((device) => !device.match(regex));
         }
@@ -561,13 +583,13 @@ function removeDevice(IDorName) {
     write();
 }
 
-function addGroup(name, ID=null) {
+export function addGroup(name: string, ID?: string): GroupSettings {
     utils.validateFriendlyName(name, true);
     if (getGroup(name) || getDevice(name)) {
         throw new Error(`friendly_name '${name}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (!settings.groups) {
         settings.groups = {};
     }
@@ -582,7 +604,7 @@ function addGroup(name, ID=null) {
         // ensure provided ID is not in use
         ID = ID.toString();
         if (settings.groups.hasOwnProperty(ID)) {
-            throw new Error(`group id '${ID}' is already in use`);
+            throw new Error(`Group ID '${ID}' is already in use`);
         }
     }
 
@@ -592,56 +614,50 @@ function addGroup(name, ID=null) {
     return getGroup(ID);
 }
 
-function groupHasDevice(group, keys) {
-    for (const device of group.devices) {
-        const index = keys.indexOf(device);
-        if (index != -1) {
-            return keys[index];
-        }
+function groupGetDevice(group: {devices?: string[]}, keys: string[]): string {
+    for (const device of group.devices ?? []) {
+        if (keys.includes(device)) return device;
     }
 
-    return false;
+    return null;
 }
 
-function addDeviceToGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function addDeviceToGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
 
     const group = settings.groups[groupID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    if (!groupHasDevice(group, keys)) {
+    if (!groupGetDevice(group, keys)) {
+        if (!group.devices) group.devices = [];
         group.devices.push(keys[0]);
         write();
     }
 }
 
-function removeDeviceFromGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeDeviceFromGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
     const group = settings.groups[groupID];
     if (!group.devices) {
-        group.devices = [];
+        return;
     }
 
-    const key = groupHasDevice(group, keys);
+    const key = groupGetDevice(group, keys);
     if (key) {
         group.devices = group.devices.filter((d) => d != key);
         write();
     }
 }
 
-function removeGroup(groupIDorName) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeGroup(IDorName: string): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
     delete settings.groups[groupID];
     write();
 }
 
-function changeEntityOptions(IDorName, newOptions) {
-    const settings = get();
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+    const settings = getInternalSettings();
     delete newOptions.friendly_name;
     delete newOptions.devices;
     if (getDevice(IDorName)) {
@@ -657,13 +673,13 @@ function changeEntityOptions(IDorName, newOptions) {
     write();
 }
 
-function changeFriendlyName(IDorName, newName) {
+export function changeFriendlyName(IDorName: string, newName: string): void {
     utils.validateFriendlyName(newName, true);
     if (getGroup(newName) || getDevice(newName)) {
         throw new Error(`friendly_name '${newName}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (getDevice(IDorName)) {
         settings.devices[getDevice(IDorName).ID].friendly_name = newName;
     } else if (getGroup(IDorName)) {
@@ -675,39 +691,18 @@ function changeFriendlyName(IDorName, newName) {
     write();
 }
 
-module.exports = {
-    validate,
-    get: getWithDefaults,
-    set,
-    apply,
-    getDevice,
-    getGroup,
-    getGroups,
-    getEntity,
-    whitelistDevice,
-    banDevice,
-    blockDevice,
-    addDevice,
-    removeDevice,
-    addGroup,
-    removeGroup,
-    addDeviceToGroup,
-    removeDeviceFromGroup,
-    changeEntityOptions,
-    changeFriendlyName,
-    schema,
-    reRead: () => {
-        _settings = null;
-        get();
-        _settingsWithDefaults = null;
-        getWithDefaults();
-    },
+export function reRead(): void {
+    _settings = null;
+    getInternalSettings();
+    _settingsWithDefaults = null;
+    get();
+}
 
-    // For tests only
-    _write: write,
-    _clear: () => {
+export const testing = {
+    write,
+    clear: (): void => {
         _settings = null;
         _settingsWithDefaults = null;
     },
-    _getDefaults: () => defaults,
+    defaults,
 };

lib/util/utils.ts

@@ -1,40 +1,36 @@
-const equals = require('fast-deep-equal/es6');
-const humanizeDuration = require('humanize-duration');
-const data = require('./data');
-const vm = require('vm');
-const fs = require('fs');
-const path = require('path');
-
-// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
-const xiaomiManufacturerID = [4151, 4447];
-const ikeaTradfriManufacturerID = [4476];
+import equals from 'fast-deep-equal/es6';
+import humanizeDuration from 'humanize-duration';
+import data from './data';
+import vm from 'vm';
+import fs from 'fs';
+import path from 'path';
 
 // construct a local ISO8601 string (instead of UTC-based)
 // Example:
 //  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
 //  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
-function toLocalISOString(dDate) {
-    const tzOffset = -dDate.getTimezoneOffset();
+function toLocalISOString(date: Date): string {
+    const tzOffset = -date.getTimezoneOffset();
     const plusOrMinus = tzOffset >= 0 ? '+' : '-';
-    const pad = function(num) {
+    const pad = (num: number): string => {
         const norm = Math.floor(Math.abs(num));
         return (norm < 10 ? '0' : '') + norm;
     };
 
-    return dDate.getFullYear() +
-        '-' + pad(dDate.getMonth() + 1) +
-        '-' + pad(dDate.getDate()) +
-        'T' + pad(dDate.getHours()) +
-        ':' + pad(dDate.getMinutes()) +
-        ':' + pad(dDate.getSeconds()) +
+    return date.getFullYear() +
+        '-' + pad(date.getMonth() + 1) +
+        '-' + pad(date.getDate()) +
+        'T' + pad(date.getHours()) +
+        ':' + pad(date.getMinutes()) +
+        ':' + pad(date.getSeconds()) +
         plusOrMinus + pad(tzOffset / 60) +
         ':' + pad(tzOffset % 60);
 }
 
-const endpointNames = [
+export const endpointNames = [
     'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
     'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
-    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay', 'usb',
     'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
     'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
     'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
@@ -45,23 +41,25 @@ const endpointNames = [
     'heat', 'cool', 'water', 'meter', 'wifi',
 ];
 
-function capitalize(s) {
+export function capitalize(s: string): string {
     return s[0].toUpperCase() + s.slice(1);
 }
 
-async function getZigbee2mqttVersion() {
-    return new Promise((resolve, reject) => {
-        const git = require('git-last-commit');
-        const packageJSON = require('../../package.json');
+export async function getZigbee2MQTTVersion(): Promise<{commitHash: string, version: string}> {
+    const git = await import('git-last-commit');
+    const packageJSON = await import('../..' + '/package.json');
+
+    return new Promise((resolve) => {
         const version = packageJSON.version;
 
-        git.getLastCommit((err, commit) => {
+        git.getLastCommit((err: Error, commit: {shortHash: string}) => {
             let commitHash = null;
 
             if (err) {
                 try {
-                    commitHash = require('../../.hash.json').hash;
+                    commitHash = fs.readFileSync(path.join(__dirname, '..', '..', 'dist', '.hash'), 'utf-8');
                 } catch (error) {
+                    /* istanbul ignore next */
                     commitHash = 'unknown';
                 }
             } else {
@@ -73,39 +71,22 @@ async function getZigbee2mqttVersion() {
     });
 }
 
-async function getDependencyVersion(depend) {
-    return new Promise((resolve, reject) => {
-        const packageJSON = require('../../node_modules/'+depend+'/package.json');
-        const version = packageJSON.version;
-        resolve({version});
-    });
+export async function getDependencyVersion(depend: string): Promise<{version: string}> {
+    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const version = packageJSON.version;
+    return {version};
 }
 
-function formatDate(date, type) {
-    let result;
-
-    switch (type) {
-    case 'ISO_8601':
-        result = new Date(date).toISOString();
-        break;
-    case 'ISO_8601_local':
-        result = toLocalISOString(new Date(date));
-        break;
-    case 'epoch':
-        result = date;
-        break;
-    case 'relative':
-        // https://github.com/EvanHahn/HumanizeDuration.js#options
-        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
-        break;
-    default:
-        throw new Error(`Unsupported type '${type}'`);
+export function formatDate(time: number, type: 'ISO_8601' | 'ISO_8601_local' | 'epoch' | 'relative'): string | number {
+    if (type === 'ISO_8601') return new Date(time).toISOString();
+    else if (type === 'ISO_8601_local') return toLocalISOString(new Date(time));
+    else if (type === 'epoch') return time;
+    else { // relative
+        return humanizeDuration(Date.now() - time, {language: 'en', largest: 2, round: true}) + ' ago';
     }
-
-    return result;
 }
 
-function objectHasProperties(object, properties) {
+export function objectHasProperties(object: {[s: string]: unknown}, properties: string[]): boolean {
     for (const property of properties) {
         if (!object.hasOwnProperty(property)) {
             return false;
@@ -115,7 +96,7 @@ function objectHasProperties(object, properties) {
     return true;
 }
 
-function equalsPartial(object, expected) {
+export function equalsPartial(object: {[s: string]: unknown}, expected: {[s: string]: unknown}): boolean {
     for (const [key, value] of Object.entries(expected)) {
         if (!equals(object[key], value)) {
             return false;
@@ -125,12 +106,14 @@ function equalsPartial(object, expected) {
     return true;
 }
 
-function getObjectProperty(object, key, defaultValue) {
+export function getObjectProperty(object: {[s: string]: unknown}, key: string, defaultValue: unknown): unknown {
     return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
 }
 
-function getResponse(request, data, error) {
-    const response = {data, status: error ? 'error' : 'ok'};
+export function getResponse(request: {transaction: string}, data: unknown, error: string):
+    {data: unknown, status: string, error?: string, transaction?: string} {
+    const response: {data: unknown, status: string, error?: string, transaction?: string} =
+        {data, status: error ? 'error' : 'ok'};
     if (error) response.error = error;
     if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
         response.transaction = request.transaction;
@@ -138,7 +121,7 @@ function getResponse(request, data, error) {
     return response;
 }
 
-function parseJSON(value, failedReturnValue) {
+export function parseJSON(value: string, failedReturnValue: unknown): unknown {
     try {
         return JSON.parse(value);
     } catch (e) {
@@ -146,7 +129,7 @@ function parseJSON(value, failedReturnValue) {
     }
 }
 
-function loadModuleFromText(moduleCode) {
+export function loadModuleFromText(moduleCode: string): unknown {
     const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
     const sandbox = {
         require: require,
@@ -160,15 +143,17 @@ function loadModuleFromText(moduleCode) {
         clearImmediate,
     };
     vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    /* eslint-disable-line */ // @ts-ignore
     return sandbox.module.exports;
 }
 
-function loadModuleFromFile(modulePath) {
+export function loadModuleFromFile(modulePath: string): unknown {
     const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
     return loadModuleFromText(moduleCode);
 }
 
-function* getExternalConvertersDefinitions(settings) {
+/* eslint-disable-next-line */
+export function* getExternalConvertersDefinitions(settings: any): any {
     const externalConverters = settings.get().external_converters;
 
     for (const moduleName of externalConverters) {
@@ -190,7 +175,7 @@ function* getExternalConvertersDefinitions(settings) {
     }
 }
 
-function removeNullPropertiesFromObject(obj) {
+export function removeNullPropertiesFromObject(obj: KeyValue): void {
     for (const key of Object.keys(obj)) {
         const value = obj[key];
         if (value == null) {
@@ -201,7 +186,8 @@ function removeNullPropertiesFromObject(obj) {
     }
 }
 
-function getKey(object, value, fallback, convertTo) {
+export function getKey(
+    object: KeyValue, value: unknown, fallback: unknown, convertTo: (v: unknown) => unknown): unknown {
     for (const key in object) {
         if (object[key]===value) {
             return convertTo ? convertTo(key) : key;
@@ -211,12 +197,13 @@ function getKey(object, value, fallback, convertTo) {
     return fallback;
 }
 
-function toNetworkAddressHex(value) {
+export function toNetworkAddressHex(value: number): string {
     const hex = value.toString(16);
     return `0x${'0'.repeat(4 - hex.length)}${hex}`;
 }
 
-function toSnakeCase(value) {
+// eslint-disable-next-line
+export function toSnakeCase(value: string | KeyValue): any {
     if (typeof value === 'object') {
         value = {...value};
         for (const key of Object.keys(value)) {
@@ -232,7 +219,7 @@ function toSnakeCase(value) {
     }
 }
 
-function validateFriendlyName(name, throwFirstError=false) {
+export function validateFriendlyName(name: string, throwFirstError=false): string[] {
     const errors = [];
     for (const endpointName of endpointNames) {
         if (name.toLowerCase().endsWith('/' + endpointName)) {
@@ -255,43 +242,48 @@ function validateFriendlyName(name, throwFirstError=false) {
 
     return errors;
 }
-function sleep(seconds) {
+
+export function sleep(seconds: number): Promise<void> {
     return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
 }
 
-function sanitizeImageParameter(parameter) {
+export function sanitizeImageParameter(parameter: string): string {
     const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
     let sanitized = parameter;
     replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
     return sanitized;
 }
 
-module.exports = {
-    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
-    secondsToMilliseconds: (seconds) => seconds * 1000,
-    getZigbee2mqttVersion,
-    getDependencyVersion,
-    objectHasProperties,
-    toSnakeCase,
-    sleep,
-    getObjectProperty,
-    getEndpointNames: () => endpointNames,
-    isXiaomiDevice: (device) => {
-        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
-            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
-    },
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
-    formatDate: (date, type) => formatDate(date, type),
-    equalsPartial,
-    getResponse,
-    capitalize,
-    toNetworkAddressHex,
-    parseJSON,
-    getExternalConvertersDefinitions,
-    validateFriendlyName,
-    loadModuleFromFile,
-    loadModuleFromText,
-    getKey,
-    sanitizeImageParameter,
-    removeNullPropertiesFromObject,
-};
+export function isAvailabilityEnabledForDevice(rd: ResolvedDevice, settings: Settings): boolean {
+    if (!settings.experimental.availability_new) return false;
+
+    if (rd.settings.hasOwnProperty('availability')) {
+        return !!rd.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(rd.name) || passlist.includes(rd.device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(rd.name) && !blocklist.includes(rd.device.ieeeAddr);
+}
+
+export function isXiaomiDevice(device: Device): boolean {
+    const xiaomiManufacturerID = [4151, 4447];
+    return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+        (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+}
+
+export function isIkeaTradfriDevice(device: Device): boolean {
+    return [4476].includes(device.manufacturerID);
+}
+
+export const hours = (hours: number): number => 1000 * 60 * 60 * hours;
+export const minutes = (minutes: number): number => 1000 * 60 * minutes;
+export const seconds = (seconds: number): number => 1000 * seconds;

lib/util/yaml.js

@@ -1,36 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-const equals = require('fast-deep-equal/es6');
-
-function read(file) {
-    try {
-        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-    } catch (error) {
-        if (error.name === 'YAMLException') {
-            error.file = file;
-        }
-
-        throw error;
-    }
-}
-
-function readIfExists(file, default_=null) {
-    return fs.existsSync(file) ? read(file) : default_;
-}
-
-function writeIfChanged(file, content) {
-    const before = readIfExists(file);
-    if (!equals(before, content)) {
-        fs.writeFileSync(file, yaml.safeDump(content));
-    }
-}
-
-function updateIfChanged(file, key, value) {
-    const content = read(file);
-    if (content[key] !== value) {
-        content[key] = value;
-        writeIfChanged(file, content);
-    }
-}
-
-module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/util/yaml.ts

@@ -0,0 +1,35 @@
+import yaml from 'js-yaml';
+import fs from 'fs';
+import equals from 'fast-deep-equal/es6';
+
+export function read(file: string): KeyValue {
+    try {
+        /* eslint-disable-line */ // @ts-ignore
+        return yaml.load(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+export function readIfExists(file: string, default_?: KeyValue): KeyValue {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+export function writeIfChanged(file: string, content: KeyValue): void {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.dump(content));
+    }
+}
+
+export function updateIfChanged(file: string, key: string, value: KeyValue): void {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}

lib/zigbee.js

@@ -9,7 +9,6 @@ const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const stringify = require('json-stable-stringify-without-jsonify');
 
-const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
 class Zigbee extends events.EventEmitter {
@@ -60,10 +59,11 @@ class Zigbee extends events.EventEmitter {
             herdsmanSettings.network.panID = newPanID;
         }
 
+        let startResult;
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
             this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, logger);
-            await this.herdsman.start();
+            startResult = await this.herdsman.start();
         } catch (error) {
             logger.error(`Error while starting zigbee-herdsman`);
             throw error;
@@ -78,8 +78,9 @@ class Zigbee extends events.EventEmitter {
         this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
         this.herdsman.on('message', (data) => this.emit('event', 'message', data));
         this.herdsman.on('permitJoinChanged', (data) => this.emit('permitJoinChanged', data));
+        this.herdsman.on('lastSeenChanged', (data) => this.emit('lastSeenChanged', data));
 
-        logger.info('zigbee-herdsman started');
+        logger.info(`zigbee-herdsman started (${startResult})`);
         logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
         logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
@@ -109,6 +110,8 @@ class Zigbee extends events.EventEmitter {
             await this.herdsman.setTransmitPower(transmitPower);
             logger.info(`Set transmit power to '${transmitPower}'`);
         }
+
+        return startResult;
     }
 
     async getCoordinatorVersion() {
@@ -215,7 +218,7 @@ class Zigbee extends events.EventEmitter {
                 };
             }
 
-            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            let endpointKey = utils.endpointNames.find((p) => key.endsWith(`/${p}`));
             const endpointByNumber = key.match(keyEndpointByNumber);
             if (!endpointKey && endpointByNumber) {
                 endpointKey = Number(endpointByNumber[1]);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.19.1",
+  "version": "1.21.1",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -20,9 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "test-with-coverage": "jest test --coverage",
-    "eslint": "node_modules/.bin/eslint lib/",
+    "build": "tsc && node index.js writehash",
+    "build-watch": "tsc --watch",
+    "eslint": "eslint lib/ --max-warnings=0",
     "start": "node index.js",
+    "test-with-coverage": "jest test --coverage",
     "test": "jest test",
     "test-watch": "jest test --watch"
   },
@@ -33,35 +35,49 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^6.12.4",
-    "debounce": "^1.2.0",
+    "ajv": "^8.6.2",
+    "debounce": "^1.2.1",
     "deep-object-diff": "^1.1.0",
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.1.2",
-    "git-last-commit": "^1.0.0",
-    "humanize-duration": "^3.23.1",
-    "js-yaml": "^3.14.0",
-    "json-stable-stringify-without-jsonify": "=1.0.1",
+    "git-last-commit": "^1.0.1",
+    "humanize-duration": "^3.27.0",
+    "js-yaml": "^4.1.0",
+    "json-stable-stringify-without-jsonify": "^1.0.1",
     "mkdir-recursive": "^0.4.0",
-    "moment": "^2.27.0",
-    "mqtt": "4.2.1",
+    "moment": "^2.29.1",
+    "mqtt": "4.2.8",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
-    "semver": "^7.3.2",
+    "semver": "^7.3.5",
     "serve-static": "^1.14.1",
+    "source-map-support": "^0.5.19",
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
-    "ws": "^7.3.1",
-    "zigbee-herdsman": "0.13.111",
-    "zigbee-herdsman-converters": "14.0.168",
-    "zigbee2mqtt-frontend": "0.4.16"
+    "ws": "^8.2.1",
+    "zigbee-herdsman": "0.13.138",
+    "zigbee-herdsman-converters": "14.0.238",
+    "zigbee2mqtt-frontend": "0.6.0"
   },
   "devDependencies": {
-    "eslint": "*",
-    "eslint-config-google": "*",
-    "eslint-plugin-jest": "*",
-    "jest": "*",
-    "tmp": "*"
+    "@babel/core": "^7.15.0",
+    "@babel/preset-env": "^7.15.0",
+    "@babel/preset-typescript": "^7.15.0",
+    "@types/debounce": "^1.2.0",
+    "@types/humanize-duration": "^3.25.1",
+    "@types/jest": "^27.0.1",
+    "@types/js-yaml": "^4.0.3",
+    "@types/object-assign-deep": "^0.4.0",
+    "@types/ws": "^7.4.7",
+    "@typescript-eslint/eslint-plugin": "^4.29.3",
+    "@typescript-eslint/parser": "^4.29.3",
+    "babel-jest": "^27.1.0",
+    "eslint": "^7.32.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-jest": "^24.4.0",
+    "jest": "^27.1.0",
+    "tmp": "^0.2.1",
+    "typescript": "^4.4.2"
   },
   "jest": {
     "coverageThreshold": {
@@ -73,7 +89,8 @@
       }
     },
     "collectCoverageFrom": [
-      "lib/**/*.js"
+      "lib/**/*.js",
+      "lib/**/*.ts"
     ]
   },
   "bin": {

test/availability.test.js

@@ -6,38 +6,41 @@ zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
-
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-
-const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const mocks = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Availability', () => {
     let controller;
+    let extension;
 
-    function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Availability');
+        await controller.enableDisableExtension(true, 'Availability');
+        extension = controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
         jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10);
         controller = new Controller(jest.fn(), jest.fn());
-        mocksClear.forEach((m) => m.mockClear());
         await controller.start();
         await flushPromises();
     });
 
-    afterEach(async () => {
-        await controller.stop();
-        await flushPromises();
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['advanced', 'availability_timeout'], 10);
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     })
 
     it('Should publish availabilty on startup', async () => {
@@ -153,7 +156,7 @@ describe('Availability', () => {
 
     it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
         MQTT.publish.mockClear();
-        getExtension().state = {};
+        extension.state = {};
         const payload = {device: zigbeeHerdsman.devices.bulb_color};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
@@ -199,11 +202,7 @@ describe('Availability', () => {
     it('Should not ping devices on blocklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -213,11 +212,7 @@ describe('Availability', () => {
     it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -227,11 +222,7 @@ describe('Availability', () => {
     it('Should not ping devices on blocklist by IEEE address', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -254,11 +245,7 @@ describe('Availability', () => {
     it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -268,11 +255,7 @@ describe('Availability', () => {
     it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -282,11 +265,7 @@ describe('Availability', () => {
     it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -295,13 +274,9 @@ describe('Availability', () => {
 
     it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        getExtension().state[device.ieeeAddr] = false;
+        extension.state[device.ieeeAddr] = false;
         settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(0);
@@ -311,7 +286,7 @@ describe('Availability', () => {
 
     it('Should not read when device has no modelID and reconnects', async () => {
         const device = zigbeeHerdsman.devices.nomodel;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -320,7 +295,7 @@ describe('Availability', () => {
 
     it('Should not read when device has is unsupported', async () => {
         const device = zigbeeHerdsman.devices.unsupported_router;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -382,13 +357,9 @@ describe('Availability', () => {
 
     it('Should republish existing state on MQTT connected', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        await controller.stop();
-        await flushPromises();
         MQTT.publish.mockClear();
-        controller = new Controller(jest.fn(), jest.fn());
-        getExtension().state[device.ieeeAddr] = false;
-        await controller.start();
-        await flushPromises();
+        extension.state[device.ieeeAddr] = false;
+        await extension.onMQTTConnected();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color/availability',
           'offline',

test/availabilityNew.test.ts

@@ -0,0 +1,277 @@
+import data from './stub/data';
+import logger from './stub/logger';
+import MQTT from './stub/mqtt';
+import zigbeeHerdsman from './stub/zigbeeHerdsman';
+
+import * as utils from '../lib/util/utils';
+import * as settings from '../lib/util/settings';
+import Controller from '../lib/controller';
+import flushPromises from './lib/flushPromises';
+
+const devices = zigbeeHerdsman.devices;
+const mocks = [MQTT.publish, logger.warn, logger.debug];
+
+zigbeeHerdsman.returnDevices.concat(
+    [devices.bulb_color.ieeeAddr, devices.bulb_color_2.ieeeAddr, devices.coordinator.ieeeAddr, devices.remote])
+
+describe('Availability', () => {
+    let controller;
+
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'AvailabilityNew');
+        await controller.enableDisableExtension(true, 'AvailabilityNew');
+    }
+
+    const advancedTime = async (value) => {
+        jest.setSystemTime(Date.now() + value);
+        jest.advanceTimersByTime(value);
+        await flushPromises();
+    }
+
+    beforeAll(async () => {
+        jest.spyOn(utils, 'sleep').mockImplementation(async (seconds: number) => {});
+        jest.useFakeTimers('modern');
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['experimental', 'availability_new'], true);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        jest.useFakeTimers('modern').setSystemTime(utils.minutes(1));
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['experimental', 'availability_new'], true);
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], false);
+        // @ts-ignore
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+        // @ts-ignore
+        Object.values(devices).forEach((d) => d.ping.mockClear());
+    });
+
+    afterEach(async () => {
+        // @ts-ignore
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+    })
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should publish availabilty on startup for device where it is enabled for', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bulb_color_2/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish offline for active device when not seen for 10 minutes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Shouldnt do anything for a device when availability: false is set for device', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2}); // Coverage satisfaction
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish offline for passive device when not seen for 25 hours', async () => {
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for active device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should ping again when first ping fails', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        devices.bulb_color.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        devices.bulb_color.lastSeen = Date.now() + utils.minutes(10);
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(2);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(2, false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for passive device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.hours(24));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.remote});
+
+        await advancedTime(utils.hours(25));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        devices.remote.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        await advancedTime(utils.hours(3));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should immediately mark device as online when it lastSeen changes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(15));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+
+        devices.bulb_color.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should allow to change availability timeout via device options', async () => {
+        settings.set(['devices', '0x000b57fffec6a5b3', 'availability'], {timeout: 40});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(17));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to change availability timeout via avaiability options', async () => {
+        settings.set(['availability'], {active: {timeout: 30}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should stop pinging device when it leaves', async () => {
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(9));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: devices.bulb_color.ieeeAddr});
+        await flushPromises();
+
+        await advancedTime(utils.minutes(3));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to be disabled', async () => {
+        settings.set(['availability'], false);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should retrieve device state when it reconnects', async () => {
+        MQTT.publish.mockClear();
+
+        const endpoint = devices.bulb_color.getEndpoint(1);
+        endpoint.read.mockClear();
+
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        await advancedTime(utils.seconds(1));
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+        await advancedTime(utils.seconds(2));
+
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
+            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);
+
+        // Should stop when one request fails
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        endpoint.read.mockImplementationOnce(() => {throw new Error('')});
+        await advancedTime(utils.seconds(3));
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
+    });
+
+    it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
+        settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Deprecated - should allow to pass certain devices via availability_passlist', async () => {
+        settings.set(['advanced', 'availability_passlist'], [devices.bulb_color_2.ieeeAddr]);
+        settings.changeEntityOptions(devices.bulb_color_2.ieeeAddr, {availability: null});
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Deprecated - should allow to enable via availability_timeout', async () => {
+        settings.set(['availability'], false);
+        settings.set(['advanced', 'availability_timeout'], 60);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+});

test/bind.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const stringify = require('json-stable-stringify-without-jsonify');
 jest.mock('debounce', () => jest.fn(fn => fn));
 const debounce = require('debounce');
@@ -12,7 +12,7 @@ const debounce = require('debounce');
 describe('Bind', () => {
     let controller;
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -23,22 +23,34 @@ describe('Bind', () => {
         }
     }
 
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Bind');
+        await controller.enableDisableExtension(true, 'Bind');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
         zigbeeHerdsman.groups.group_1.members = [];
         zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
         zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
         zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
         debounce.mockClear();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
         MQTT.publish.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Should bind to device and configure reporting', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);

test/configure.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const stringify = require('json-stable-stringify-without-jsonify');
 
@@ -12,13 +12,14 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Configure', () => {
     let controller;
+    let coordinatorEndpoint;
 
-    expectRemoteConfigured = () => {
+    const expectRemoteConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(2);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
 
         const endpoint2 = device.getEndpoint(2);
         expect(endpoint2.write).toHaveBeenCalledTimes(1);
@@ -26,7 +27,7 @@ describe('Configure', () => {
         expect(device.meta.configured).toBe(1);
     }
 
-    expectBulbConfigured = () => {
+    const expectBulbConfigured = () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.read).toHaveBeenCalledTimes(2);
@@ -34,19 +35,19 @@ describe('Configure', () => {
         expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorTempPhysicalMin', 'colorTempPhysicalMax' ]);
     }
 
-    expectBulbNotConfigured = () => {
+    const expectBulbNotConfigured = () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.read).toHaveBeenCalledTimes(0);
     }
 
-    expectRemoteNotConfigured = () => {
+    const expectRemoteNotConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(0);
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -55,18 +56,30 @@ describe('Configure', () => {
         }
     }
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Configure');
+        await controller.enableDisableExtension(true, 'Configure');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
-        mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
     });
 
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+     coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Should configure Router on startup', async () => {
         expectBulbConfigured();
     });

test/controller.test.js

@@ -6,7 +6,7 @@ const path = require('path');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const tmp = require('tmp');
 const mocksClear = [
     zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
@@ -20,6 +20,10 @@ describe('Controller', () => {
     let controller;
     let mockExit;
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
         mockExit = jest.fn();
@@ -30,6 +34,10 @@ describe('Controller', () => {
         data.writeDefaultState();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Start controller', async () => {
         await controller.start();
         expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
@@ -132,14 +140,12 @@ describe('Controller', () => {
     });
 
     it('Log when MQTT client is unavailable', async () => {
-        jest.useFakeTimers();
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
         controller.mqtt.client.reconnecting = true;
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
         controller.mqtt.client.reconnecting = false;
     });
@@ -623,4 +629,13 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
     });
+
+    it('Should disable legacy options on new network start', async () => {
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        expect(settings.get().advanced.legacy_api).toBeTruthy();
+        zigbeeHerdsman.start.mockReturnValueOnce('reset');
+        await controller.start();
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().advanced.legacy_api).toBeFalsy();
+    });
 });

test/externalConverters.test.js

@@ -6,7 +6,7 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const path = require('path');
 const fs = require('fs');
 
@@ -41,20 +41,32 @@ jest.mock(
 describe('Loads external converters', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'ExternalConverters');
+        await controller.enableDisableExtension(true, 'ExternalConverters');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
         await flushPromises();
-        mocksClear.forEach((m) => m.mockClear());
+    });
+
+    beforeEach(async () => {
         data.writeDefaultConfiguration();
-        data.writeEmptyState();
         settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Does not load external converters', async () => {
         settings.set(['external_converters'], []);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
     });
 
@@ -62,9 +74,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter.js']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
             mock: true,
@@ -82,9 +92,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1,
@@ -110,9 +118,7 @@ describe('Loads external converters', () => {
 
     it('Loads external converters from package', async () => {
         settings.set(['external_converters'], ['mock-external-converter-module']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
             mock: true
@@ -121,9 +127,7 @@ describe('Loads external converters', () => {
 
     it('Loads multiple external converters from package', async () => {
         settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1

test/externalExtension.test.js

@@ -6,8 +6,7 @@ const path = require('path');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
-const tmp = require('tmp');
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [
     zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
     MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
@@ -20,12 +19,24 @@ const unlinkSyncSpy = jest.spyOn(fs, 'unlinkSync');
 
 describe('User extensions', () => {
     let controller;
-    let mockExit;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
 
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
-        mockExit = jest.fn();
-        controller = new Controller(jest.fn(), mockExit);
+        controller = new Controller(jest.fn(), jest.fn());
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
         settings.reRead();

test/frontend.test.js

@@ -5,7 +5,7 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 jest.spyOn(process, 'exit').mockImplementation(() => {});
 
@@ -31,6 +31,7 @@ const mockWS = {
             cb(mockWSocket)
         }),
         emit: jest.fn(),
+        close: jest.fn(),
     },
     variables: {},
     events: {},
@@ -70,6 +71,10 @@ jest.mock('ws', () => ({
 describe('Frontend', () => {
     let controller;
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
     beforeEach(async () => {
         mockWS.implementation.clients = [];
         data.writeDefaultConfiguration();
@@ -80,6 +85,10 @@ describe('Frontend', () => {
         zigbeeHerdsman.devices.bulb.linkquality = 10;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     afterEach(async() => {
         delete zigbeeHerdsman.devices.bulb.linkquality;
     });
@@ -92,15 +101,16 @@ describe('Frontend', () => {
 
         const mockWSClient = {
             implementation: {
-                close: jest.fn(),
+                terminate: jest.fn(),
                 send: jest.fn(),
             },
             events: {},
         };
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await controller.stop();
-        expect(mockWSClient.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockWSClient.implementation.terminate).toHaveBeenCalledTimes(1);
         expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockWS.implementation.close).toHaveBeenCalledTimes(1);
     });
 
     it('Websocket interaction', async () => {
@@ -125,7 +135,7 @@ describe('Frontend', () => {
         // Message
         MQTT.publish.mockClear();
         mockWSClient.implementation.send.mockClear();
-        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}), false)
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -134,9 +144,9 @@ describe('Frontend', () => {
             { retain: false, qos: 0 },
             expect.any(Function)
         );
-        mockWSClient.events.message(undefined);
-        mockWSClient.events.message("");
-        mockWSClient.events.message(null);
+        mockWSClient.events.message(undefined, false);
+        mockWSClient.events.message("", false);
+        mockWSClient.events.message(null, false);
         await flushPromises();
 
         // Received message on socket
@@ -146,7 +156,7 @@ describe('Frontend', () => {
         // Shouldnt set when not ready
         mockWSClient.implementation.send.mockClear();
         mockWSClient.implementation.readyState = 'close';
-        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}), false)
         expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(0);
 
         // Send last seen on connect

test/group.test.js

@@ -13,28 +13,42 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45724');
 
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const settings = require('../lib/util/settings');
 
 describe('Groups', () => {
     let controller;
 
-    beforeEach(() => {
-        data.writeEmptyState();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Groups');
+        await controller.enableDisableExtension(true, 'Groups');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(() => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
         settings.reRead();
         MQTT.publish.mockClear();
         zigbeeHerdsman.groups.gledopto_group.command.mockClear();
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+        controller.state.state = {};
     })
 
     it('Apply group updates add', async () => {
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
         zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -46,8 +60,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
@@ -58,8 +71,7 @@ describe('Groups', () => {
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
         logger.error.mockClear();
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(logger.error).toHaveBeenCalledWith(`Failed to remove 'bulb_color' from 'group_1'`);
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([endpoint]);
     });
@@ -70,32 +82,28 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
     it('Add non standard endpoint to group with name', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
     });
 
     it('Add non standard endpoint to group with number', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
     });
 
     it('Shouldnt crash on non-existing devices', async () => {
         logger.error.mockClear();
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
@@ -103,8 +111,7 @@ describe('Groups', () => {
     it('Should resolve device friendly names', async () => {
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'friendly_name'], 'bulb_friendly_name');
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb_friendly_name', 'bulb_color']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -117,8 +124,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -132,8 +138,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -146,8 +151,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -159,8 +163,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
@@ -175,8 +178,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -190,8 +192,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -204,8 +205,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -218,8 +218,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -232,8 +231,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -246,8 +244,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -261,8 +258,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -270,8 +266,7 @@ describe('Groups', () => {
     });
 
     it('Legacy api: Log when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
         await flushPromises();
@@ -279,8 +274,7 @@ describe('Groups', () => {
     });
 
     it('Legacy api: Log when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
         await flushPromises();
@@ -293,8 +287,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -310,18 +303,19 @@ describe('Groups', () => {
         const device1 = zigbeeHerdsman.devices.bulb_2;
         const device2 = zigbeeHerdsman.devices.bulb_color_2;
         const group = zigbeeHerdsman.groups.group_tradfri_remote;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(6);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/ha_discovery_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/switch_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -330,8 +324,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
@@ -348,8 +341,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
@@ -375,8 +367,7 @@ describe('Groups', () => {
 
     it('Should publish state of device with endpoint name', async () => {
         const group = zigbeeHerdsman.groups.gledopto_group;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/gledopto_group/set', stringify({state: 'ON'}));
@@ -390,8 +381,7 @@ describe('Groups', () => {
 
     it('Should publish state of group when specific state of specific endpoint is changed', async () => {
         const group = zigbeeHerdsman.groups.gledopto_group;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/GLEDOPTO_2ID/set', stringify({state_cct: 'ON'}));
@@ -408,8 +398,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
@@ -425,8 +414,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -447,8 +435,7 @@ describe('Groups', () => {
             '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
             '3': {friendly_name: 'group_3', retain: false, devices: []}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
@@ -470,8 +457,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -495,8 +481,7 @@ describe('Groups', () => {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
             '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -520,8 +505,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -547,8 +531,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
 
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF', color_temp: 200}));
@@ -575,8 +558,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -597,8 +579,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({transaction: "123", group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -618,7 +599,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
+        await resetExtension();
         endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
         await flushPromises();
         MQTT.publish.mockClear();
@@ -640,8 +621,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
         await flushPromises();
@@ -660,8 +640,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
@@ -680,8 +659,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
@@ -703,8 +681,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -724,8 +701,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color', skip_disable_reporting: true}));
         await flushPromises();
@@ -745,8 +721,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -766,8 +741,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
         await flushPromises();
@@ -787,8 +761,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
         await flushPromises();
@@ -808,8 +781,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
         await flushPromises();
@@ -829,8 +801,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
         await flushPromises();
@@ -845,8 +816,7 @@ describe('Groups', () => {
     });
 
     it('Error when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
@@ -860,8 +830,7 @@ describe('Groups', () => {
     });
 
     it('Error when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
@@ -873,4 +842,30 @@ describe('Groups', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
+
+    it('Should only include relevant properties when publishing member states', async () => {
+        const bulbColor = zigbeeHerdsman.devices.bulb_color;
+        const bulbColorTemp = zigbeeHerdsman.devices.bulb;
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(bulbColor.getEndpoint(1));
+        group.members.push(bulbColorTemp.getEndpoint(1));
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [bulbColor.ieeeAddr, bulbColorTemp.ieeeAddr]}});
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 50}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color: {x: 0.5, y: 0.3}}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":548}), {"retain": true, qos: 0}, expect.any(Function));
+    });
 });

test/homeassistant.test.js

@@ -3,7 +3,7 @@ const settings = require('../lib/util/settings');
 const stringify = require('json-stable-stringify-without-jsonify');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
 const fs = require('fs');
@@ -13,10 +13,11 @@ const HomeAssistant = require('../lib/extension/homeassistant');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 describe('HomeAssistant extension', () => {
+    let version;
+
     beforeEach(async () => {
-        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
-        this.version = `Zigbee2MQTT ${this.version.version}`;
-        jest.useRealTimers();
+        version = await require('../lib/util/utils').getZigbee2MQTTVersion();
+        version = `Zigbee2MQTT ${version.version}`;
         data.writeDefaultConfiguration();
         settings.reRead();
         data.writeEmptyState();
@@ -24,6 +25,14 @@ describe('HomeAssistant extension', () => {
         settings.set(['homeassistant'], true);
     });
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
@@ -44,16 +53,72 @@ describe('HomeAssistant extension', () => {
         expect(duplicated).toHaveLength(0);
     });
 
-    it('Should discover devices', async () => {
-        controller = new Controller(false);
+    it('Should discover devices and groups', async () => {
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
         await flushPromises();
 
+        payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version":version,
+            },
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version":version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "name":"ha_discovery_group",
+            "payload_off":"OFF",
+            "payload_on":"ON",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "unique_id":"9_switch_zigbee2mqtt",
+            "value_template":"{{ value_json.state }}"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/switch/1221051039810110150109113116116_9/switch/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -62,7 +127,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -79,6 +144,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -87,7 +153,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -104,6 +170,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -112,7 +179,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -129,6 +196,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'battery',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.battery }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -137,7 +205,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -153,7 +221,9 @@ describe('HomeAssistant extension', () => {
 
         payload = {
             'icon': 'mdi:signal',
+            'enabled_by_default': false,
             'unit_of_measurement': 'lqi',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.linkquality }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -162,7 +232,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -177,11 +247,7 @@ describe('HomeAssistant extension', () => {
         );
 
         payload = {
-            "availability":[
-                {
-                    "topic":"zigbee2mqtt/bridge/state"
-                }
-            ],
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
             "command_topic":"zigbee2mqtt/wall_switch_double/left/set",
             "device":{
                 "identifiers":[
@@ -190,7 +256,7 @@ describe('HomeAssistant extension', () => {
                 "manufacturer":"Xiaomi",
                 "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
                 "name":"wall_switch_double",
-                "sw_version":this.version
+                "sw_version":version
             },
             "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
             "name":"wall_switch_double_left",
@@ -209,11 +275,7 @@ describe('HomeAssistant extension', () => {
         );
 
         payload = {
-            "availability":[
-                {
-                    "topic":"zigbee2mqtt/bridge/state"
-                }
-            ],
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
             "command_topic":"zigbee2mqtt/wall_switch_double/right/set",
             "device":{
                 "identifiers":[
@@ -222,7 +284,7 @@ describe('HomeAssistant extension', () => {
                 "manufacturer":"Xiaomi",
                 "model":"Aqara double key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG03LM)",
                 "name":"wall_switch_double",
-                "sw_version":this.version
+                "sw_version":version
             },
             "json_attributes_topic":"zigbee2mqtt/wall_switch_double",
             "name":"wall_switch_double_right",
@@ -241,11 +303,7 @@ describe('HomeAssistant extension', () => {
         );
 
         payload = {
-            "availability":[
-                {
-                    "topic":"zigbee2mqtt/bridge/state"
-                }
-            ],
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
             "brightness":true,
             "brightness_scale":254,
             "color_mode": true,
@@ -260,7 +318,7 @@ describe('HomeAssistant extension', () => {
                 "manufacturer":"IKEA",
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "name":"bulb",
-                "sw_version":this.version,
+                "sw_version":version,
             },
             "effect":true,
             "effect_list":[
@@ -295,7 +353,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -304,6 +362,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -312,7 +371,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -329,6 +388,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -337,7 +397,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -354,6 +414,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -362,7 +423,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -401,7 +462,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -410,6 +471,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -437,6 +499,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -444,7 +507,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'custom model',
                 'manufacturer': 'Not from Xiaomi',
             },
@@ -477,7 +540,7 @@ describe('HomeAssistant extension', () => {
             },
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -493,7 +556,7 @@ describe('HomeAssistant extension', () => {
               "manufacturer": "Xiaomi",
               "model": "Aqara single key wired wall switch without neutral wire. Doesn't work as a router and doesn't support power meter (QBKG04LM)",
               "name": "my_switch",
-              "sw_version": this.version
+              "sw_version": version
             },
             "json_attributes_topic": "zigbee2mqtt/my_switch",
             "name": "my_light_name_override",
@@ -520,7 +583,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         await flushPromises();
@@ -538,7 +601,7 @@ describe('HomeAssistant extension', () => {
             retain: false,
         })
 
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         await flushPromises();
@@ -549,7 +612,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover devices with fan', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -559,16 +622,18 @@ describe('HomeAssistant extension', () => {
             "state_topic":"zigbee2mqtt/fan",
             "state_value_template":"{{ value_json.fan_state }}",
             "command_topic":"zigbee2mqtt/fan/set/fan_state",
-            "speed_state_topic":"zigbee2mqtt/fan",
-            "speed_command_topic":"zigbee2mqtt/fan/set/fan_mode",
-            "speed_value_template":"{{ value_json.fan_mode }}",
-            "speeds":[
-               "low",
-               "medium",
-               "high",
-               "on",
+            "percentage_state_topic":"zigbee2mqtt/fan",
+            "percentage_command_topic":"zigbee2mqtt/fan/set/fan_mode",
+            "percentage_value_template":"{{ {'off':0, 'low':1, 'medium':2, 'high':3, 'on':4}[value_json.fan_mode] | default('None') }}",
+            "percentage_command_template":"{{ {0:'off', 1:'low', 2:'medium', 3:'high', 4:'on'}[value] | default('') }}",
+            "preset_mode_state_topic":"zigbee2mqtt/fan",
+            "preset_mode_command_topic":"zigbee2mqtt/fan/set/fan_mode",
+            "preset_mode_value_template":"{{ value_json.fan_mode if value_json.fan_mode in ['smart'] else 'None' | default('None') }}",
+            "preset_modes":[
                "smart"
             ],
+            "speed_range_min":1,
+            "speed_range_max":4,
             "json_attributes_topic":"zigbee2mqtt/fan",
             "name":"fan",
             "unique_id":"0x0017880104e45548_fan_zigbee2mqtt",
@@ -577,7 +642,7 @@ describe('HomeAssistant extension', () => {
                   "zigbee2mqtt_0x0017880104e45548"
                ],
                "name":"fan",
-               "sw_version":this.version,
+               "sw_version":version,
                "model":"Universal wink enabled white ceiling fan premier remote control (99432)",
                "manufacturer":"Hampton Bay"
             },
@@ -593,7 +658,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover thermostat devices', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -613,7 +678,7 @@ describe('HomeAssistant extension', () => {
                "manufacturer":"TuYa",
                "model":"Radiator valve with thermostat (TS0601_thermostat)",
                "name":"TS0601_thermostat",
-               "sw_version":  this.version,
+               "sw_version":  version,
             },
             "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
             "hold_modes":[
@@ -653,7 +718,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover devices with cover_position', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -672,7 +737,7 @@ describe('HomeAssistant extension', () => {
             {
                 identifiers: [ 'zigbee2mqtt_0x0017880104e45551' ],
                 name: 'smart vent',
-                sw_version: this.version,
+                sw_version: version,
                 model: 'Smart vent (SV01)',
                 manufacturer: 'Keen Home'
             },
@@ -689,7 +754,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should discover devices with custom homeassistant_discovery_topic', async () => {
         settings.set(['advanced', 'homeassistant_discovery_topic'], 'my_custom_discovery_topic')
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -698,6 +763,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -706,7 +772,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -724,20 +790,20 @@ describe('HomeAssistant extension', () => {
     it('Should throw error when starting with attributes output', async () => {
         settings.set(['experimental', 'output'], 'attribute')
         expect(() => {
-            controller = new Controller(false);
-        }).toThrowError('Home Assitant integration is not possible with attribute output!');
+            const controller = new Controller(false);
+        }).toThrowError('Home Assistant integration is not possible with attribute output!');
     });
 
     it('Should warn when starting with cache_state false', async () => {
         settings.set(['advanced', 'cache_state'], false);
         logger.warn.mockClear();
-        controller = new Controller(false);
-        expect(logger.warn).toHaveBeenCalledWith("In order for HomeAssistant integration to work properly set `cache_state: true");
+        const controller = new Controller(false);
+        expect(logger.warn).toHaveBeenCalledWith("In order for Home Assistant integration to work properly set `cache_state: true");
     });
 
     it('Should set missing values to null', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/6987
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
@@ -756,7 +822,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should copy hue/saturtion to h/s if present', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -775,7 +841,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should not copy hue/saturtion if properties are missing', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -794,7 +860,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should not copy hue/saturtion if color is missing', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -813,7 +879,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldt discover when already discovered', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
@@ -827,7 +893,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover when not discovered yet', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         controller.extensions.find((e) => e.constructor.name === 'HomeAssistant').discovered = {};
@@ -840,6 +906,7 @@ describe('HomeAssistant extension', () => {
         const payloadHA = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -848,7 +915,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -864,7 +931,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt discover when device leaves', async () => {
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         controller.extensions.find((e) => e.constructor.name === 'HomeAssistant').discovered = {};
@@ -876,9 +943,8 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should send all status when home assistant comes online (default topic)', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         expect(MQTT.subscribe).toHaveBeenCalledWith('homeassistant/status');
         await flushPromises();
@@ -902,9 +968,8 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should send all status when home assistant comes online', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(MQTT.subscribe).toHaveBeenCalledWith('hass/status');
@@ -928,9 +993,8 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt send all status when home assistant comes offline', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -942,9 +1006,8 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt send all status when home assistant comes online with different topic', async () => {
-        jest.useFakeTimers();
         data.writeDefaultState();
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -957,7 +1020,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should discover devices with availability', async () => {
         settings.set(['advanced', 'availability_timeout'], 1)
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
 
         let payload;
@@ -966,6 +1029,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
@@ -974,7 +1038,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -990,7 +1054,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should clear discovery when device is removed', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1030,7 +1094,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should not clear discovery when unsupported device is removed', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1040,7 +1104,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should refresh discovery when device is renamed', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         await MQTT.events.message('homeassistant/device_automation/0x0017880104e45522/action_double/config', stringify({topic: 'zigbee2mqtt/weather_sensor/action'}));
@@ -1052,6 +1116,7 @@ describe('HomeAssistant extension', () => {
         const payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
@@ -1060,7 +1125,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor_renamed',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -1094,7 +1159,7 @@ describe('HomeAssistant extension', () => {
                         "zigbee2mqtt_0x0017880104e45522"
                     ],
                     "name":"weather_sensor_renamed",
-                    "sw_version": this.version,
+                    "sw_version": version,
                     "model":"Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)",
                     "manufacturer":"Xiaomi"
                 }
@@ -1105,7 +1170,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt refresh discovery when device is renamed and homeassistant_rename is false', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1122,6 +1187,7 @@ describe('HomeAssistant extension', () => {
         const payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
+            'state_class': 'measurement',
             'value_template': '{{ value_json.temperature }}',
             'state_topic': 'zigbee2mqtt/weather_sensor_renamed',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor_renamed',
@@ -1130,7 +1196,7 @@ describe('HomeAssistant extension', () => {
             'device': {
                 'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
                 'name': 'weather_sensor_renamed',
-                'sw_version': this.version,
+                'sw_version': version,
                 'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
                 'manufacturer': 'Xiaomi',
             },
@@ -1146,13 +1212,14 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover update_available sensor when device supports it', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         const payload = {
             "payload_on":true,
             "payload_off":false,
             "value_template":"{{ value_json.update_available}}",
+            "enabled_by_default": false,
             "state_topic":"zigbee2mqtt/bulb",
             "json_attributes_topic":"zigbee2mqtt/bulb",
             "name":"bulb update available",
@@ -1162,7 +1229,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x000b57fffec6a5b2"
                 ],
                 "name":"bulb",
-                'sw_version': this.version,
+                'sw_version': version,
                 "model":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)",
                 "manufacturer":"IKEA"
             },
@@ -1178,7 +1245,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should discover trigger when click is published', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
 
@@ -1205,7 +1272,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x0017880104e45520"
                 ],
                 "name":"button",
-                "sw_version": this.version,
+                "sw_version": version,
                 "model":"Aqara wireless switch (WXKG11LM)",
                 "manufacturer":"Xiaomi"
             }
@@ -1229,7 +1296,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x0017880104e45520"
                 ],
                 "name":"button",
-                "sw_version": this.version,
+                "sw_version": version,
                 "model":"Aqara wireless switch (WXKG11LM)",
                 "manufacturer":"Xiaomi"
             }
@@ -1330,11 +1397,11 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x0017880104e45520/action_double/config', expect.any(String), expect.any(Object), expect.any(Function));
     });
 
-    it('Should not discover device_automtation when disabled', async () => {
+    it('Should not discover device_automation when disabled', async () => {
         settings.set(['device_options'], {
             homeassistant: {device_automation: null},
         })
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1358,7 +1425,7 @@ describe('HomeAssistant extension', () => {
             friendly_name: 'weather_sensor',
             retain: false,
         })
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
 
@@ -1371,7 +1438,7 @@ describe('HomeAssistant extension', () => {
 
     it('Should disable Home Assistant legacy triggers', async () => {
         settings.set(['advanced', 'homeassistant_legacy_triggers'], false);
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
 
@@ -1399,7 +1466,7 @@ describe('HomeAssistant extension', () => {
                     "zigbee2mqtt_0x0017880104e45520"
                 ],
                 "name":"button",
-                "sw_version": this.version,
+                "sw_version": version,
                 "model":"Aqara wireless switch (WXKG11LM)",
                 "manufacturer":"Xiaomi"
             }
@@ -1430,7 +1497,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should republish payload to postfix topic with lightWithPostfix config', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1443,7 +1510,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Shouldnt crash in onPublishEntityState on group publish', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -1455,7 +1522,7 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should counter an action payload with an empty payload', async () => {
-        controller = new Controller(false);
+        const controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
@@ -1479,7 +1546,7 @@ describe('HomeAssistant extension', () => {
     it('Load Home Assistant mapping from external converters', async () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller(jest.fn(), jest.fn());
+        const controller = new Controller(jest.fn(), jest.fn());
         const ha = controller.extensions.find((e) => e.constructor.name === 'HomeAssistant');
         await controller.start();
         await flushPromises();
@@ -1498,10 +1565,37 @@ describe('HomeAssistant extension', () => {
     });
 
     it('Should clear outdated configs', async () => {
-        controller = new Controller(jest.fn(), jest.fn());
+        let controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
 
+        // Non-existing group -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/1221051039810110150109113116116_91231/light/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/light/1221051039810110150109113116116_91231/light/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Existing group -> dont clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/1221051039810110150109113116116_9/light/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Existing group with old topic structure (1.20.0) -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/9/light/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/light/9/light/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Existing group, non existing config ->  clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/light/1221051039810110150109113116116_9/switch/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/light/1221051039810110150109113116116_9/switch/config', null, {qos: 0, retain: true}, expect.any(Function));
+
         // Non-existing device -> clear
         MQTT.publish.mockClear();
         await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability: [{topic: 'zigbee2mqtt/bridge/state'}]}));
@@ -1563,4 +1657,78 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/device_automation/0x000b57fffec6a5b2/action_button_3_single/config', null, {qos: 0, retain: true}, expect.any(Function));
     });
+
+    it('Should not have Home Assistant legacy entity attributes when disabled', async () => {
+        settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+        const controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            'unit_of_measurement': '°C',
+            'device_class': 'temperature',
+            'state_class': 'measurement',
+            'value_template': '{{ value_json.temperature }}',
+            'state_topic': 'zigbee2mqtt/weather_sensor',
+            'name': 'weather_sensor_temperature',
+            'unique_id': '0x0017880104e45522_temperature_zigbee2mqtt',
+            'device': {
+                'identifiers': ['zigbee2mqtt_0x0017880104e45522'],
+                'name': 'weather_sensor',
+                'sw_version': version,
+                'model': 'Aqara temperature, humidity and pressure sensor (WSDCGQ11LM)',
+                'manufacturer': 'Xiaomi',
+            },
+            'availability': [{topic: 'zigbee2mqtt/bridge/state'}],
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should rediscover group when device is added to it', async () => {
+        const controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'ha_discovery_group', device: 'wall_switch_double/left'}));
+        await flushPromises();
+
+        const payload = {
+            "availability":[{"topic":"zigbee2mqtt/bridge/state"}],
+            "brightness":true,
+            "brightness_scale":254,
+            "color_mode":true,
+            "command_topic":"zigbee2mqtt/ha_discovery_group/set",
+            "device":{
+               "identifiers":["zigbee2mqtt_1221051039810110150109113116116_9"],
+               "name":"ha_discovery_group",
+               "sw_version":version,
+            },
+            "json_attributes_topic":"zigbee2mqtt/ha_discovery_group",
+            "max_mireds": 454,
+            "min_mireds": 250,
+            "name":"ha_discovery_group",
+            "schema":"json",
+            "state_topic":"zigbee2mqtt/ha_discovery_group",
+            "supported_color_modes":[
+               "xy",
+               "color_temp"
+            ],
+            "unique_id":"9_light_zigbee2mqtt"
+         };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/light/1221051039810110150109113116116_9/light/config',
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -7,14 +7,16 @@ const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 
 
 describe('Bridge legacy', () => {
     let controller;
+    let version;
 
     beforeAll(async () => {
-        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
+        jest.useFakeTimers();
+        version = await require('../../lib/util/utils').getZigbee2MQTTVersion();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
     })
@@ -22,15 +24,18 @@ describe('Bridge legacy', () => {
     beforeEach(() => {
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeDefaultState();
         logger.info.mockClear();
         logger.warn.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should publish bridge configuration on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/config',
-          stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
+          stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -188,7 +193,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group"},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false}],"type":"groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","retain":false},{"ID":9,"devices":["bulb_color_2","bulb_2","wall_switch_double/right"],"friendly_name":"ha_discovery_group"},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},{"ID":14,"devices":["power_plug","bulb_2"],"friendly_name":"switch_group","retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group"},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {

test/legacy/report.test.js

@@ -13,13 +13,14 @@ zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
 const MQTT = require('../stub/mqtt');
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Report', () => {
     let controller;
+    let extension;
 
     function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
@@ -59,7 +60,7 @@ describe('Report', () => {
         }
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -69,23 +70,34 @@ describe('Report', () => {
         }
     }
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        settings.set(['advanced', 'report'], true);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        extension = controller.extensions.find((e) => e.constructor.name === 'Report');
+    });
+
     beforeEach(async () => {
+        extension.enabled = true;
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
-        settings.set(['advanced', 'report'], true);
         for (const device of Object.values(zigbeeHerdsman.devices)) {
             mockClear(device);
             delete device.meta.reporting;
         }
-
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
         mocksClear.forEach((m) => m.mockClear());
-        await flushPromises();
+        extension.queue = new Set();
+        extension.failed = new Set();
+        await extension.onZigbeeStarted();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should configure reporting on startup', async () => {
+        await extension.onZigbeeStarted();
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         expectOnOffBrightnessColorReport(endpoint, true);
@@ -96,10 +108,8 @@ describe('Report', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         delete device.meta.report;
-        settings.set(['advanced', 'report'], false);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        extension.enabled = false;
+        await extension.onZigbeeStarted();
         expect(device.meta.reporting).toBe(undefined);
         expect(endpoint.bind).toHaveBeenCalledTimes(0);
     });
@@ -108,11 +118,9 @@ describe('Report', () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         device.meta.reporting = 1;
         const endpoint = device.getEndpoint(1);
-        settings.set(['advanced', 'report'], false);
+        extension.enabled = false;
         mockClear(device);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await extension.onZigbeeStarted();
         expectOnOffBrightnessColorReportDisabled(endpoint, true);
     });
 
@@ -131,7 +139,6 @@ describe('Report', () => {
     });
 
     it('Should not configure reporting when still configuring', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         endpoint.bind.mockImplementationOnce(async () => await wait(1000));
@@ -143,8 +150,6 @@ describe('Report', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
-        jest.useRealTimers();
     });
 
     it('Should not mark as configured when reporting setup fails', async () => {

test/lib/flushPromises.js

@@ -0,0 +1,2 @@
+const globalSetImmediate = setImmediate;
+module.exports = () => new Promise(globalSetImmediate);

test/networkMap.test.js

@@ -16,9 +16,7 @@ zigbeeHerdsman.returnDevices.push(external_converter_device.ieeeAddr)
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
-Date.now = jest.fn()
-Date.now.mockReturnValue(10000);
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 const setTimeoutNative = setTimeout;
 
@@ -26,6 +24,9 @@ describe('Networkmap', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
+        Date.now = jest.fn()
+        Date.now.mockReturnValue(10000);
         data.writeDefaultConfiguration();
         settings.reRead();
         data.writeEmptyState();
@@ -50,6 +51,10 @@ describe('Networkmap', () => {
         global.setTimeout = setTimeoutNative;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     function mock() {
         /**
          * Topology

test/onEvent.test.js

@@ -6,7 +6,7 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45560');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
@@ -22,14 +22,24 @@ describe('On event', () => {
     const device = zigbeeHerdsman.devices.LIVOLO;
 
     beforeEach(async () => {
+        jest.useFakeTimers();
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         zigbeeHerdsmanConverters.onEvent.mockClear();
-        await flushPromises();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should call with start event', async () => {

test/otaUpdate.test.js

@@ -4,26 +4,44 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('OTA update', () => {
     let controller;
 
-    mockClear = (mapped) => {
+    const mockClear = (mapped) => {
         mapped.ota.updateToLatest = jest.fn();
         mapped.ota.isUpdateAvailable = jest.fn();
     }
 
-    beforeEach(async () => {
+    beforeAll(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
+        data.writeDefaultConfiguration();
         settings.set(['advanced', 'ikea_ota_use_test_url'], true);
-        data.writeEmptyState();
+        settings.reRead();
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
+    });
+
+    afterEach(async () => {
+        settings.set(['ota', 'disable_automatic_update_check'], false);
+        jest.runOnlyPendingTimers();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(async () => {
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        extension.inProgress = new Set();
+        controller.state.state = {};
         MQTT.publish.mockClear();
     });
 
@@ -176,7 +194,6 @@ describe('OTA update', () => {
     });
 
     it('Should refuse to check/update when already in progress', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -189,7 +206,7 @@ describe('OTA update', () => {
         MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
@@ -201,7 +218,6 @@ describe('OTA update', () => {
     it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
-        let count = 0;
         endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
 
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
@@ -217,6 +233,7 @@ describe('OTA update', () => {
 
     it('Should check for update when device requests it', async () => {
         const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
         const data = {imageType: 12382};
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -236,8 +253,6 @@ describe('OTA update', () => {
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
 
-        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
-        extension.lastChecked = {};
         logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
         await zigbeeHerdsman.events.message(payload);
@@ -250,6 +265,28 @@ describe('OTA update', () => {
         );
     });
 
+    it('Should check for update when device requests it and it is not available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Should not check for update when device requests it and disable_automatic_update_check is set to true', async () => {
         settings.set(['ota', 'disable_automatic_update_check'], true);
         const device = zigbeeHerdsman.devices.bulb;

test/publish.test.js

@@ -6,7 +6,7 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
@@ -27,6 +27,7 @@ describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
         data.writeEmptyState();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
@@ -34,8 +35,6 @@ describe('Publish', () => {
     });
 
     beforeEach(async () => {
-        jest.useRealTimers();
-        await flushPromises();
         data.writeDefaultConfiguration();
         controller.state.state = {};
         settings.reRead();
@@ -54,6 +53,11 @@ describe('Publish', () => {
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
+    afterAll(async () => {
+        jest.runOnlyPendingTimers();
+        jest.useRealTimers();
+    });
+
     it('Should publish messages to zigbee devices', async () => {
         const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
@@ -176,10 +180,10 @@ describe('Publish', () => {
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_right: 'OFF'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state_right: 'OFF'});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should publish messages to zigbee devices to non default-ep with state_[EP]', async () => {
@@ -231,13 +235,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON'}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy and brightness', async () => {
@@ -248,13 +251,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', brightness: 20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state on', async () => {
@@ -265,13 +267,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, state: 'ON', brightness: 20, color_mode: 'xy'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color: {x: 100, y: 50}, state: 'ON', brightness: 20, color_mode: 'xy'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state off', async () => {
@@ -282,13 +284,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, state: 'OFF'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'OFF'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color rgb object', async () => {
@@ -403,10 +405,10 @@ describe('Publish', () => {
 
     it('Should create and publish to group which is in configuration.yaml but not in zigbee-herdsman', async () => {
         delete zigbeeHerdsman.groups.group_2;
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(8);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(9);
         await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'ON'}));
         await flushPromises();
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(9);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(10);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
     });
@@ -684,14 +686,13 @@ describe('Publish', () => {
     });
 
     it('Should read after write when enabled', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'retrieve_state'], true);
         const endpoint = device.getEndpoint(1);
         const payload = {'state': 'ON', 'color': {'x': 0.701, 'y': 0.299}};
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify(payload));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColor", {"colorx": 45940, "colory": 19595, "transtime": 0}, {}]);
@@ -718,7 +719,7 @@ describe('Publish', () => {
 
         MQTT.publish.mockClear();
         group.command.mockClear();
-        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF', brightness: 100}));
+        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
@@ -1108,11 +1109,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', color_temp: 100, color_mode: 'color_temp'});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON', color_temp: 100, color_mode: 'color_temp'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Home Assistant: should not set state when color is also set', async () => {
@@ -1214,7 +1217,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 22, "warningduration": 100}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 22, "warningduration": 100, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E emergency warning', async () => {
@@ -1223,7 +1226,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 51, "warningduration": 10}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 51, "warningduration": 10, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E emergency without level', async () => {
@@ -1232,7 +1235,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 49, "warningduration": 10}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 49, "warningduration": 10, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E wrong payload (should use defaults)', async () => {
@@ -1241,7 +1244,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 53, "warningduration": 10}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 53, "warningduration": 10, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('Shouldnt do anythign when device is not supported', async () => {
@@ -1278,19 +1281,17 @@ describe('Publish', () => {
 
     it('Should publish separate genOnOff to GL-S-007ZS when setting state and brightness as bulb doesnt turn on with moveToLevelWithOnOff', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/2757
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices['GL-S-007ZS'];
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/GL-S-007ZS/set', stringify({state: 'ON', brightness: 20}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual([ 'genOnOff', 'on', {}, {} ]);
         expect(endpoint.command.mock.calls[1]).toEqual([ 'genLevelCtrl', 'moveToLevelWithOnOff', { level: 20, transtime: 0 }, {} ]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/GL-S-007ZS', stringify({"state":"ON","brightness":20}), { qos: 0, retain: false }, expect.any(Function)]);
-        jest.useRealTimers();
     });
 
     it('Should log as error when setting property with no defined converter', async () => {
@@ -1400,7 +1401,7 @@ describe('Publish', () => {
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith('genScenes', 'recall', { groupid: 15071, sceneid: 1 }, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledTimes(8);
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
             'zigbee2mqtt/group_tradfri_remote',
             stringify({"brightness":50,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
@@ -1422,10 +1423,25 @@ describe('Publish', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(5,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":50,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(6,
             'zigbee2mqtt/group_with_tradfri',
             stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(7,
+            'zigbee2mqtt/switch_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(8,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should sync colors', async () => {

test/receive.test.js

@@ -5,24 +5,31 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Receive', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should handle a zigbee message', async () => {
         const device = zigbeeHerdsman.devices.WXKG11LM;
         device.linkquality = 10;
@@ -100,7 +107,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce messages', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         const data1 = {measuredValue: 8}
@@ -115,7 +121,7 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
@@ -124,7 +130,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce and retain messages when set via device_options', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['device_options', 'debounce'], 0.1);
         settings.set(['device_options', 'retain'], true);
@@ -141,7 +146,7 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
@@ -150,7 +155,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         settings.set(['devices', device.ieeeAddr, 'debounce_ignore'], ['temperature']);
@@ -166,7 +170,7 @@ describe('Receive', () => {
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2});
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03});
@@ -174,13 +178,12 @@ describe('Receive', () => {
 
     it('Shouldnt republish old state', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/3572
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON'}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
@@ -327,6 +330,19 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should publish last_seen for unhandled messages', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genRssiLocation', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('number')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
     it('Should publish last_seen epoch', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'epoch');

test/settings.test.js

@@ -1,8 +1,8 @@
 require('./stub/logger');
 require('./stub/data');
 const data = require('../lib/util/data.js');
-const utils = require('../lib/util/utils.js');
-const settings = require('../lib/util/settings.js');
+const utils = require('../lib/util/utils');
+const settings = require('../lib/util/settings.ts');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
@@ -20,12 +20,12 @@ const minimalConfig = {
 
 describe('Settings', () => {
     const write = (file, json, reread=true) => {
-        fs.writeFileSync(file, yaml.safeDump(json))
+        fs.writeFileSync(file, yaml.dump(json))
         if (reread) {
             settings.reRead();
         }
     };
-    const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const read = (file) => yaml.load(fs.readFileSync(file, 'utf8'));
     const remove = (file) => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
@@ -47,7 +47,7 @@ describe('Settings', () => {
     it('Should return default settings', () => {
         write(configurationFile, {});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expect(s).toStrictEqual(expected);
@@ -56,7 +56,7 @@ describe('Settings', () => {
     it('Should return settings', () => {
         write(configurationFile, {permit_join: true});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.permit_join = true;
@@ -73,7 +73,7 @@ describe('Settings', () => {
 
         write(configurationFile, {});
         const s = settings.get();
-        const expected = objectAssignDeep.noMutate({}, settings._getDefaults());
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.serial.disable_led = true;
@@ -163,7 +163,7 @@ describe('Settings', () => {
         expect(settings.get().mqtt).toStrictEqual(expected);
         expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
 
-        settings._write();
+        settings.testing.write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
         expect(read(secretFile)).toStrictEqual(contentSecret);
 
@@ -493,7 +493,7 @@ describe('Settings', () => {
         settings.addGroup('test123', 123);
         expect(() => {
             settings.addGroup('test_id_123', 123);
-        }).toThrow(new Error("group id '123' is already in use"));
+        }).toThrow(new Error("Group ID '123' is already in use"));
         const expected = {
             '123': {
                 friendly_name: 'test123',
@@ -676,7 +676,7 @@ describe('Settings', () => {
 
         settings.reRead();
 
-        const error = `advanced should be object`;
+        const error = `advanced must be object`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
@@ -702,13 +702,13 @@ describe('Settings', () => {
              \t wrong
         `)
 
-        settings._clear();
+        settings.testing.clear();
         const error = `Your YAML file: '${configurationFile}' is invalid (use https://jsonformatter.org/yaml-validator to find and fix the issue)`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should throw error when yaml file does not exist', () => {
-        settings._clear();
+        settings.testing.clear();
         const error = `ENOENT: no such file or directory, open '${configurationFile}'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
@@ -794,7 +794,7 @@ describe('Settings', () => {
 
         settings.reRead();
 
-        const error = `Following friendly_name are not allowed: '${utils.getEndpointNames()}'`;
+        const error = `Following friendly_name are not allowed: '${utils.endpointNames}'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 

test/stub/data.js

@@ -202,12 +202,16 @@ function writeDefaultConfiguration() {
             '14': {
                 friendly_name: 'switch_group',
                 retain: false,
-                devices: ['power_plug'],
+                devices: ['power_plug', 'bulb_2'],
             },
             '21': {
                 friendly_name: 'gledopto_group',
                 devices: ['GLEDOPTO_2ID/cct'],
-            }
+            },
+            '9': {
+                friendly_name: 'ha_discovery_group',
+                devices: ['bulb_color_2', 'bulb_2', 'wall_switch_double/right']
+            },
         },
         external_converters: [],
     };
@@ -229,20 +233,24 @@ function stateExists() {
     return fs.existsSync(stateFile);
 }
 
-function writeDefaultState() {
-    const state = {
-        "0x000b57fffec6a5b2": {
-            "state": "ON",
-            "brightness": 50,
-            "color_temp": 370,
-            "linkquality": 99,
-        },
-        "0x0017880104e45517": {
-            "brightness": 255
-        },
-    }
+const defaultState = {
+    "0x000b57fffec6a5b2": {
+        "state": "ON",
+        "brightness": 50,
+        "color_temp": 370,
+        "linkquality": 99,
+    },
+    "0x0017880104e45517": {
+        "brightness": 255
+    },
+}
 
-    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(state));
+function getDefaultState() {
+    return defaultState;
+}
+
+function writeDefaultState() {
+    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(defaultState));
 }
 
 jest.mock('../../lib/util/data', () => ({
@@ -261,4 +269,5 @@ module.exports = {
     removeState,
     writeEmptyState,
     stateExists,
+    getDefaultState,
 };

test/stub/zigbeeHerdsman.js

@@ -132,6 +132,7 @@ const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new End
 const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
 const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
+const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
 
 const groups = {
     'group_1': new Group(1, []),
@@ -139,9 +140,10 @@ const groups = {
     'group/with/slashes': new Group(99, []),
     'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
     'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
-    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0]]),
+    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0], bulb_2.endpoints[0]]),
     'gledopto_group': new Group(21, [GLEDOPTO_2ID.endpoints[3]]),
     'default_bind_group': new Group(901, []),
+    'ha_discovery_group': new Group(9, [bulb_color_2.endpoints[0], bulb_2.endpoints[0], QBKG03LM.endpoints[1]]),
 }
 
 const devices = {
@@ -162,7 +164,7 @@ const devices = {
     'ZNCZ02LM': ZNCZ02LM,
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
     'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
-    'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
+    'QBKG03LM':QBKG03LM,
     'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO_2ID': GLEDOPTO_2ID,

test/utils.test.js

@@ -1,4 +1,4 @@
-const utils = require('../lib/util/utils.js');
+const utils = require('../lib/util/utils');
 const version = require('../package.json').version;
 const versionHerdsman = require('../node_modules/zigbee-herdsman/package.json').version;
 
@@ -20,10 +20,6 @@ describe('Utils', () => {
         });
     });
 
-    it('Convert milliseconds to seconds', () => {
-        expect(utils.millisecondsToSeconds(2000)).toBe(2);
-    })
-
     it('Object has properties', () => {
         expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b'])).toBeTruthy();
         expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b', 'd'])).toBeFalsy();
@@ -36,10 +32,10 @@ describe('Utils', () => {
         }));
 
         mockReturnValue = [false, {shortHash: '123'}]
-        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "123", "version": version});
+        expect(await utils.getZigbee2MQTTVersion()).toStrictEqual({"commitHash": "123", "version": version});
 
         mockReturnValue = [true, null]
-        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
+        expect(await utils.getZigbee2MQTTVersion()).toStrictEqual({"commitHash": expect.any(String), "version": version});
     })
 
     it('Check dependency version', async () => {
@@ -57,11 +53,6 @@ describe('Utils', () => {
         Date.prototype.getTimezoneOffset = getTimezoneOffset;
     })
 
-    it('Throw exception when formating with invalid date', async () => {
-        var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
-        expect(() => utils.formatDate(date, 'invalid', 1)).toThrowError("Unsupported type 'invalid'")
-    })
-
     it('Get key', async () => {
         expect(utils.getKey({'1': '1'}, '1', 2, null)).toBe('1');
         expect(utils.getKey({'1': '1'}, '2', 2, null)).toBe(2);

tsconfig.json

@@ -0,0 +1,22 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "esModuleInterop": true,
+        "target": "es2018",
+        "lib": ["es2018"],
+        "noImplicitAny": true,
+        "noImplicitThis": true,
+        "moduleResolution": "node",
+        "declaration": true,
+        "declarationMap": true,
+        "outDir": "dist",
+        "baseUrl": ".",
+        "allowJs": true,
+        "rootDir": "lib",
+        "inlineSourceMap": true,
+        "resolveJsonModule": true
+    },
+    "include": [
+        "lib/**/*"
+    ],
+}

update.sh

@@ -7,8 +7,12 @@ if [ -d data-backup ]; then
    exit 1
 fi
 
-echo "Stopping Zigbee2MQTT..."
-sudo systemctl stop zigbee2mqtt
+if which systemctl 2> /dev/null > /dev/null; then
+       echo "Stopping Zigbee2MQTT..."
+       sudo systemctl stop zigbee2mqtt
+else
+       echo "Skipped stopping Zigbee2MQTT, no systemctl found"
+fi
 
 echo "Creating backup of configuration..."
 cp -R data data-backup
@@ -24,7 +28,11 @@ echo "Restore configuration..."
 cp -R data-backup/* data
 rm -rf data-backup
 
-echo "Starting Zigbee2MQTT..."
-sudo systemctl start zigbee2mqtt
+if which systemctl 2> /dev/null > /dev/null; then
+       echo "Starting Zigbee2MQTT..."
+       sudo systemctl start zigbee2mqtt
+else
+       echo "Skipped starting Zigbee2MQTT, no systemctl found"
+fi
 
 echo "Done!"