1.21.2

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -1,5 +1,5 @@
 .eslintignore
-.eslintrc.json
+.eslintrc.js
 .git
 .github
 .gitignore
@@ -13,4 +13,5 @@ images
 node_modules
 scripts
 test
+coverage
 update.sh

.eslintrc.js

@@ -0,0 +1,45 @@
+
+module.exports = {
+    'env': {
+        'jest/globals': true,
+        'es6': true,
+        'node': true,
+    },
+    'extends': ['eslint:recommended', 'google', 'plugin:jest/recommended', 'plugin:jest/style'],
+    'parserOptions': {
+        'ecmaVersion': 2018,
+        'sourceType': 'module',
+    },
+    'rules': {
+        'require-jsdoc': 'off',
+        'indent': ['error', 4],
+        'max-len': ['error', {'code': 120}],
+        'no-prototype-builtins': 'off',
+    },
+    'plugins': [
+        'jest',
+    ],
+    'overrides': [{
+        files: ['*.ts'],
+        parser: '@typescript-eslint/parser',
+        plugins: ['@typescript-eslint'],
+        extends: ['plugin:@typescript-eslint/recommended'],
+        parserOptions: {
+            project: './tsconfig.json',
+        },
+        rules: {
+            '@typescript-eslint/await-thenable': 'error',
+            '@typescript-eslint/ban-ts-comment': 'off',
+            '@typescript-eslint/explicit-function-return-type': 'error',
+            '@typescript-eslint/no-empty-function': 'off',
+            '@typescript-eslint/no-explicit-any': 'error',
+            '@typescript-eslint/no-unused-vars': 'error',
+            '@typescript-eslint/semi': ['error'],
+            'array-bracket-spacing': ['error', 'never'],
+            'indent': ['error', 4],
+            'max-len': ['error', {'code': 120}],
+            'no-return-await': 'error',
+            'object-curly-spacing': ['error', 'never'],
+        },
+    }],
+};

.eslintrc.json

@@ -1,21 +0,0 @@
-{
-    "env": {
-        "jest/globals": true,
-        "es6": true,
-        "node": true
-    },
-    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
-    "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module"
-    },
-    "rules": {
-        "require-jsdoc": "off",
-        "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }],
-        "no-prototype-builtins": "off"
-    },
-    "plugins": [
-        "jest"
-    ]
-}

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
         registry-url: https://registry.npmjs.org/
     - name: Get NodeJS Version
       id: nodeversion
@@ -24,6 +24,8 @@ jobs:
     - name: Install dependencies
       if: steps.cache-node-modules.outputs.cache-hit != 'true'
       run: npm ci
+    - name: Build
+      run: npm run build
     - name: Test
       run: npm run test-with-coverage
     - name: Lint

.github/workflows/codeql-analysis.yml

@@ -1,67 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master, dev ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-  schedule:
-    - cron: '28 12 * * 3'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/update_deps.yml

@@ -14,10 +14,10 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
-    - run: npm install
-    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
-    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+        node-version: 14
+    - run: rm -rf npm-shrinkwrap.json
+    - run: npx npm-check-updates -u
+    - run: npm install && npm shrinkwrap --dev
     - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies

.github/workflows/update_frontend.yml

@@ -13,7 +13,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
     - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
     - uses: peter-evans/create-pull-request@v3
       with:

.github/workflows/update_zh.yml

@@ -13,7 +13,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
     - uses: peter-evans/create-pull-request@v3
       with:

.github/workflows/update_zhc.yml

@@ -13,7 +13,7 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
     - uses: peter-evans/create-pull-request@v3
       with:

.gitignore

@@ -11,6 +11,9 @@ pids
 *.seed
 *.pid.lock
 
+# Compiled source
+dist/*
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
@@ -72,5 +75,8 @@ data/extension
 # MacOS indexing file
 .DS_Store
 
+# IDE specific folders
+.idea
+
 # Ignore config
-data/*.yaml
+data/*.yaml

.npmignore

@@ -7,6 +7,7 @@ coverage
 .jenkins.yml
 .codeclimate.yml
 .github
+babel.config.js
 
 #linters
 .jscsrc
@@ -14,11 +15,13 @@ coverage
 .eslintrc*
 .dockerignore
 .eslintignore
+.gitignore
 
 #editor settings
 .vscode
 
 #src
+lib
 docs
 docker
 images

README.md

@@ -69,6 +69,9 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 ### Internal Architecture
 Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
 
+### Developing
+Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

babel.config.js

@@ -0,0 +1,11 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+    assumptions: {setPublicClassFields: true},
+    plugins: [
+        ['@babel/plugin-proposal-decorators', {'legacy': true}],
+        ['@babel/plugin-proposal-class-properties'],
+    ],
+};

docker/Dockerfile

@@ -5,20 +5,24 @@ RUN apk add --no-cache tzdata eudev tini
 
 COPY package.json ./
 
-# Dependencies
-FROM base as dependencies
+# Dependencies and build
+FROM base as dependencies_and_build
 
-COPY npm-shrinkwrap.json ./
+COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY lib ./lib
 
 RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
-    npm ci --production && \
+    npm ci --no-audit --no-optional --no-update-notifier && \
+    npm run build && \
+    rm -rf node_modules && \
+    npm ci --production --no-audit --no-optional --no-update-notifier && \
     apk del .buildtools
 
 # Release
 FROM base as release
 
-COPY --from=dependencies /app/node_modules ./node_modules
-COPY lib ./lib
+COPY --from=dependencies_and_build /app/node_modules ./node_modules
+COPY --from=dependencies_and_build /app/dist ./dist
 COPY LICENSE index.js data/configuration.yaml ./
 
 COPY docker/docker-entrypoint.sh /usr/local/bin/
@@ -27,7 +31,7 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 RUN mkdir /app/data
 
 ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+RUN echo "$COMMIT" > dist/.hash
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD [ "/sbin/tini", "--", "node", "index.js"]

docker/README.md

@@ -1 +0,0 @@
-qemu-aarch64-static and qemu-arm-static are taken from https://github.com/resin-io/qemu/releases/tag/v2.9.0%2Bresin1

index.js

@@ -1,10 +1,17 @@
 const semver = require('semver');
 const engines = require('./package.json').engines;
 const indexJsRestart = 'indexjs.restart';
+const fs = require('fs');
+const path = require('path');
+const {exec} = require('child_process');
+const rimraf = require('rimraf');
+require('source-map-support').install();
 
 let controller;
 let stopping = false;
 
+const hashFile = path.join(__dirname, 'dist', '.hash');
+
 async function restart() {
     await stop(indexJsRestart);
     await start();
@@ -16,14 +23,59 @@ async function exit(code, reason) {
     }
 }
 
+async function currentHash() {
+    const git = require('git-last-commit');
+    return new Promise((resolve) => {
+        git.getLastCommit((err, commit) => {
+            if (err) resolve('unknown');
+            else resolve(commit.shortHash);
+        });
+    });
+}
+
+async function writeHash() {
+    const hash = await currentHash();
+    fs.writeFileSync(hashFile, hash);
+}
+
+async function build(reason) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
+        rimraf.sync('dist');
+        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+            if (err) {
+                process.stdout.write(', failed\n');
+                reject(err);
+            } else {
+                process.stdout.write(', finished\n');
+                resolve();
+            }
+        });
+    });
+}
+
+async function checkDist() {
+    if (!fs.existsSync(hashFile)) {
+        await build('initial build');
+    }
+
+    const distHash = fs.readFileSync(hashFile, 'utf-8');
+    const hash = await currentHash();
+    if (hash !== 'unknown' && distHash !== hash) {
+        await build('hash changed');
+    }
+}
+
 async function start() {
+    await checkDist();
+
     const version = engines.node;
     if (!semver.satisfies(process.version, version)) {
         console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
     }
 
     // Validate settings
-    const settings = require('./lib/util/settings');
+    const settings = require('./dist/util/settings');
     settings.reRead();
     const errors = settings.validate();
     if (errors.length > 0) {
@@ -38,7 +90,7 @@ async function start() {
         exit(1);
     }
 
-    const Controller = require('./lib/controller');
+    const Controller = require('./dist/controller');
     controller = new Controller(restart, exit);
     await controller.start();
 }
@@ -54,7 +106,10 @@ async function handleQuit() {
     }
 }
 
-process.on('SIGINT', handleQuit);
-process.on('SIGTERM', handleQuit);
-
-start();
+if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+    writeHash();
+} else {
+    process.on('SIGINT', handleQuit);
+    process.on('SIGTERM', handleQuit);
+    start();
+}

lib/controller.js

@@ -20,6 +20,7 @@ const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMe
 const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
 const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
+const ExtensionAvailabilityLegacy = require('./extension/legacy/availability');
 const ExtensionAvailability = require('./extension/availability');
 const ExtensionBind = require('./extension/bind');
 const ExtensionReport = require('./extension/legacy/report');
@@ -31,15 +32,18 @@ const ExtensionExternalExtension = require('./extension/externalExtension');
 const AllExtensions = [
     ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
     ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
-    ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
-    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension,
+    ExtensionAvailabilityLegacy, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension, ExtensionAvailability,
 ];
 
 class Controller {
     constructor(restartCallback, exitCallback) {
-        this.zigbee = new Zigbee();
-        this.mqtt = new MQTT();
-        this.eventBus = new EventBus();
+        // eslint-disable-next-line
+        this.eventBus = new EventBus.default();
+        // eslint-disable-next-line
+        this.zigbee = new Zigbee.default(this.eventBus);
+        // eslint-disable-next-line
+        this.mqtt = new MQTT.default(this.eventBus);
         this.state = new State(this.eventBus);
         this.restartCallback = restartCallback;
         this.exitCallback = exitCallback;
@@ -50,9 +54,10 @@ class Controller {
         this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
-        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+            this.enableDisableExtension, this.restartCallback, this.addExtension];
         this.extensions = [
-            new ExtensionBridge(...args, this.enableDisableExtension, this.restartCallback),
+            new ExtensionBridge(...args),
             new ExtensionPublish(...args),
             new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
@@ -86,25 +91,32 @@ class Controller {
             this.extensions.push(new ExtensionSoftReset(...args));
         }
 
-        if (settings.get().advanced.availability_timeout) {
+        /* istanbul ignore else */
+        if (settings.get().experimental.availability_new) {
             this.extensions.push(new ExtensionAvailability(...args));
+        } else {
+            if (settings.get().advanced.availability_timeout) {
+                this.extensions.push(new ExtensionAvailabilityLegacy(...args));
+            }
         }
-        this.extensions.push(new ExtensionExternalExtension(...args, this.addExtension, this.enableDisableExtension));
+
+        this.extensions.push(new ExtensionExternalExtension(...args));
     }
 
     async start() {
         this.state.start();
         logger.logOutput();
 
-        const info = await utils.getZigbee2mqttVersion();
+        const info = await utils.getZigbee2MQTTVersion();
         logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
         // Start zigbee
+        let startResult;
         try {
-            await this.zigbee.start();
+            startResult = await this.zigbee.start();
             this.callExtensionMethod('onZigbeeStarted', []);
-            this.zigbee.on('event', this.onZigbeeEvent.bind(this));
-            this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
+            this.eventBus.on('event', this.onZigbeeEvent.bind(this), this.constructor.name);
+            this.eventBus.onAdapterDisconnected(this, this.onZigbeeAdapterDisconnected);
         } catch (error) {
             logger.error('Failed to start zigbee');
             // eslint-disable-next-line
@@ -114,11 +126,19 @@ class Controller {
             this.exitCallback(1);
         }
 
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
+        }
+
         // Log zigbee clients on startup
-        const devices = this.zigbee.getClients();
+        const devices = this.zigbee.getClientsLegacy();
         logger.info(`Currently ${devices.length} devices are joined:`);
         for (const device of devices) {
-            const entity = this.zigbee.resolveEntity(device);
+            const entity = this.zigbee.resolveEntityLegacy(device);
             const model = entity.definition ?
                 `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
                 'Not supported';
@@ -133,18 +153,19 @@ class Controller {
         }
 
         try {
-            await this.zigbee.permitJoin(settings.get().permit_join);
+            await this.zigbee.permitJoinLegacy(settings.get().permit_join);
         } catch (error) {
             logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
         }
 
         // MQTT
-        this.mqtt.on('message', this.onMQTTMessage.bind(this));
+        this.onMQTTMessage = this.onMQTTMessage.bind(this);
+        this.eventBus.onMQTTMessage(this.constructor.name, this.onMQTTMessage);
         await this.mqtt.connect();
 
         // Send all cached states.
         if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
-            for (const device of this.zigbee.getClients()) {
+            for (const device of this.zigbee.getClientsLegacy()) {
                 if (this.state.exists(device.ieeeAddr)) {
                     this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
                 }
@@ -152,7 +173,7 @@ class Controller {
         }
 
         // Add devices which are in database but not in configuration to configuration
-        for (const device of this.zigbee.getClients()) {
+        for (const device of this.zigbee.getClientsLegacy()) {
             if (!settings.getDevice(device.ieeeAddr)) {
                 settings.addDevice(device.ieeeAddr);
             }
@@ -160,6 +181,7 @@ class Controller {
 
         // Call extensions
         await this.callExtensionMethod('onMQTTConnected', []);
+        await this.callExtensionMethod('start', []);
     }
 
     async enableDisableExtension(enable, name) {
@@ -172,20 +194,27 @@ class Controller {
         } else {
             const Extension = AllExtensions.find((e) => e.name === name);
             assert(Extension, `Extension '${name}' does not exist`);
-            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+                this.enableDisableExtension, this.restartCallback, this.addExtension);
+
             this.extensions.push(extension);
-            this.callExtensionMethod('onZigbeeStarted', [], [extension]);
-            this.callExtensionMethod('onMQTTConnected', [], [extension]);
+            await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            await this.callExtensionMethod('onMQTTConnected', [], [extension]);
+            await this.callExtensionMethod('start', [], [extension]);
         }
     }
 
-    addExtension(extension) {
+    async addExtension(extension) {
         this.extensions.push(extension);
+        await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+        await this.callExtensionMethod('onMQTTConnected', [], [extension]);
+        await this.callExtensionMethod('start', [], [extension]);
     }
 
     async stop(reason=null) {
         // Call extensions
         await this.callExtensionMethod('stop', []);
+        this.eventBus.removeListeners(this.constructor.name);
 
         // Wrap-up
         this.state.stop();
@@ -207,7 +236,7 @@ class Controller {
     }
 
     async onZigbeeEvent(type, data) {
-        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(data.device || data.ieeeAddr);
         if (data.device && data.device.type === 'Coordinator') {
             logger.debug('Ignoring message from coordinator');
             return;
@@ -252,16 +281,28 @@ class Controller {
             }
         } else if (type === 'deviceAnnounce') {
             logger.debug(`Device '${name}' announced itself`);
+        } else if (type === 'deviceNetworkAddressChanged') {
+            logger.debug(`Device '${name}' changed network address`);
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
-                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                const settingsDevice = settings.getDevice(data.ieeeAddr);
+                logger.warn(`Device '${settingsDevice ? settingsDevice.friendlyName : data.ieeeAddr}' ` +
+                    `left the network`);
                 this.state.remove(data.ieeeAddr);
             }
         }
 
         // Call extensions
-        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+        await this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+
+        // In case this message is not handled by the receive extension and last_seen is enabled, publish
+        // a message to update the last_seen.
+        // TODO
+        // if (name && !result[ExtensionReceive.prototype.constructor.name] &&
+        //     settings.get().advanced.last_seen !== 'disable' && type !== 'deviceLeave') {
+        //     this.publishEntityState(name, {});
+        // }
     }
 
     onMQTTMessage(payload) {
@@ -273,7 +314,7 @@ class Controller {
     }
 
     async publishEntityState(IDorName, payload, stateChangeReason=null) {
-        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(IDorName);
         if (!resolvedEntity || !resolvedEntity.settings) {
             logger.error(`'${IDorName}' does not exist, skipping publish`);
             return;
@@ -347,7 +388,8 @@ class Controller {
             }
         }
 
-        this.eventBus.emit('publishEntityState', {messagePayload, entity: resolvedEntity, stateChangeReason, payload});
+        this.eventBus.emit('publishEntityState', {messagePayload, entity: resolvedEntity, stateChangeReason, payload,
+            ieeeAddr: resolvedEntity.type === 'device' && resolvedEntity.device.ieeeAddr});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {
@@ -378,10 +420,11 @@ class Controller {
     }
 
     async callExtensionMethod(method, parameters, extensions=null) {
+        const result = {};
         for (const extension of extensions || this.extensions) {
             if (extension[method]) {
                 try {
-                    await extension[method](...parameters);
+                    result[extension.constructor.name] = await extension[method](...parameters);
                 } catch (error) {
                     /* istanbul ignore next */
                     logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
@@ -392,6 +435,7 @@ class Controller {
                 }
             }
         }
+        return result;
     }
 }
 

lib/eventBus.js

@@ -1,43 +0,0 @@
-const events = require('events');
-const assert = require('assert');
-
-const allowedEvents = [
-    'deviceRemoved', // Device has been removed
-    'deviceRenamed', // Device has been renamed
-    'groupRenamed', // Group has been renamed
-    'publishEntityState', // Entity state will be published
-    'stateChange', // Entity changes its state
-    'groupMembersChanged', // Members of a group has been changed
-    'reportingDisabled', // Reporting is disabled for a device
-    'devicesChanged', // Devices changed
-];
-
-class EventBus extends events.EventEmitter {
-    constructor() {
-        super();
-        this.callbackByExtension = {};
-    }
-
-    emit(event, data) {
-        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
-        super.emit(event, data);
-    }
-
-    on(event, callback, extension=null) {
-        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
-        if (extension) {
-            if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
-            this.callbackByExtension[extension].push({event, callback});
-        }
-
-        super.on(event, callback);
-    }
-
-    removeListenersExtension(extension) {
-        for (const entry of this.callbackByExtension[extension] || []) {
-            super.removeListener(entry.event, entry.callback);
-        }
-    }
-}
-
-module.exports = EventBus;

lib/eventBus.ts

@@ -0,0 +1,139 @@
+/* eslint-disable brace-style */
+import events from 'events';
+import * as ZHEvents from 'zigbee-herdsman/dist/controller/events';
+import ExtensionTS from './extension/extensionts';
+
+declare global {
+    interface EventDeviceRenamed { device: Device, homeAssisantRename: boolean, from: string, to: string }
+    // TODO: remove resolved entity, replace by Device
+    interface EventDeviceRemoved { resolvedEntity: ResolvedEntity}
+    type EventMQTTMessage = { topic: string, message: string };
+    type EventMQTTMessagePublished = { topic: string, payload: string, options: {retain: boolean, qos: number} };
+    type EventPermitJoinChanged = ZHEvents.PermitJoinChangedPayload;
+    type EventLastSeenChanged = { device: Device };
+    type EventDeviceNetworkAddressChanged = { device: Device };
+    type EventDeviceAnnounce = { device: Device };
+    type EventDeviceInterview = { device: Device, status: 'started' | 'successful' | 'failed' };
+    type EventDeviceJoined = { device: Device };
+    type EventReportingDisabled = { device: ZHDevice }; // TODO zhdevice -> device
+    type EventDeviceLeave = { ieeeAddr: string };
+    // TODO: remove resolved entity, replace by Group
+    type EventGroupMembersChanged = { group: ResolvedEntity }; // TODO fill
+    type EventPublishEntityState = {
+        // TODO: remove resolved entity, replace by Device | Group and remove ieeeAddr
+        messagePayload: KeyValue, entity: ResolvedEntity, stateChangeReason: 'publishDebounce', payload: KeyValue,
+        ieeeAddr: string,
+    };
+    type EventDeviceMessage = {
+        type: ZHEvents.MessagePayloadType;
+        device: Device;
+        endpoint: ZHEndpoint;
+        linkquality: number;
+        groupID: number;
+        cluster: string | number;
+        data: KeyValue | Array<string | number>;
+        meta: {
+            zclTransactionSequenceNumber?: number;
+        };
+    };
+}
+
+type ListenerKey = string | ExtensionTS;
+
+export default class EventBus {
+    /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+    private callbacksByExtension: { [s: string]: { event: string, callback: (...args: any[]) => void }[] } = {};
+    private emitter = new events.EventEmitter();
+
+    public emitAdapterDisconnected(): void {this.emitter.emit('adapterDisconnected');}
+    public onAdapterDisconnected(key: ListenerKey, callback: () => void): void {
+        this.on('adapterDisconnected', callback, key);}
+
+    public emitPermitJoinChanged(data: EventPermitJoinChanged): void {this.emitter.emit('permitJoinChanged', data);}
+    public onPermitJoinChanged(key: ListenerKey, callback: (data: EventPermitJoinChanged) => void): void {
+        this.on('permitJoinChanged', callback, key);}
+
+    // public emitDeviceRenamed(data: EventDeviceRenamed): void {this.emitter.emit('deviceRenamed', data);}
+    public onDeviceRenamed(key: ListenerKey, callback: (data: EventDeviceRenamed) => void): void {
+        this.on('deviceRenamed', callback, key);}
+
+    // public emitDeviceRemoved(data: EventDeviceRemoved): void {this.emitter.emit('deviceRemoved', data);}
+    public onDeviceRemoved(key: ListenerKey, callback: (data: EventDeviceRemoved) => void): void {
+        this.on('deviceRemoved', callback, key);}
+
+    public emitLastSeenChanged(data: EventLastSeenChanged): void {this.emitter.emit('lastSeenChanged', data);}
+    public onLastSeenChanged(key: ListenerKey, callback: (data: EventLastSeenChanged) => void): void {
+        this.on('lastSeenChanged', callback, key);}
+
+    public emitDeviceNetworkAddressChanged(data: EventDeviceNetworkAddressChanged): void {
+        this.emitter.emit('deviceNetworkAddressChanged', data);}
+    public onDeviceNetworkAddressChanged(
+        key: ListenerKey, callback: (data: EventDeviceNetworkAddressChanged) => void): void {
+        this.on('deviceNetworkAddressChanged', callback, key);}
+
+    public emitDeviceAnnounce(data: EventDeviceAnnounce): void {this.emitter.emit('deviceAnnounce', data);}
+    public onDeviceAnnounce(key: ListenerKey, callback: (data: EventDeviceAnnounce) => void): void {
+        this.on('deviceAnnounce', callback, key);}
+
+    public emitDeviceInterview(data: EventDeviceInterview): void {this.emitter.emit('deviceInterview', data);}
+    public onDeviceInterview(key: ListenerKey, callback: (data: EventDeviceInterview) => void): void {
+        this.on('deviceInterview', callback, key);}
+
+    public emitDeviceJoined(data: EventDeviceJoined): void {this.emitter.emit('deviceJoined', data);}
+    public onDeviceJoined(key: ListenerKey, callback: (data: EventDeviceJoined) => void): void {
+        this.on('deviceJoined', callback, key);}
+
+    public emitDeviceLeave(data: EventDeviceLeave): void {this.emitter.emit('deviceLeave', data);}
+    public onDeviceLeave(key: ListenerKey, callback: (data: EventDeviceLeave) => void): void {
+        this.on('deviceLeave', callback, key);}
+
+    public emitDeviceMessage(data: EventDeviceMessage): void {this.emitter.emit('deviceMessage', data);}
+    public onDeviceMessage(key: ListenerKey, callback: (data: EventDeviceMessage) => void): void {
+        this.on('deviceMessage', callback, key);}
+
+    public emitMQTTMessage(data: EventMQTTMessage): void {this.emitter.emit('mqttMessage', data);}
+    public onMQTTMessage(key: ListenerKey, callback: (data: EventMQTTMessage) => void): void {
+        this.on('mqttMessage', callback, key);}
+
+    public emitMQTTMessagePublished(data: EventMQTTMessagePublished): void {
+        this.emitter.emit('mqttMessagePublished', data);}
+    public onMQTTMessagePublished(key: ListenerKey, callback: (data: EventMQTTMessagePublished) => void): void {
+        this.on('mqttMessagePublished', callback, key);}
+
+    // public emitPublishEntityState(data: EventPublishEntityState): void {
+    //     this.emitter.emit('publishEntityState', data);}
+    public onPublishEntityState(key: ListenerKey, callback: (data: EventPublishEntityState) => void): void {
+        this.on('publishEntityState', callback, key);}
+
+    // public emitGroupMembersChanged(data: EventGroupMembersChanged): void {
+    //     this.emitter.emit('groupMembersChanged', data);}
+    public onGroupMembersChanged(key: ListenerKey, callback: (data: EventGroupMembersChanged) => void): void {
+        this.on('groupMembersChanged', callback, key);}
+
+    public emitDevicesChanged(): void {this.emitter.emit('devicesChanged');}
+    public onDevicesChanged(key: ListenerKey, callback: () => void): void {this.on('devicesChanged', callback, key);}
+
+    // public emitReportingDisabled(data: EventReportingDisabled): void {
+    //     this.emitter.emit('reportingDisabled', data);}
+    public onReportingDisabled(key: ListenerKey, callback: (data: EventReportingDisabled) => void): void {
+        this.on('reportingDisabled', callback, key);}
+
+    /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+    private on(event: string, callback: (...args: any[]) => void, key: ListenerKey): void {
+        key = typeof key === 'string' ? key : key.constructor.name;
+        if (!this.callbacksByExtension[key]) this.callbacksByExtension[key] = [];
+        this.callbacksByExtension[key].push({event, callback});
+        this.emitter.on(event, callback);
+    }
+
+    // TODO: remove
+    /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+    emit(event: string, ...args: any[]): void {
+        this.emitter.emit(event, ...args);
+    }
+
+    public removeListeners(key: ListenerKey): void {
+        key = typeof key === 'string' ? key : key.constructor.name;
+        this.callbacksByExtension[key]?.forEach((e) => this.emitter.removeListener(e.event, e.callback));
+    }
+}

lib/extension/availability.ts

@@ -0,0 +1,194 @@
+import ExtensionTS from './extensionts';
+import logger from '../util/logger';
+import {sleep, hours, minutes, seconds, isAvailabilityEnabledForDevice} from '../util/utils';
+import * as settings from '../util/settings';
+import debounce from 'debounce';
+import bind from 'bind-decorator';
+
+// TODO
+// - Enable for HA addon
+// - Add to setting schema (when old availability is removed)
+export default class Availability extends ExtensionTS {
+    private timers: {[s: string]: NodeJS.Timeout} = {};
+    private availabilityCache: {[s: string]: boolean} = {};
+    private retrieveStateDebouncers: {[s: string]: () => void} = {};
+    private pingQueue: Device[] = [];
+    private pingQueueExecuting = false;
+
+    private getTimeout(device: Device): number {
+        if (typeof device.settings.availability === 'object' && device.settings.availability?.timeout != null) {
+            return minutes(device.settings.availability.timeout);
+        }
+
+        const key = this.isActiveDevice(device) ? 'active' : 'passive';
+        const availabilitySettings = settings.get().availability;
+        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
+            return minutes(availabilitySettings[key]?.timeout);
+        }
+
+        return key === 'active' ? minutes(10) : hours(25);
+    }
+
+    private isActiveDevice(device: Device): boolean {
+        return (device.type === 'Router' && device.powerSource !== 'Battery') ||
+            device.powerSource === 'Mains (single phase)';
+    }
+
+    private isAvailable(device: Device): boolean {
+        const ago = Date.now() - device.lastSeen;
+        return ago < this.getTimeout(device);
+    }
+
+    private resetTimer(device: Device): void {
+        clearTimeout(this.timers[device.ieeeAddr]);
+
+        // If the timer triggers, the device is not avaiable anymore otherwise resetTimer already have been called
+        if (this.isActiveDevice(device)) {
+            // If device did not check in, ping it, if that fails it will be marked as offline
+            this.timers[device.ieeeAddr] = setTimeout(
+                () => this.addToPingQueue(device), this.getTimeout(device) + seconds(1));
+        } else {
+            this.timers[device.ieeeAddr] = setTimeout(
+                () => this.publishAvailability(device, true), this.getTimeout(device) + seconds(1));
+        }
+    }
+
+    private addToPingQueue(device: Device): void {
+        this.pingQueue.push(device);
+        this.pingQueueExecuteNext();
+    }
+
+    private removeFromPingQueue(device: Device): void {
+        const index = this.pingQueue.findIndex((d) => d.ieeeAddr === device.ieeeAddr);
+        index != -1 && this.pingQueue.splice(index, 1);
+    }
+
+    private async pingQueueExecuteNext(): Promise<void> {
+        if (this.pingQueue.length === 0 || this.pingQueueExecuting) return;
+        this.pingQueueExecuting = true;
+
+        const device = this.pingQueue[0];
+        let pingedSuccessfully = false;
+        const available = this.availabilityCache[device.ieeeAddr] || this.isAvailable(device);
+        const attempts = available ? 2 : 1;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                // Enable recovery if device is marked as available and first ping fails.
+                const disableRecovery = !(i == 1 && available);
+                await device.ping(disableRecovery);
+                pingedSuccessfully = true;
+                logger.debug(`Succesfully pinged '${device.name}' (attempt ${i + 1}/${attempts})`);
+                break;
+            } catch (error) {
+                logger.error(`Failed to ping '${device.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
+                // Try again in 3 seconds.
+                const lastAttempt = i - 1 === attempts;
+                !lastAttempt && await sleep(3);
+            }
+        }
+
+        this.publishAvailability(device, !pingedSuccessfully);
+        this.resetTimer(device);
+        this.removeFromPingQueue(device);
+
+        // Sleep 2 seconds before executing next ping
+        await sleep(2);
+        this.pingQueueExecuting = false;
+        this.pingQueueExecuteNext();
+    }
+
+    override async start(): Promise<void> {
+        logger.warn('Using experimental new availability feature');
+
+        this.eventBus.onDeviceRenamed(this, (data: EventDeviceRenamed) =>
+            this.publishAvailability(data.device, false, true));
+        this.eventBus.onDeviceLeave(this, (data: EventDeviceLeave) => clearTimeout(this.timers[data.ieeeAddr]));
+        this.eventBus.onDeviceAnnounce(this, (data: EventDeviceAnnounce) => this.retrieveState(data.device));
+        this.eventBus.onLastSeenChanged(this, this.onLastSeenChanged);
+
+        for (const device of this.zigbee.getClients()) {
+            if (isAvailabilityEnabledForDevice(device, settings.get())) {
+                // Publish initial availablility
+                this.publishAvailability(device, true);
+
+                this.resetTimer(device);
+
+                // If an active device is initially unavailable, ping it.
+                if (this.isActiveDevice(device) && !this.isAvailable(device)) {
+                    this.addToPingQueue(device);
+                }
+            }
+        }
+    }
+
+    private publishAvailability(device: Device, logLastSeen: boolean, forcePublish=false): void {
+        if (logLastSeen) {
+            const ago = Date.now() - device.lastSeen;
+            if (this.isActiveDevice(device)) {
+                logger.debug(
+                    `Active device '${device.name}' was last seen '${(ago / minutes(1)).toFixed(2)}' minutes ago.`);
+            } else {
+                logger.debug(
+                    `Passive device '${device.name}' was last seen '${(ago / hours(1)).toFixed(2)}' hours ago.`);
+            }
+        }
+
+        const available = this.isAvailable(device);
+        if (!forcePublish && this.availabilityCache[device.ieeeAddr] == available) {
+            return;
+        }
+
+        if (device.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[device.ieeeAddr] === false) {
+            logger.debug(`Device '${device.name}' reconnected`);
+            this.retrieveState(device);
+        }
+
+        const topic = `${device.name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        this.availabilityCache[device.ieeeAddr] = available;
+        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+    }
+
+    @bind private onLastSeenChanged(data: EventLastSeenChanged): void {
+        if (isAvailabilityEnabledForDevice(data.device, settings.get())) {
+            // Remove from ping queue, not necessary anymore since we know the device is online.
+            this.removeFromPingQueue(data.device);
+            this.resetTimer(data.device);
+            this.publishAvailability(data.device, false);
+        }
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        Object.values(this.timers).forEach((t) => clearTimeout(t));
+    }
+
+    private retrieveState(device: Device): void {
+        /**
+         * Retrieve state of a device in a debounced manner, this function is called on a 'deviceAnnounce' which a
+         * device can send multiple times after each other.
+         */
+        if (device.definition && !device.interviewing && !this.retrieveStateDebouncers[device.ieeeAddr]) {
+            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(async () => {
+                try {
+                    logger.debug(`Retrieving state of '${device.name}' after reconnect`);
+                    // Color and color temperature converters do both, only needs to be called once.
+                    const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
+                    for (const keys of keySet) {
+                        const converter = device.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
+                        await converter?.convertGet?.(device.endpoint(), keys[0],
+                            {message: this.state.get(device.ieeeAddr) || {}, mapped: device.definition});
+                    }
+                } catch (error) {
+                    logger.error(`Failed to read state of '${device.name}' after reconnect (${error.message})`);
+                }
+            }, seconds(2));
+        }
+
+        this.retrieveStateDebouncers[device.ieeeAddr]?.();
+    }
+}
+
+// TODO_finished: : change class to export default
+module.exports = Availability;

lib/extension/bind.js

@@ -74,6 +74,9 @@ const pollOnMessage = [
                 {type: 'commandMove', data: {}},
                 {type: 'commandMoveToLevelWithOnOff', data: {}},
             ],
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
         },
         // Read the following attributes
         read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
@@ -99,6 +102,9 @@ const pollOnMessage = [
                 {type: 'commandOffWithEffect', data: {}},
                 {type: 'commandToggle', data: {}},
             ],
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
             manuSpecificPhilips: [
                 {type: 'commandHueNotification', data: {button: 1}},
                 {type: 'commandHueNotification', data: {button: 4}},
@@ -112,13 +118,39 @@ const pollOnMessage = [
             zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
         ],
     },
+    {
+        cluster: {
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+        },
+        read: {
+            cluster: 'lightingColorCtrl',
+            attributes: [],
+            // Since not all devices support the same attributes they need to be calculated dynamically
+            // depending on the capabilities of the endpoint.
+            attributesForEndpoint: async (endpoint) => {
+                const supportedAttrs = await getColorCapabilities(endpoint);
+                const readAttrs = [];
+                supportedAttrs.colorXY && readAttrs.push('currentX', 'currentY');
+                supportedAttrs.colorTemperature && readAttrs.push('colorTemperature');
+                return readAttrs;
+            },
+        },
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
 ];
 
 class Bind extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.legacyApi = settings.get().advanced.legacy_api;
-        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d));
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d), this.constructor.name);
         this.pollDebouncers = {};
     }
 
@@ -127,6 +159,7 @@ class Bind extends Extension {
         let sourceKey = null;
         let targetKey = null;
         let clusters = null;
+        let skipDisableReporting = false;
 
         if (this.legacyApi && topic.match(legacyTopicRegex)) {
             topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
@@ -139,19 +172,21 @@ class Bind extends Extension {
             sourceKey = message.from;
             targetKey = message.to;
             clusters = message.clusters;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
         }
 
-        return {type, sourceKey, targetKey, clusters};
+        return {type, sourceKey, targetKey, clusters, skipDisableReporting};
     }
 
     async onMQTTMessage(topic, message) {
-        const {type, sourceKey, targetKey, clusters} = this.parseMQTTMessage(topic, message);
+        const {type, sourceKey, targetKey, clusters, skipDisableReporting} = this.parseMQTTMessage(topic, message);
         if (!type) return null;
         message = utils.parseJSON(message, message);
 
         let error = null;
-        const source = this.zigbee.resolveEntity(sourceKey);
-        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
+        const source = this.zigbee.resolveEntityLegacy(sourceKey);
+        const target = targetKey === 'default_bind_group' ?
+            defaultBindGroup : this.zigbee.resolveEntityLegacy(targetKey);
         const responseData = {from: sourceKey, to: targetKey};
 
         if (!source || source.type !== 'device') {
@@ -255,7 +290,7 @@ class Bind extends Extension {
                 if (type === 'bind') {
                     await this.setupReporting(source.endpoint.binds.filter((b) =>
                         successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
-                } else if (target.type !== 'group_number') {
+                } else if ((target.type !== 'group_number') && !skipDisableReporting) {
                     await this.disableUnnecessaryReportings(bindTarget);
                 }
             }
@@ -275,13 +310,15 @@ class Bind extends Extension {
     }
 
     async groupMembersChanged(data) {
-        const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
-            .flat().filter((b) => b.target === data.group.group);
-
         if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.getClientsLegacy().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v)).map((e) => e.binds)
+                .reduce((a, v) => a.concat(v)).filter((b) => b.target === data.group.group);
             await this.setupReporting(bindsToGroup);
         } else { // action === remove/remove_all
-            await this.disableUnnecessaryReportings(data.endpoint);
+            if (!data.skipDisableReporting) {
+                await this.disableUnnecessaryReportings(data.endpoint);
+            }
         }
     }
 
@@ -296,11 +333,11 @@ class Bind extends Extension {
     }
 
     async setupReporting(binds) {
-        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinator = this.zigbee.getDevicesByTypeLegacy('Coordinator')[0];
         const coordinatorEndpoint = coordinator.getEndpoint(1);
         for (const bind of binds.filter((b) => b.cluster.name in reportClusters)) {
             for (const endpoint of this.getSetupReportingEndpoints(bind, coordinatorEndpoint)) {
-                const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+                const entity = `${this.zigbee.resolveEntityLegacy(endpoint.getDevice()).name}/${endpoint.ID}`;
                 try {
                     await endpoint.bind(bind.cluster.name, coordinatorEndpoint);
 
@@ -323,15 +360,16 @@ class Bind extends Extension {
     }
 
     async disableUnnecessaryReportings(target) {
-        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinator = this.zigbee.getDevicesByTypeLegacy('Coordinator')[0];
         const coordinatorEndpoint = coordinator.getEndpoint(1);
         const endpoints = target.constructor.name === 'Group' ? target.members : [target];
         for (const endpoint of endpoints) {
-            const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+            const entity = `${this.zigbee.resolveEntityLegacy(endpoint.getDevice()).name}/${endpoint.ID}`;
             const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
                 .map((b) => b.cluster.name);
-            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
-                .flat().filter((bind) => {
+            const requiredClusters = this.zigbee.getClientsLegacy().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v))
+                .map((e) => e.binds).reduce((a, v) => a.concat(v)).filter((bind) => {
                     if (bind.target.constructor.name === 'Group') {
                         return bind.target.members.includes(endpoint);
                     } else {
@@ -366,7 +404,7 @@ class Bind extends Extension {
         }
     }
 
-    poll(message) {
+    async poll(message) {
         /**
          * This method poll bound endpoints and group members for state changes.
          *
@@ -391,7 +429,7 @@ class Bind extends Extension {
             toPoll = toPoll.map((e) => e.target);
 
             // If message is published to a group, add members of the group
-            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByIDLegacy(message.groupID) : null;
             if (group) {
                 toPoll = toPoll.concat(group.members);
             }
@@ -400,14 +438,21 @@ class Bind extends Extension {
 
             for (const endpoint of toPoll) {
                 for (const poll of polls) {
-                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) ||
+                        !endpoint.supportsInputCluster(poll.read.cluster)) {
                         continue;
                     }
 
+                    let readAttrs = poll.read.attributes;
+                    if (poll.read.attributesForEndpoint) {
+                        const attrsForEndpoint = await poll.read.attributesForEndpoint(endpoint);
+                        readAttrs = [...poll.read.attributes, ...attrsForEndpoint];
+                    }
+
                     const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
                     if (!this.pollDebouncers[key]) {
                         this.pollDebouncers[key] = debounce(async () => {
-                            await endpoint.read(poll.read.cluster, poll.read.attributes);
+                            await endpoint.read(poll.read.cluster, readAttrs);
                         }, 1000);
                     }
 

lib/extension/bridge.ts

@@ -1,164 +1,160 @@
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const Extension = require('./extension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const settings = require('../util/settings');
-const Transport = require('winston-transport');
-const stringify = require('json-stable-stringify-without-jsonify');
-const objectAssignDeep = require(`object-assign-deep`);
-const {updatedDiff, addedDiff} = require('deep-object-diff');
+/* eslint-disable camelcase */
+import logger from '../util/logger';
+import * as utils from '../util/utils';
+import * as settings from '../util/settings';
+import Transport from 'winston-transport';
+import bind from 'bind-decorator';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import objectAssignDeep from 'object-assign-deep';
+import {detailedDiff} from 'deep-object-diff';
+import ExtensionTS from './extensionts';
+import Device from '../model/device';
+import Group from '../model/group';
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
-class Bridge extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.enableDisableExtension = enableDisableExtension;
-        this.restartCallback = restartCallback;
-        this.lastJoinedDeviceIeeeAddr = null;
-        this.setupMQTTLogging();
-        this.restartRequired = false;
+class Bridge extends ExtensionTS {
+    private zigbee2mqttVersion: {commitHash: string, version: string};
+    private coordinatorVersion: CoordinatorVersion;
+    private restartRequired = false;
+    private lastJoinedDeviceIeeeAddr: string;
+    private requestLookup: {[key: string]: (message: KeyValue | string) => Promise<MQTTResponse>};
 
+    override async start(): Promise<void> {
         this.requestLookup = {
-            'device/options': this.deviceOptions.bind(this),
-            'device/configure_reporting': this.deviceConfigureReporting.bind(this),
-            'device/remove': this.deviceRemove.bind(this),
-            'device/rename': this.deviceRename.bind(this),
-            'group/add': this.groupAdd.bind(this),
-            'group/options': this.groupOptions.bind(this),
-            'group/remove': this.groupRemove.bind(this),
-            'group/rename': this.groupRename.bind(this),
-            'permit_join': this.permitJoin.bind(this),
-            'restart': this.restart.bind(this),
-            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
-            'touchlink/identify': this.touchlinkIdentify.bind(this),
-            'touchlink/scan': this.touchlinkScan.bind(this),
-            'health_check': this.healthCheck.bind(this),
-            'options': this.bridgeOptions.bind(this),
+            'device/options': this.deviceOptions,
+            'device/configure_reporting': this.deviceConfigureReporting,
+            'device/remove': this.deviceRemove,
+            'device/rename': this.deviceRename,
+            'group/add': this.groupAdd,
+            'group/options': this.groupOptions,
+            'group/remove': this.groupRemove,
+            'group/rename': this.groupRename,
+            'permit_join': this.permitJoin,
+            'restart': this.restart,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+            'touchlink/identify': this.touchlinkIdentify,
+            'touchlink/scan': this.touchlinkScan,
+            'health_check': this.healthCheck,
+            'options': this.bridgeOptions,
             // Below are deprecated
-            'config/last_seen': this.configLastSeen.bind(this),
-            'config/homeassistant': this.configHomeAssistant.bind(this),
-            'config/elapsed': this.configElapsed.bind(this),
-            'config/log_level': this.configLogLevel.bind(this),
+            'config/last_seen': this.configLastSeen,
+            'config/homeassistant': this.configHomeAssistant,
+            'config/elapsed': this.configElapsed,
+            'config/log_level': this.configLogLevel,
         };
-    }
 
-    async onMQTTConnected() {
-        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
-        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
-
-        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
-        this.eventBus.on(`devicesChanged`, () => {
-            this.publishDevices();
-            this.publishInfo();
-        });
-        this.eventBus.on(`deviceRenamed`, () => {
-            this.publishInfo();
-        });
-        this.eventBus.on(`groupRenamed`, () => {
-            this.publishInfo();
-        });
-        this.zigbee.on('permitJoinChanged', (data) => this.permitJoinChanged(data));
-        await this.publishInfo();
-        await this.publishDevices();
-        await this.publishGroups();
-    }
-
-    setupMQTTLogging() {
         const mqtt = this.mqtt;
         class EventTransport extends Transport {
-            log(info, callback) {
+            log(info: {message: string, level: string}, callback: () => void): void {
                 const payload = stringify({message: info.message, level: info.level});
                 mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
                 callback();
             }
         }
-
+        // @ts-ignore TODO fix typing
         logger.addTransport(new EventTransport());
+
+        this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.eventBus.onDeviceRenamed(this, () => this.publishInfo());
+        this.eventBus.onGroupMembersChanged(this, () => this.publishGroups());
+        this.eventBus.onDevicesChanged(this, () => this.publishDevices() && this.publishInfo());
+        this.eventBus.onPermitJoinChanged(this, () => !this.zigbee.isStopping() && this.publishInfo());
+
+        // Zigbee events
+        const publishEvent = (type: string, data: KeyValue): Promise<void> =>
+            this.mqtt.publish('bridge/event', stringify({type, data}), {retain: false, qos: 0});
+        this.eventBus.onDeviceJoined(this, (data) => {
+            this.lastJoinedDeviceIeeeAddr = data.device.ieeeAddr;
+            this.publishDevices();
+            publishEvent('device_joined', {friendly_name: data.device.name, ieee_address: data.device.ieeeAddr});
+        });
+        this.eventBus.onDeviceLeave(this, (data) => {
+            this.publishDevices();
+            publishEvent('device_leave',
+                {ieee_address: data.ieeeAddr, friendly_name: settings.getDevice(data.ieeeAddr)?.friendlyName});
+        });
+        this.eventBus.onDeviceNetworkAddressChanged(this, () => this.publishDevices());
+        this.eventBus.onDeviceInterview(this, (data) => {
+            this.publishDevices();
+            const payload: KeyValue =
+                {friendly_name: data.device.name, status: data.status, ieee_address: data.device.ieeeAddr};
+            if (data.status === 'successful') {
+                payload.supported = !!data.device.definition;
+                payload.definition = this.getDefinitionPayload(data.device);
+            }
+            publishEvent('device_interview', payload);
+        });
+        this.eventBus.onDeviceAnnounce(this, (data) => {
+            this.publishDevices();
+            publishEvent('device_announce', {friendly_name: data.device.name, ieee_address: data.device.ieeeAddr});
+        });
+
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+
+
+        this.onMQTTMessage_ = this.onMQTTMessage_.bind(this);
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
     }
 
-    permitJoinChanged(data) {
-        if (!this.zigbee.isStopping()) {
-            this.publishInfo();
-        }
-    }
-
-    async onMQTTMessage(topic, message) {
-        const match = topic.match(requestRegex);
-        if (match && this.requestLookup[match[1].toLowerCase()]) {
-            message = utils.parseJSON(message, message);
+    // TODO remove trailing _
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        const match = data.topic.match(requestRegex);
+        const key = match?.[1]?.toLowerCase();
+        if (key in this.requestLookup) {
+            const message = utils.parseJSON(data.message, data.message);
 
             try {
-                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                const response = await this.requestLookup[key](message);
                 await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             } catch (error) {
-                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
                 const response = utils.getResponse(message, {}, error.message);
                 await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             }
         }
     }
 
-    async onZigbeeEvent(type, data, resolvedEntity) {
-        if (type === 'deviceJoined' && resolvedEntity) {
-            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
-        }
-
-        if (['deviceJoined', 'deviceLeave', 'deviceInterview', 'deviceAnnounce'].includes(type)) {
-            let payload;
-            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
-            if (type === 'deviceJoined') {
-                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
-            } else if (type === 'deviceInterview') {
-                payload = {
-                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
-                };
-                if (data.status === 'successful') {
-                    const definition = resolvedEntity.definition;
-                    payload.supported = !!definition;
-                    payload.definition = this.getDefinitionPayload(definition, resolvedEntity);
-                }
-            } else if (type === 'deviceAnnounce') {
-                payload = {
-                    friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
-                };
-            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
-
-            await this.mqtt.publish(
-                'bridge/event',
-                stringify({type: utils.toSnakeCase(type), data: payload}),
-                {retain: false, qos: 0},
-            );
-        }
-
-        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
-            await this.publishDevices();
-        }
-    }
-
     /**
      * Requests
      */
 
-    async deviceOptions(message) {
+    @bind async deviceOptions(message: KeyValue | string): Promise<MQTTResponse> {
         return this.changeEntityOptions('device', message);
     }
 
-    async groupOptions(message) {
+    @bind async groupOptions(message: KeyValue | string): Promise<MQTTResponse> {
         return this.changeEntityOptions('group', message);
     }
 
-    async bridgeOptions(message) {
+    @bind async bridgeOptions(message: KeyValue | string): Promise<MQTTResponse> {
         if (typeof message !== 'object' || typeof message.options !== 'object') {
             throw new Error(`Invalid payload`);
         }
 
-        const diffUpdated = updatedDiff(settings.get(), message.options);
-        const diffAdded = addedDiff(settings.get(), message.options);
-        const newSettings = objectAssignDeep.noMutate(diffUpdated, diffAdded);
+        const diff: KeyValue = detailedDiff(settings.get(), message.options);
+
+        // Remove any settings that are in the deleted.diff but not in the passed options
+        const cleanupDeleted = (options: KeyValue, deleted: KeyValue): void => {
+            for (const key of Object.keys(deleted)) {
+                if (!(key in options)) {
+                    delete deleted[key];
+                } else if (!Array.isArray(options[key])) {
+                    cleanupDeleted(options[key], deleted[key]);
+                }
+            }
+        };
+        cleanupDeleted(message.options, diff.deleted);
+
+        const newSettings = objectAssignDeep({}, diff.added, diff.updated, diff.deleted);
 
         // deep-object-diff converts arrays to objects, set original array back here
-        const convertBackArray = (before, after) => {
+        const convertBackArray = (before: KeyValue, after: KeyValue): void => {
             for (const [key, afterValue] of Object.entries(after)) {
                 const beforeValue = before[key];
                 if (Array.isArray(beforeValue)) {
@@ -179,10 +175,11 @@ class Bridge extends Extension {
         }
 
         if (newSettings.hasOwnProperty('homeassistant')) {
-            this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
+            await this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
         }
 
         if (newSettings.hasOwnProperty('advanced') && newSettings.advanced.hasOwnProperty('log_level')) {
+            // @ts-ignore fix type
             logger.setLevel(newSettings.advanced.log_level);
         }
 
@@ -191,19 +188,19 @@ class Bridge extends Extension {
         return utils.getResponse(message, {restart_required: this.restartRequired}, null);
     }
 
-    async deviceRemove(message) {
+    @bind async deviceRemove(message: string | KeyValue): Promise<MQTTResponse> {
         return this.removeEntity('device', message);
     }
 
-    async groupRemove(message) {
+    @bind async groupRemove(message: string | KeyValue): Promise<MQTTResponse> {
         return this.removeEntity('group', message);
     }
 
-    async healthCheck(message) {
+    @bind async healthCheck(message: string | KeyValue): Promise<MQTTResponse> {
         return utils.getResponse(message, {healthy: true}, null);
     }
 
-    async groupAdd(message) {
+    @bind async groupAdd(message: string | KeyValue): Promise<MQTTResponse> {
         if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
             throw new Error(`Invalid payload`);
         }
@@ -216,35 +213,37 @@ class Bridge extends Extension {
         return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
     }
 
-    async deviceRename(message) {
+    @bind async deviceRename(message: string | KeyValue): Promise<MQTTResponse> {
         return this.renameEntity('device', message);
     }
 
-    async groupRename(message) {
+    @bind async groupRename(message: string | KeyValue): Promise<MQTTResponse> {
         return this.renameEntity('group', message);
     }
 
-    async restart(message) {
+    @bind async restart(message: string | KeyValue): Promise<MQTTResponse> {
         // Wait 500 ms before restarting so response can be send.
         setTimeout(this.restartCallback, 500);
         logger.info('Restarting Zigbee2MQTT');
         return utils.getResponse(message, {}, null);
     }
 
-    async permitJoin(message) {
+    @bind async permitJoin(message: KeyValue | string): Promise<MQTTResponse> {
         if (typeof message === 'object' && !message.hasOwnProperty('value')) {
             throw new Error('Invalid payload');
         }
 
-        let value;
-        let time;
-        let resolvedEntity;
+        let value: boolean | string;
+        let time: number;
+        let device: Device = null;
         if (typeof message === 'object') {
             value = message.value;
             time = message.time;
             if (message.device) {
-                resolvedEntity = this.zigbee.resolveEntity(message.device);
-                if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                const resolved = this.zigbee.resolveEntity(message.device);
+                if (resolved instanceof Device) {
+                    device = resolved;
+                } else {
                     throw new Error(`Device '${message.device}' does not exist`);
                 }
             }
@@ -256,18 +255,18 @@ class Bridge extends Extension {
             value = value.toLowerCase() === 'true';
         }
 
-        await this.zigbee.permitJoin(value, resolvedEntity, time);
-        const response = {value};
-        if (resolvedEntity) response.device = message.device;
-        if (time) response.time = message.time;
+        await this.zigbee.permitJoin(value, device, time);
+        const response: {value: boolean, device?: string, time?: number} = {value};
+        if (device && typeof message === 'object') response.device = message.device;
+        if (time && typeof message === 'object') response.time = message.time;
         return utils.getResponse(message, response, null);
     }
 
     // Deprecated
-    configLastSeen(message) {
+    @bind async configLastSeen(message: KeyValue | string): Promise<MQTTResponse> {
         const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
         const value = this.getValue(message);
-        if (!allowed.includes(value)) {
+        if (typeof value !== 'string' || !allowed.includes(value)) {
             throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
         }
 
@@ -277,24 +276,24 @@ class Bridge extends Extension {
     }
 
     // Deprecated
-    configHomeAssistant(message) {
+    @bind async configHomeAssistant(message: string | KeyValue): Promise<MQTTResponse> {
         const allowed = [true, false];
         const value = this.getValue(message);
-        if (!allowed.includes(value)) {
+        if (typeof value !== 'boolean' || !allowed.includes(value)) {
             throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
         }
 
-        this.enableDisableExtension(value, 'HomeAssistant');
+        await this.enableDisableExtension(value, 'HomeAssistant');
         settings.set(['homeassistant'], value);
         this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
     // Deprecated
-    configElapsed(message) {
+    @bind async configElapsed(message: KeyValue | string): Promise<MQTTResponse> {
         const allowed = [true, false];
         const value = this.getValue(message);
-        if (!allowed.includes(value)) {
+        if (typeof value !== 'boolean' || !allowed.includes(value)) {
             throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
         }
 
@@ -304,19 +303,20 @@ class Bridge extends Extension {
     }
 
     // Deprecated
-    configLogLevel(message) {
+    @bind async configLogLevel(message: KeyValue | string): Promise<MQTTResponse> {
         const allowed = ['error', 'warn', 'info', 'debug'];
         const value = this.getValue(message);
-        if (!allowed.includes(value)) {
+        if (typeof value !== 'string' || !allowed.includes(value)) {
             throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
         }
 
+        // @ts-ignore TODO fix type
         logger.setLevel(value);
         this.publishInfo();
         return utils.getResponse(message, {value}, null);
     }
 
-    async touchlinkIdentify(message) {
+    @bind async touchlinkIdentify(message: KeyValue | string): Promise<MQTTResponse> {
         if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
             !message.hasOwnProperty('channel')) {
             throw new Error('Invalid payload');
@@ -327,9 +327,9 @@ class Bridge extends Extension {
         return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
     }
 
-    async touchlinkFactoryReset(message) {
+    @bind async touchlinkFactoryReset(message: KeyValue | string): Promise<MQTTResponse> {
         let result = false;
-        const payload = {};
+        const payload: {ieee_address?: string, channel?: number} = {};
         if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
             message.hasOwnProperty('channel')) {
             logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
@@ -350,7 +350,7 @@ class Bridge extends Extension {
         }
     }
 
-    async touchlinkScan(message) {
+    @bind async touchlinkScan(message: KeyValue | string): Promise<MQTTResponse> {
         logger.info('Start Touchlink scan');
         const result = await this.zigbee.touchlinkScan();
         const found = result.map((r) => {
@@ -364,7 +364,7 @@ class Bridge extends Extension {
      * Utils
      */
 
-    getValue(message) {
+    getValue(message: KeyValue | string): string | boolean | number {
         if (typeof message === 'object') {
             if (!message.hasOwnProperty('value')) {
                 throw new Error('No value given');
@@ -376,19 +376,20 @@ class Bridge extends Extension {
         }
     }
 
-    async changeEntityOptions(entityType, message) {
+    async changeEntityOptions(entityType: 'device' | 'group', message: KeyValue | string): Promise<MQTTResponse> {
         if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
             throw new Error(`Invalid payload`);
         }
 
-        const ID = message.id;
-        const entity = this.getEntity(entityType, ID);
-        settings.changeEntityOptions(ID, message.options);
-        const cleanup = (o) => {
+        const cleanup = (o: KeyValue): KeyValue => {
             delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
             return o;
         };
-        const oldOptions = cleanup(entity.settings);
+
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        const oldOptions = objectAssignDeep({}, cleanup(entity.settings));
+        settings.changeEntityOptions(ID, message.options);
         const newOptions = cleanup(settings.getEntity(ID));
         await this.publishInfo();
 
@@ -397,18 +398,17 @@ class Bridge extends Extension {
         return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
     }
 
-    async deviceConfigureReporting(message) {
+    @bind async deviceConfigureReporting(message: string | KeyValue): Promise<MQTTResponse> {
         if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('cluster') ||
             !message.hasOwnProperty('maximum_report_interval') || !message.hasOwnProperty('minimum_report_interval') ||
             !message.hasOwnProperty('reportable_change') || !message.hasOwnProperty('attribute')) {
             throw new Error(`Invalid payload`);
         }
 
-        const ID = message.id;
-        const endpoint = this.getEntity('device', ID).endpoint;
+        const parsedID = utils.parseEntityID(message.id);
+        const endpoint = (this.getEntity('device', parsedID.ID) as Device).endpoint(parsedID.endpoint);
 
-        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
-        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        const coordinatorEndpoint = this.zigbee.getFirstCoordinatorEndpoint();
         await endpoint.bind(message.cluster, coordinatorEndpoint);
 
         await endpoint.configureReporting(message.cluster, [{
@@ -427,7 +427,7 @@ class Bridge extends Extension {
         }, null);
     }
 
-    renameEntity(entityType, message) {
+    async renameEntity(entityType: 'group' | 'device', message: string | KeyValue): Promise<MQTTResponse> {
         const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
         if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
             !message.hasOwnProperty('to')) {
@@ -443,19 +443,19 @@ class Bridge extends Extension {
         const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
             message.homeassistant_rename : false;
         const entity = this.getEntity(entityType, from);
+        const oldFriendlyName = entity.settings.friendlyName;
 
         settings.changeFriendlyName(from, to);
 
         // Clear retained messages
-        this.mqtt.publish(entity.name, '', {retain: true});
+        this.mqtt.publish(oldFriendlyName, '', {retain: true});
 
-        const oldFriendlyName = entity.settings.friendlyName;
-        if (entity.type === 'device') {
+        if (entity instanceof Device) {
             this.publishDevices();
-            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename, from: oldFriendlyName, to});
+            this.eventBus.emit(`deviceRenamed`, {device: entity, homeAssisantRename, from: oldFriendlyName, to});
         } else {
             this.publishGroups();
-            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename, from: oldFriendlyName, to});
+            this.publishInfo();
         }
 
         // Repulish entity state
@@ -468,9 +468,11 @@ class Bridge extends Extension {
         );
     }
 
-    async removeEntity(entityType, message) {
+    async removeEntity(entityType: 'group' | 'device', message: string | KeyValue): Promise<MQTTResponse> {
         const ID = typeof message === 'object' ? message.id : message.trim();
         const entity = this.getEntity(entityType, ID);
+        const friendlyName = entity.name;
+        const entityID = entity.ID;
 
         let block = false;
         let force = false;
@@ -486,46 +488,46 @@ class Bridge extends Extension {
         }
 
         try {
-            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
-            if (entity.type === 'device') {
+            logger.info(`Removing ${entityType} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity instanceof Device) {
                 if (block) {
-                    settings.blockDevice(entity.settings.ID);
+                    settings.blockDevice(entity.ieeeAddr);
                 }
 
                 if (force) {
-                    await entity.device.removeFromDatabase();
+                    await entity.zhDevice.removeFromDatabase();
                 } else {
-                    await entity.device.removeFromNetwork();
+                    await entity.zhDevice.removeFromNetwork();
                 }
             } else {
                 if (force) {
-                    await entity.group.removeFromDatabase();
+                    entity.zhGroup.removeFromDatabase();
                 } else {
-                    await entity.group.removeFromNetwork();
+                    await entity.zhGroup.removeFromNetwork();
                 }
             }
 
             // Fire event
-            if (entity.type === 'device') {
+            if (entity instanceof Device) {
                 this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
             }
 
             // Remove from configuration.yaml
-            if (entity.type === 'device') {
-                settings.removeDevice(entity.settings.ID);
+            if (entity instanceof Device) {
+                settings.removeDevice(entityID as string);
             } else {
-                settings.removeGroup(entity.settings.ID);
+                settings.removeGroup(entityID);
             }
 
             // Remove from state
-            this.state.remove(entity.settings.ID);
+            this.state.remove(entityID);
 
             // Clear any retained messages
-            this.mqtt.publish(entity.name, '', {retain: true});
+            this.mqtt.publish(friendlyName, '', {retain: true});
 
-            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+            logger.info(`Successfully removed ${entityType} '${friendlyName}'${blockForceLog}`);
 
-            if (entity.type === 'device') {
+            if (entity instanceof Device) {
                 this.publishGroups();
                 this.publishDevices();
                 return utils.getResponse(message, {id: ID, block, force}, null);
@@ -535,21 +537,21 @@ class Bridge extends Extension {
             }
         } catch (error) {
             throw new Error(
-                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
+                `Failed to remove ${entityType} '${friendlyName}'${blockForceLog} (${error})`,
             );
         }
     }
 
-    getEntity(type, ID) {
+    getEntity(type: 'group' | 'device', ID: string): Device | Group {
         const entity = this.zigbee.resolveEntity(ID);
-        if (!entity || entity.type !== type) {
+        if (!entity || entity.constructor.name.toLowerCase() !== type) {
             throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
         }
         return entity;
     }
 
-    async publishInfo() {
-        const config = objectAssignDeep.noMutate({}, settings.get());
+    async publishInfo(): Promise<void> {
+        const config = objectAssignDeep({}, settings.get());
         delete config.advanced.network_key;
         delete config.mqtt.password;
         config.frontend && delete config.frontend.auth_token;
@@ -558,6 +560,7 @@ class Bridge extends Extension {
             commit: this.zigbee2mqttVersion.commitHash,
             coordinator: this.coordinatorVersion,
             network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
+            // @ts-ignore TODO fix type
             log_level: logger.getLevel(),
             permit_join: this.zigbee.getPermitJoin(),
             permit_join_timeout: this.zigbee.getPermitJoinTimeout(),
@@ -570,13 +573,18 @@ class Bridge extends Extension {
             'bridge/info', stringify(payload), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    async publishDevices() {
+    async publishDevices(): Promise<void> {
+        interface Data {
+            bindings: {cluster: string, target: {type: string, endpoint?: number, ieee_address?: string, id?: number}}[]
+            configured_reportings: {cluster: string, attribute: string | number,
+                minimum_report_interval: number, maximum_report_interval: number, reportable_change: number}[],
+            clusters: {input: string[], output: string[]}
+        }
+
         const devices = this.zigbee.getDevices().map((device) => {
-            const definition = zigbeeHerdsmanConverters.findByDevice(device);
-            const resolved = this.zigbee.resolveEntity(device);
-            const endpoints = {};
+            const endpoints: {[s: number]: Data} = {};
             for (const endpoint of device.endpoints) {
-                const data = {
+                const data: Data = {
                     bindings: [],
                     configured_reportings: [],
                     clusters: {
@@ -586,16 +594,9 @@ class Bridge extends Extension {
                 };
 
                 for (const bind of endpoint.binds) {
-                    let target;
-
-                    if (bind.target.constructor.name === 'Endpoint') {
-                        target = {
-                            type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID,
-                        };
-                    } else {
-                        target = {type: 'group', id: bind.target.groupID};
-                    }
-
+                    const target = utils.isEndpoint(bind.target) ?
+                        {type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID} :
+                        {type: 'group', id: bind.target.groupID};
                     data.bindings.push({cluster: bind.cluster.name, target});
                 }
 
@@ -616,71 +617,54 @@ class Bridge extends Extension {
                 ieee_address: device.ieeeAddr,
                 type: device.type,
                 network_address: device.networkAddress,
-                supported: !!definition,
-                friendly_name: resolved.name,
-                definition: this.getDefinitionPayload(definition, resolved),
+                supported: !!device.definition,
+                friendly_name: device.name,
+                definition: this.getDefinitionPayload(device),
                 power_source: device.powerSource,
                 software_build_id: device.softwareBuildID,
                 date_code: device.dateCode,
                 model_id: device.modelID,
                 interviewing: device.interviewing,
                 interview_completed: device.interviewCompleted,
+                manufacturer: device.manufacturerName,
                 endpoints,
             };
         });
 
-        await this.mqtt.publish(
-            'bridge/devices', stringify(devices), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+        await this.mqtt.publish('bridge/devices', stringify(devices),
+            {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    async publishGroups() {
-        const groups = this.zigbee.getGroups().map((group) => {
-            const resolved = this.zigbee.resolveEntity(group);
+    async publishGroups(): Promise<void> {
+        const groups = this.zigbee.getGroups().map((g) => {
             return {
-                id: group.groupID,
-                friendly_name: group.groupID === 901 ? 'default_bind_group' : resolved.name,
-                members: group.members.map((m) => {
-                    return {
-                        ieee_address: m.deviceIeeeAddress,
-                        endpoint: m.ID,
-                    };
+                id: g.ID,
+                friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
+                members: g.members.map((e) => {
+                    return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};
                 }),
             };
         });
-
         await this.mqtt.publish(
             'bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    getDefinitionPayload(definition, resolvedDevice) {
-        if (definition) {
-            let icon = null;
-            if (resolvedDevice && resolvedDevice.hasOwnProperty('settings')) {
-                icon = resolvedDevice.settings.icon;
-            }
-            if (!icon) {
-                icon = definition.icon;
-            }
-            if (icon) {
-                let zigbeeModel = '';
-                /* istanbul ignore else */
-                if (resolvedDevice && resolvedDevice.hasOwnProperty('device')) {
-                    zigbeeModel = resolvedDevice.device.modelID;
-                }
-                icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(zigbeeModel));
-                icon = icon.replace('${model}', utils.sanitizeImageParameter(definition.model));
-            }
-            return {
-                model: definition.model,
-                vendor: definition.vendor,
-                description: definition.description,
-                exposes: definition.exposes,
-                supports_ota: !!definition.ota,
-                icon,
-            };
-        } else {
-            return null;
+    getDefinitionPayload(device: Device):
+        {model: string, vendor: string, description: string, exposes: unknown[], supports_ota: boolean, icon: string} {
+        if (!device.definition) return null;
+        let icon = device.settings.icon ? device.settings.icon : device.definition.icon;
+        if (icon) {
+            icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(device.modelID));
+            icon = icon.replace('${model}', utils.sanitizeImageParameter(device.definition.model));
         }
+        return {
+            model: device.definition.model,
+            vendor: device.definition.vendor,
+            description: device.definition.description,
+            exposes: device.definition.exposes,
+            supports_ota: !!device.definition.ota,
+            icon,
+        };
     }
 }
 

lib/extension/configure.js

@@ -1,156 +0,0 @@
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const logger = require('../util/logger');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-/**
- * This extension calls the zigbee-herdsman-converters definition configure() method
- */
-class Configure extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        this.configuring = new Set();
-        this.onReportingDisabled = this.onReportingDisabled.bind(this);
-        this.attempts = {};
-
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
-        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled);
-    }
-
-    onReportingDisabled(data) {
-        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
-        const device = data.device;
-
-        const resolvedEntity = this.zigbee.resolveEntity(device);
-        if (resolvedEntity.device.meta && resolvedEntity.device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(resolvedEntity)) {
-            this.configure(resolvedEntity, 'reporting_disabled');
-        }
-    }
-
-    shouldConfigure(resolvedEntity, event) {
-        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
-            return false;
-        }
-
-        if (resolvedEntity.device.meta &&
-            resolvedEntity.device.meta.hasOwnProperty('configured') &&
-            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
-            return false;
-        }
-
-        if (resolvedEntity.device.interviewing === true) {
-            return false;
-        }
-
-        // Only configure end devices when a message is received, otherwise it will likely fails as they are sleeping.
-        if (resolvedEntity.device.type === 'EndDevice' && event !== 'message_received') {
-            return false;
-        }
-
-        return true;
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (topic === this.legacyTopic) {
-            const resolvedEntity = this.zigbee.resolveEntity(message);
-            if (!resolvedEntity || resolvedEntity.type !== 'device') {
-                logger.error(`Device '${message}' does not exist`);
-                return;
-            }
-
-            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
-                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
-                return;
-            }
-
-            this.configure(resolvedEntity, true);
-        } else if (topic === this.topic) {
-            message = utils.parseJSON(message, message);
-            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
-            let error = null;
-
-            const resolvedEntity = this.zigbee.resolveEntity(ID);
-            if (!resolvedEntity || resolvedEntity.type !== 'device') {
-                error = `Device '${ID}' does not exist`;
-            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
-                error = `Device '${resolvedEntity.name}' cannot be configured`;
-            } else {
-                try {
-                    await this.configure(resolvedEntity, true, true);
-                } catch (e) {
-                    error = `Failed to configure (${e.message})`;
-                }
-            }
-
-            const response = utils.getResponse(message, {id: ID}, error);
-            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
-        }
-    }
-
-    async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
-
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            if (this.shouldConfigure(resolvedEntity, 'started')) {
-                await this.configure(resolvedEntity);
-            }
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        const device = data.device;
-
-        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(resolvedEntity, 'message_received')) {
-            this.configure(resolvedEntity);
-        }
-    }
-
-    async configure(resolvedEntity, force=false, thowError=false) {
-        const device = resolvedEntity.device;
-        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
-            return false;
-        }
-
-        this.configuring.add(device.ieeeAddr);
-
-        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
-            this.attempts[device.ieeeAddr] = 0;
-        }
-
-        logger.info(`Configuring '${resolvedEntity.name}'`);
-        try {
-            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint, logger);
-            logger.info(`Successfully configured '${resolvedEntity.name}'`);
-            device.meta.configured = resolvedEntity.definition.meta.configureKey;
-            device.save();
-            this.eventBus.emit(`devicesChanged`);
-        } catch (error) {
-            this.attempts[device.ieeeAddr]++;
-            const attempt = this.attempts[device.ieeeAddr];
-            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
-            logger.error(msg);
-
-            if (thowError) {
-                throw error;
-            }
-        }
-
-        this.configuring.delete(device.ieeeAddr);
-    }
-}
-
-module.exports = Configure;

lib/extension/configure.ts

@@ -0,0 +1,140 @@
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import logger from '../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends ExtensionTS {
+    private configuring = new Set();
+    private attempts: {[s: string]: number} = {};
+    private topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
+    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+
+    @bind private async onReportingDisabled(data: EventReportingDisabled): Promise<void> {
+        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
+        const device = this.zigbee.resolveEntity(data.device) as Device; // TODO assert device
+
+        if (device.zhDevice.meta?.hasOwnProperty('configured')) {
+            delete device.zhDevice.meta.configured;
+            device.zhDevice.save();
+        }
+
+        await this.configure(device, 'reporting_disabled');
+    }
+
+    // TODO remove trailing _
+    @bind private async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        if (data.topic === this.legacyTopic) {
+            const device = this.zigbee.resolveEntity(data.message);
+            if (!device || !(device instanceof Device)) {
+                logger.error(`Device '${data.message}' does not exist`);
+                return;
+            }
+
+            if (!device.definition || !device.definition.configure) {
+                logger.warn(`Skipping configure of '${device.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(device, 'mqtt_message', true);
+        } else if (data.topic === this.topic) {
+            const message = utils.parseJSON(data.message, data.message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const device = this.zigbee.resolveEntity(ID);
+            if (!device || !(device instanceof Device)) {
+                error = `Device '${ID}' does not exist`;
+            } else if (!device.definition || !device.definition.configure) {
+                error = `Device '${device.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(device, 'mqtt_message', true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
+        }
+    }
+
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.getClients()) {
+            await this.configure(device, 'started');
+        }
+
+        this.eventBus.onDeviceJoined(this, (data) => {
+            if (data.device.zhDevice.meta.hasOwnProperty('configured')) {
+                delete data.device.zhDevice.meta.configured;
+                data.device.zhDevice.save();
+            }
+
+            this.configure(data.device, 'zigbee_event');
+        });
+        this.eventBus.onLastSeenChanged(this, (data) => this.configure(data.device, 'zigbee_event'));
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.eventBus.onReportingDisabled(this, this.onReportingDisabled);
+    }
+
+    private async configure(device: Device, event: 'started' | 'zigbee_event' | 'reporting_disabled' | 'mqtt_message',
+        force=false, thowError=false): Promise<void> {
+        if (!force) {
+            if (!device.definition?.configure || device.zhDevice.interviewing) {
+                return;
+            }
+
+            if (device.zhDevice.meta?.hasOwnProperty('configured') &&
+                device.zhDevice.meta.configured === zhc.getConfigureKey(device.definition)) {
+                return;
+            }
+
+            // Only configure end devices when it is active, otherwise it will likely fails as they are sleeping.
+            if (device.zhDevice.type === 'EndDevice' && event !== 'zigbee_event') {
+                return;
+            }
+        }
+
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${device.name}'`);
+        try {
+            await device.definition.configure(device.zhDevice, this.zigbee.getFirstCoordinatorEndpoint(), logger);
+            logger.info(`Successfully configured '${device.name}'`);
+            device.zhDevice.meta.configured = zhc.getConfigureKey(device.definition);
+            device.zhDevice.save();
+            this.eventBus.emitDevicesChanged();
+            this.eventBus.emit(`devicesChanged`);
+        } catch (error) {
+            this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${device.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/extension.js

@@ -30,7 +30,7 @@ class Extension {
      * Is called when a Zigbee message from a device is received.
      * @param {string} type Type of the message
      * @param {Object} data Data of the message
-     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntityLegacy()
      * @param {Object?} settingsDevice Device settings
      */
     // onZigbeeEvent(type, data, resolvedEntity) {}
@@ -47,7 +47,7 @@ class Extension {
      * Is called once the extension has to stop
      */
     stop() {
-        this.eventBus.removeListenersExtension(this.constructor.name);
+        this.eventBus.removeListeners(this.constructor.name);
     }
 }
 

lib/extension/extensionts.ts

@@ -0,0 +1,51 @@
+// TODO: tempState -> State, rename to extension
+abstract class ExtensionTS {
+    protected zigbee: Zigbee;
+    protected mqtt: MQTT;
+    protected state: TempState;
+    protected publishEntityState: PublishEntityState;
+    protected eventBus: EventBus;
+    protected enableDisableExtension: (enable: boolean, name: string) => Promise<void>;
+    protected restartCallback: () => void;
+    protected addExtension: (extension: ExternalConverterClass) => void;
+
+    /**
+     * Besides intializing variables, the constructor should do nothing!
+     *
+     * @param {Zigbee} zigbee Zigbee controller
+     * @param {MQTT} mqtt MQTT controller
+     * @param {State} state State controller
+     * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
+     * @param {enableDisableExtension} enableDisableExtension Enable/disable extension method
+     * @param {restartCallback} restartCallback Restart Zigbee2MQTT
+     * @param {addExtension} addExtension Add an extension
+     */
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: TempState, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: ExternalConverterClass) => void) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
+        this.enableDisableExtension = enableDisableExtension;
+        this.restartCallback = restartCallback;
+        this.addExtension = addExtension;
+    }
+
+    /**
+     * Is called once the extension has to start
+     */
+    /* istanbul ignore next */
+    async start(): Promise<void> {}
+
+    /**
+     * Is called once the extension has to stop
+     */
+    async stop(): Promise<void> {
+        this.eventBus.removeListeners(this);
+    }
+}
+
+export default ExtensionTS;

lib/extension/externalConverters.js

@@ -1,18 +0,0 @@
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const settings = require('../util/settings');
-const Extension = require('./extension');
-const utils = require('../util/utils');
-
-class ExternalConverters extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
-            const toAdd = {...definition};
-            delete toAdd['homeassistant'];
-            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
-        }
-    }
-}
-
-module.exports = ExternalConverters;

lib/extension/externalConverters.ts

@@ -0,0 +1,17 @@
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import ExtensionTS from './extensionts';
+
+class ExternalConverters extends ExtensionTS {
+    override async start(): Promise<void> {
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zhc.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/externalExtension.ts

@@ -1,32 +1,31 @@
-const settings = require('../util/settings');
-const Extension = require('./extension');
-const utils = require('../util/utils');
-const fs = require('fs');
-const data = require('./../util/data');
-const path = require('path');
-const logger = require('./../util/logger');
-const stringify = require('json-stable-stringify-without-jsonify');
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import fs from 'fs';
+import * as data from './../util/data';
+import path from 'path';
+import logger from './../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
+import ExtensionTS from './extensionts';
+
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
 
-class ExternalExtension extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus, addExtension, enableDisableExtension) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
-        this.addExtension = addExtension;
-        this.enableDisableExtension = enableDisableExtension;
-        this.extensionsBaseDir = 'extension';
-        this.loadExtension = this.loadExtension.bind(this);
-        this.requestLookup = {
-            'save': this.saveExtension.bind(this),
-            'remove': this.removeExtension.bind(this),
-        };
+class ExternalExtension extends ExtensionTS {
+    private requestLookup: {[s: string]: (message: KeyValue) => MQTTResponse};
+
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.requestLookup = {'save': this.saveExtension, 'remove': this.removeExtension};
         this.loadUserDefinedExtensions();
-    }
-    getExtensionsBasePath() {
-        return data.joinPath(this.extensionsBaseDir);
+        await this.publishExtensions();
     }
 
-    getListOfUserDefinedExtensions() {
+    private getExtensionsBasePath(): string {
+        return data.joinPath('extension');
+    }
+
+    private getListOfUserDefinedExtensions(): {name: string, code: string}[] {
         const basePath = this.getExtensionsBasePath();
         if (fs.existsSync(basePath)) {
             return fs.readdirSync(basePath).filter((f) => f.endsWith('.js')).map((fileName) => {
@@ -37,7 +36,8 @@ class ExternalExtension extends Extension {
             return [];
         }
     }
-    removeExtension(message) {
+
+    @bind private removeExtension(message: KeyValue): MQTTResponse {
         const {name} = message;
         const extensions = this.getListOfUserDefinedExtensions();
         const extensionToBeRemoved = extensions.find((e) => e.name === name);
@@ -45,8 +45,8 @@ class ExternalExtension extends Extension {
         if (extensionToBeRemoved) {
             this.enableDisableExtension(false, extensionToBeRemoved.name);
             const basePath = this.getExtensionsBasePath();
-            const extensonFilePath = path.join(basePath, path.basename(name));
-            fs.unlinkSync(extensonFilePath);
+            const extensionFilePath = path.join(basePath, path.basename(name));
+            fs.unlinkSync(extensionFilePath);
             this.publishExtensions();
             logger.info(`Extension ${name} removed`);
             return utils.getResponse(message, {}, null);
@@ -54,9 +54,10 @@ class ExternalExtension extends Extension {
             return utils.getResponse(message, {}, `Extension ${name} doesn't exists`);
         }
     }
-    saveExtension(message) {
+
+    @bind private saveExtension(message: KeyValue): MQTTResponse {
         const {name, code} = message;
-        const ModuleConstructor = utils.loadModuleFromText(code);
+        const ModuleConstructor = utils.loadModuleFromText(code) as ExternalConverterClass;
         this.loadExtension(ModuleConstructor);
         const basePath = this.getExtensionsBasePath();
         /* istanbul ignore else */
@@ -70,38 +71,35 @@ class ExternalExtension extends Extension {
         return utils.getResponse(message, {}, null);
     }
 
-
-    async onMQTTMessage(topic, message) {
-        const match = topic.match(requestRegex);
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        const match = data.topic.match(requestRegex);
         if (match && this.requestLookup[match[1].toLowerCase()]) {
-            message = utils.parseJSON(message, message);
+            const message = utils.parseJSON(data.message, data.message) as KeyValue;
             try {
-                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                const response = this.requestLookup[match[1].toLowerCase()](message);
                 await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
             } catch (error) {
-                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
                 const response = utils.getResponse(message, {}, error.message);
                 await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
             }
         }
     }
 
-    loadExtension(ConstructorClass) {
+    @bind private loadExtension(ConstructorClass: ExternalConverterClass): void {
         this.enableDisableExtension(false, ConstructorClass.name);
-        this.addExtension(new ConstructorClass(...this.args, settings, logger));
+        this.addExtension(new ConstructorClass(
+            this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus, settings, logger));
     }
 
-    loadUserDefinedExtensions() {
+    private loadUserDefinedExtensions(): void {
         const extensions = this.getListOfUserDefinedExtensions();
         extensions
             .map(({code}) => utils.loadModuleFromText(code))
             .map(this.loadExtension);
     }
-    async onMQTTConnected() {
-        this.publishExtensions();
-    }
 
-    async publishExtensions() {
+    private async publishExtensions(): Promise<void> {
         const extensions = this.getListOfUserDefinedExtensions();
         await this.mqtt.publish('bridge/extensions', stringify(extensions), {
             retain: true,

lib/extension/frontend.js

@@ -1,128 +0,0 @@
-const http = require('http');
-const serveStatic = require('serve-static');
-const finalhandler = require('finalhandler');
-const Extension = require('./extension');
-const logger = require('../util/logger');
-const frontend = require('zigbee2mqtt-frontend');
-const WebSocket = require('ws');
-const url = require('url');
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-/**
- * This extension servers the frontend
- */
-class Frontend extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.onRequest = this.onRequest.bind(this);
-        this.onUpgrade = this.onUpgrade.bind(this);
-        this.mqttBaseTopic = settings.get().mqtt.base_topic;
-        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
-        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
-        this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
-        this.server = http.createServer(this.onRequest);
-        this.server.on('upgrade', this.onUpgrade);
-        this.host = settings.get().frontend.host || '0.0.0.0';
-        this.port = settings.get().frontend.port || 8080;
-        this.authToken = settings.get().frontend.auth_token || false;
-        this.retainedMessages = new Map();
-        /* istanbul ignore next */
-        const options = {setHeaders: (res, path) => {
-            if (path.endsWith('index.html')) {
-                res.setHeader('Cache-Control', 'no-store');
-            }
-        }};
-        this.fileServer = serveStatic(frontend.getPath(), options);
-        this.wss = new WebSocket.Server({noServer: true});
-        this.wss.on('connection', this.onWebSocketConnection);
-    }
-
-    onZigbeeStarted() {
-        this.server.listen(this.port, this.host);
-        logger.info(`Started frontend on port ${this.host}:${this.port}`);
-    }
-
-    async stop() {
-        for (const client of this.wss.clients) {
-            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
-            client.close();
-        }
-
-        return new Promise((resolve) => {
-            this.server.close(resolve);
-        });
-    }
-    onRequest(request, response) {
-        this.fileServer(request, response, finalhandler(request, response));
-    }
-    authenticate(request, cb) {
-        const {query} = url.parse(request.url, true);
-        cb(!this.authToken || this.authToken === query.token);
-    }
-
-    onUpgrade(request, socket, head) {
-        this.wss.handleUpgrade(request, socket, head, (ws) => {
-            this.authenticate(request, (isAuthentificated) => {
-                if (isAuthentificated) {
-                    this.wss.emit('connection', ws, request);
-                } else {
-                    ws.close(4401, 'Unauthorized');
-                }
-            });
-        });
-    }
-
-    onWebSocketConnection(ws) {
-        ws.on('message', (message) => {
-            if (message) {
-                const {topic, payload} = utils.parseJSON(message, message);
-                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
-            }
-        });
-
-        for (const [key, value] of this.retainedMessages) {
-            ws.send(stringify({topic: key, payload: value}));
-        }
-
-        for (const device of this.zigbee.getClients()) {
-            let payload = {};
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            if (this.state.exists(device.ieeeAddr)) {
-                payload = {...payload, ...this.state.get(device.ieeeAddr)};
-            }
-
-            const lastSeen = settings.get().advanced.last_seen;
-            if (lastSeen !== 'disable') {
-                payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
-            }
-
-            if (resolvedEntity.device.linkquality !== undefined) {
-                payload.linkquality = resolvedEntity.device.linkquality;
-            }
-
-            ws.send(stringify({topic: resolvedEntity.name, payload}));
-        }
-    }
-
-    onMQTTPublishedMessage(data) {
-        let {topic, payload, options} = data;
-        if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
-            // Send topic without base_topic
-            topic = topic.substring(this.mqttBaseTopic.length + 1);
-            payload = utils.parseJSON(payload, payload);
-            if (options.retain) {
-                this.retainedMessages.set(topic, payload);
-            }
-
-            for (const client of this.wss.clients) {
-                if (client.readyState === WebSocket.OPEN) {
-                    client.send(stringify({topic, payload}));
-                }
-            }
-        }
-    }
-}
-
-module.exports = Frontend;

lib/extension/frontend.ts

@@ -0,0 +1,143 @@
+import http from 'http';
+import serveStatic from 'serve-static';
+import finalhandler from 'finalhandler';
+import logger from '../util/logger';
+// @ts-ignore
+import frontend from 'zigbee2mqtt-frontend';
+import WebSocket from 'ws';
+import net from 'net';
+import url from 'url';
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+
+/**
+ * This extension servers the frontend
+ */
+class Frontend extends ExtensionTS {
+    private mqttBaseTopic = settings.get().mqtt.base_topic;
+    private host = settings.get().frontend.host || '0.0.0.0';
+    private port = settings.get().frontend.port || 8080;
+    private authToken = settings.get().frontend.auth_token || false;
+    private retainedMessages = new Map();
+    private server: http.Server;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    private fileServer: serveStatic.RequestHandler<any>;
+    private wss: WebSocket.Server = null;
+
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: TempState, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: ExternalConverterClass) => void) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
+        this.eventBus.onMQTTMessagePublished(this, this.onMQTTPublishMessage);
+    }
+
+    override async start(): Promise<void> {
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+
+        /* istanbul ignore next */ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const options = {setHeaders: (res: any, path: any): void => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        for (const client of this.wss.clients) {
+            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
+            client.terminate();
+        }
+        this.wss.close();
+        return new Promise((cb: () => void) => this.server.close(cb));
+    }
+
+    @bind private onRequest(request: http.IncomingMessage, response: http.ServerResponse): void {
+        // @ts-ignore
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+
+    private authenticate(request: http.IncomingMessage, cb: (authenticate: boolean) => void): void {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    @bind private onUpgrade(request: http.IncomingMessage, socket: net.Socket, head: Buffer): void {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    @bind private onWebSocketConnection(ws: WebSocket): void {
+        ws.on('message', (data: Buffer, isBinary: boolean) => {
+            if (!isBinary && data) {
+                const message = data.toString();
+                const {topic, payload} = JSON.parse(message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            let payload: KeyValue = {};
+            if (this.state.exists(device.ieeeAddr)) {
+                payload = {...payload, ...this.state.get(device.ieeeAddr)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            /* istanbul ignore if */
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(device.lastSeen, lastSeen);
+            }
+
+            if (device.zhDevice.linkquality !== undefined) {
+                payload.linkquality = device.zhDevice.linkquality;
+            }
+
+            ws.send(stringify({topic: device.name, payload}));
+        }
+    }
+
+    @bind private onMQTTPublishMessage(data: EventMQTTMessagePublished): void {
+        if (data.topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            const topic = data.topic.substring(this.mqttBaseTopic.length + 1);
+            const payload = utils.parseJSON(data.payload, data.payload);
+            if (data.options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            if (this.wss) {
+                for (const client of this.wss.clients) {
+                    /* istanbul ignore else */
+                    if (client.readyState === WebSocket.OPEN) {
+                        client.send(stringify({topic, payload}));
+                    }
+                }
+            }
+        }
+    }
+}
+
+module.exports = Frontend;

lib/extension/groups.js

@@ -2,7 +2,6 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 const utils = require('../util/utils');
-const postfixes = utils.getEndpointNames();
 const stringify = require('json-stable-stringify-without-jsonify');
 const equals = require('fast-deep-equal/es6');
 
@@ -11,6 +10,21 @@ const topicRegex =
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
+const stateProperties = {
+    'state': (value, definition) => true,
+    'brightness': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
 class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
@@ -20,13 +34,13 @@ class Groups extends Extension {
     }
 
     async onZigbeeStarted() {
-        this.eventBus.on('stateChange', this.onStateChange);
+        this.eventBus.on('stateChange', this.onStateChange, this.constructor.name);
         await this.syncGroupsWithSettings();
     }
 
     async syncGroupsWithSettings() {
         const settingsGroups = settings.getGroups();
-        const zigbeeGroups = this.zigbee.getGroups();
+        const zigbeeGroups = this.zigbee.getGroupsLegacy();
 
         const addRemoveFromGroup = async (action, deviceName, groupName, endpoint, group) => {
             try {
@@ -38,14 +52,16 @@ class Groups extends Extension {
                 }
             } catch (error) {
                 logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+                logger.debug(error.stack);
             }
         };
 
         for (const settingGroup of settingsGroups) {
             const groupID = settingGroup.ID;
-            const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
+            const zigbeeGroup = zigbeeGroups.find((g) =>
+                g.groupID === groupID) || this.zigbee.createGroupLegacy(groupID);
             const settingsEntity = settingGroup.devices.map((d) => {
-                const entity = this.zigbee.resolveEntity(d);
+                const entity = this.zigbee.resolveEntityLegacy(d);
                 if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendlyName}'`);
                 return entity;
             }).filter((e) => e != null);
@@ -82,27 +98,27 @@ class Groups extends Extension {
             return;
         }
 
-        const properties = ['state', 'brightness', 'color_temp', 'color', 'color_mode'];
         const payload = {};
 
         let endpointName;
         for (let [prop, value] of Object.entries(data.update)) {
-            const endpointNameMatch = utils.getEndpointNames().find((n) => prop.endsWith(`_${n}`));
+            const endpointNameMatch = utils.endpointNames.find((n) => prop.endsWith(`_${n}`));
             if (endpointNameMatch) {
                 prop = prop.substring(0, prop.length - endpointNameMatch.length - 1);
                 endpointName = endpointNameMatch;
             }
 
-            if (properties.includes(prop)) {
+            if (prop in stateProperties) {
                 payload[prop] = value;
             }
         }
 
         if (Object.keys(payload).length) {
-            const resolvedEntity = this.zigbee.resolveEntity(`${data.ID}${endpointName ? `/${endpointName}` : ''}`);
-            const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
+            const resolvedEntity =
+                this.zigbee.resolveEntityLegacy(`${data.ID}${endpointName ? `/${endpointName}` : ''}`);
+            const zigbeeGroups = this.zigbee.getGroupsLegacy().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
-                return settingsGroup && settingsGroup.optimistic;
+                return settingsGroup && (!settingsGroup.hasOwnProperty('optimistic') || settingsGroup.optimistic);
             });
 
             if (resolvedEntity.type === 'device') {
@@ -121,8 +137,15 @@ class Groups extends Extension {
 
                 const groupIDsToPublish = new Set();
                 for (const member of resolvedEntity.group.members) {
-                    const memberPayload = {...payload};
-                    const endpointName = this.zigbee.resolveEntity(member).endpointName;
+                    const resolvedEntity = this.zigbee.resolveEntityLegacy(member);
+                    const memberPayload = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], resolvedEntity.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = resolvedEntity.endpointName;
                     if (endpointName) {
                         Object.keys(memberPayload).forEach((key) => {
                             memberPayload[`${key}_${endpointName}`] = memberPayload[key];
@@ -169,6 +192,7 @@ class Groups extends Extension {
         let groupKey = null;
         let deviceKey = null;
         let triggeredViaLegacyApi = false;
+        let skipDisableReporting = false;
 
         /* istanbul ignore else */
         const topicRegexMatch = topic.match(topicRegex);
@@ -178,7 +202,7 @@ class Groups extends Extension {
         if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
             triggeredViaLegacyApi = true;
             if (legacyTopicRegexMatch) {
-                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
+                resolvedEntityGroup = this.zigbee.resolveEntityLegacy(legacyTopicRegexMatch[1]);
                 type = legacyTopicRegexMatch[2];
 
                 if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
@@ -201,7 +225,7 @@ class Groups extends Extension {
                 type = 'remove_all';
             }
 
-            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            resolvedEntityDevice = this.zigbee.resolveEntityLegacy(message);
             if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
                 logger.error(`Device '${message}' does not exist`);
 
@@ -219,38 +243,39 @@ class Groups extends Extension {
                 return {};
             }
 
-            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+            hasEndpointName = utils.endpointNames.find((p) => message.endsWith(`/${p}`));
         } else if (topicRegexMatch) {
             type = topicRegexMatch[1];
             message = JSON.parse(message);
             deviceKey = message.device;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
 
             if (type !== 'remove_all') {
                 groupKey = message.group;
-                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
+                resolvedEntityGroup = this.zigbee.resolveEntityLegacy(message.group);
                 if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
                     error = `Group '${message.group}' does not exist`;
                 }
             }
 
-            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
+            resolvedEntityDevice = this.zigbee.resolveEntityLegacy(message.device);
             if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
                 error = `Device '${message.device}' does not exist`;
             }
 
-            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
+            hasEndpointName = utils.endpointNames.find((p) => message.device.endsWith(`/${p}`));
         }
 
         return {
             resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
-            triggeredViaLegacyApi,
+            triggeredViaLegacyApi, skipDisableReporting,
         };
     }
 
     async onMQTTMessage(topic, message) {
         let {
             resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
-            groupKey, deviceKey,
+            groupKey, deviceKey, skipDisableReporting,
         } = this.parseMQTTMessage(topic, message);
         if (!type) return;
         message = utils.parseJSON(message, message);
@@ -270,8 +295,11 @@ class Groups extends Extension {
                 const definition = resolvedEntityDevice.definition;
                 const endpoints = definition && definition.endpoint ?
                     definition.endpoint(resolvedEntityDevice.device) : null;
-                const endpointName = endpoints ?
-                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+                let endpointName = null;
+                if (endpoints) {
+                    const matches = Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID);
+                    endpointName = matches && matches[0];
+                }
 
                 if (endpointName) {
                     keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
@@ -327,6 +355,7 @@ class Groups extends Extension {
                 }
             } catch (e) {
                 error = `Failed to ${type} from group (${e.message})`;
+                logger.debug(e.stack);
             }
         }
 
@@ -339,7 +368,8 @@ class Groups extends Extension {
             logger.error(error);
         } else {
             this.eventBus.emit('groupMembersChanged',
-                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint});
+                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint,
+                    skipDisableReporting: skipDisableReporting});
         }
     }
 }

lib/extension/homeassistant.js

@@ -1,966 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const zigbee2mqttVersion = require('../../package.json').version;
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const assert = require('assert');
-
-const sensorClick = {
-    type: 'sensor',
-    object_id: 'click',
-    discovery_payload: {
-        icon: 'mdi:toggle-switch',
-        value_template: '{{ value_json.click }}',
-    },
-};
-
-const ACCESS_STATE = 1;
-const defaultStatusTopic = 'homeassistant/status';
-
-/**
- * This extensions handles integration with HomeAssistant
- */
-class HomeAssistant extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        // A map of all discoverd devices
-        this.discovered = {};
-        this.mapping = {};
-        this.discoveredTriggers = {};
-        this.legacyApi = settings.get().advanced.legacy_api;
-
-        if (!settings.get().advanced.cache_state) {
-            logger.warn('In order for HomeAssistant integration to work properly set `cache_state: true');
-        }
-
-        if (settings.get().experimental.output === 'attribute') {
-            throw new Error('Home Assitant integration is not possible with attribute output!');
-        }
-
-        this.discoveryTopic = settings.get().advanced.homeassistant_discovery_topic;
-        this.statusTopic = settings.get().advanced.homeassistant_status_topic;
-
-        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
-        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data), this.constructor.name);
-        this.eventBus.on('deviceRenamed', (data) =>
-            this.onDeviceRenamed(data.device, data.homeAssisantRename), this.constructor.name,
-        );
-
-        this.populateMapping();
-    }
-
-    populateMapping() {
-        for (const def of zigbeeHerdsmanConverters.definitions) {
-            this.mapping[def.model] = [];
-
-            if (['WXKG01LM', 'HS1EB/HS1EB-E', 'ICZB-KPD14S', 'TERNCY-SD01', 'TERNCY-PP01', 'ICZB-KPD18S',
-                'E1766', 'ZWallRemote0', 'ptvo.switch', '2AJZ4KPKEY', 'ZGRC-KEY-013', 'HGZB-02S', 'HGZB-045',
-                'HGZB-1S', 'AV2010/34', 'IM6001-BTP01', 'WXKG11LM', 'WXKG03LM', 'WXKG02LM_rev1', 'WXKG02LM_rev2',
-                'QBKG04LM', 'QBKG03LM', 'QBKG11LM', 'QBKG21LM', 'QBKG22LM', 'WXKG12LM', 'QBKG12LM',
-                'E1743'].includes(def.model)) {
-                // deprecated
-                this.mapping[def.model].push(sensorClick);
-            }
-
-            if (['ICTC-G-1'].includes(def.model)) {
-                // deprecated
-                this.mapping[def.model].push({
-                    type: 'sensor',
-                    object_id: 'brightness',
-                    discovery_payload: {
-                        unit_of_measurement: 'brightness',
-                        icon: 'mdi:brightness-5',
-                        value_template: '{{ value_json.brightness }}',
-                    },
-                });
-            }
-
-            for (const expose of def.exposes) {
-                let discoveryEntry = null;
-                /* istanbul ignore else */
-                if (expose.type === 'light') {
-                    const colorXY = expose.features.find((e) => e.name === 'color_xy');
-                    const colorHS = expose.features.find((e) => e.name === 'color_hs');
-                    const brightness = expose.features.find((e) => e.name === 'brightness');
-                    const colorTemp = expose.features.find((e) => e.name === 'color_temp');
-
-                    discoveryEntry = {
-                        type: 'light',
-                        object_id: expose.endpoint ? `light_${expose.endpoint}` : 'light',
-                        discovery_payload: {
-                            brightness: !!brightness,
-                            schema: 'json',
-                            command_topic: true,
-                            brightness_scale: 254,
-                            command_topic_prefix: expose.endpoint ? expose.endpoint : undefined,
-                            state_topic_postfix: expose.endpoint ? expose.endpoint : undefined,
-                        },
-                    };
-
-                    const colorModes = [
-                        colorXY ? 'xy' : null,
-                        !colorXY && colorHS ? 'hs' : null,
-                        colorTemp ? 'color_temp' : null,
-                    ].filter((c) => c);
-
-                    if (colorModes.length) {
-                        discoveryEntry.discovery_payload.color_mode = true;
-                        discoveryEntry.discovery_payload.supported_color_modes = colorModes;
-                    }
-
-
-                    if (colorTemp) {
-                        discoveryEntry.discovery_payload.max_mireds = colorTemp.value_max;
-                        discoveryEntry.discovery_payload.min_mireds = colorTemp.value_min;
-                    }
-
-                    const effect = def.exposes.find((e) => e.type === 'enum' && e.name === 'effect');
-                    if (effect) {
-                        discoveryEntry.discovery_payload.effect = true;
-                        discoveryEntry.discovery_payload.effect_list = effect.values;
-                    }
-                } else if (expose.type === 'switch') {
-                    const state = expose.features.find((f) => f.name === 'state');
-                    discoveryEntry = {
-                        type: 'switch',
-                        object_id: expose.endpoint ? `switch_${expose.endpoint}` : 'switch',
-                        discovery_payload: {
-                            payload_off: state.value_off,
-                            payload_on: state.value_on,
-                            value_template: `{{ value_json.${state.property} }}`,
-                            command_topic: true,
-                            command_topic_prefix: expose.endpoint ? expose.endpoint : undefined,
-                        },
-                    };
-
-                    const different = ['valve_detection', 'window_detection', 'auto_lock', 'away_mode'];
-                    if (different.includes(state.property)) {
-                        discoveryEntry.discovery_payload.command_topic_postfix = state.property;
-                        discoveryEntry.discovery_payload.state_off = state.value_off;
-                        discoveryEntry.discovery_payload.state_on = state.value_on;
-                        discoveryEntry.discovery_payload.state_topic = true;
-                        discoveryEntry.object_id = state.property;
-
-                        if (state.property === 'window_detection') {
-                            discoveryEntry.discovery_payload.icon = 'mdi:window-open-variant';
-                        }
-                    }
-                } else if (expose.type === 'climate') {
-                    const setpointProperties = ['occupied_heating_setpoint', 'current_heating_setpoint'];
-                    const setpoint = expose.features.find((f) => setpointProperties.includes(f.name));
-                    assert(setpoint, 'No setpoint found');
-                    const temperature = expose.features.find((f) => f.name === 'local_temperature');
-                    assert(temperature, 'No temperature found');
-
-                    discoveryEntry = {
-                        type: 'climate',
-                        object_id: expose.endpoint ? `climate_${expose.endpoint}` : 'climate',
-                        discovery_payload: {
-                            // Static
-                            state_topic: false,
-                            temperature_unit: 'C',
-                            // Setpoint
-                            temp_step: setpoint.value_step,
-                            min_temp: setpoint.value_min.toString(),
-                            max_temp: setpoint.value_max.toString(),
-                            // Temperature
-                            current_temperature_topic: true,
-                            current_temperature_template: `{{ value_json.${temperature.property} }}`,
-                        },
-                    };
-
-                    const mode = expose.features.find((f) => f.name === 'system_mode');
-                    if (mode) {
-                        if (mode.values.includes('sleep')) {
-                            // 'sleep' is not supported by homeassistent, but is valid according to ZCL
-                            // TRV that support sleep (e.g. Viessmann) will have it removed from here,
-                            // this allows other expose consumers to still use it, e.g. the frontend.
-                            mode.values.splice(mode.values.indexOf('sleep'), 1);
-                        }
-                        discoveryEntry.discovery_payload.mode_state_topic = true;
-                        discoveryEntry.discovery_payload.mode_state_template = `{{ value_json.${mode.property} }}`;
-                        discoveryEntry.discovery_payload.modes = mode.values;
-                        discoveryEntry.discovery_payload.mode_command_topic = true;
-                    }
-
-                    const state = expose.features.find((f) => f.name === 'running_state');
-                    if (state) {
-                        discoveryEntry.discovery_payload.action_topic = true;
-                        discoveryEntry.discovery_payload.action_template = `{% set values = ` +
-                                `{'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'}` +
-                                ` %}{{ values[value_json.${state.property}] }}`;
-                    }
-
-                    const coolingSetpoint = expose.features.find((f) => f.name === 'occupied_cooling_setpoint');
-                    if (coolingSetpoint) {
-                        discoveryEntry.discovery_payload.temperature_low_command_topic = setpoint.name;
-                        discoveryEntry.discovery_payload.temperature_low_state_template =
-                            `{{ value_json.${setpoint.property} }}`;
-                        discoveryEntry.discovery_payload.temperature_low_state_topic = true;
-                        discoveryEntry.discovery_payload.temperature_high_command_topic = coolingSetpoint.name;
-                        discoveryEntry.discovery_payload.temperature_high_state_template =
-                            `{{ value_json.${coolingSetpoint.property} }}`;
-                        discoveryEntry.discovery_payload.temperature_high_state_topic = true;
-                    } else {
-                        discoveryEntry.discovery_payload.temperature_command_topic = setpoint.name;
-                        discoveryEntry.discovery_payload.temperature_state_template =
-                            `{{ value_json.${setpoint.property} }}`;
-                        discoveryEntry.discovery_payload.temperature_state_topic = true;
-                    }
-
-                    const fanMode = expose.features.find((f) => f.name === 'fan_mode');
-                    if (fanMode) {
-                        discoveryEntry.discovery_payload.fan_modes = fanMode.values;
-                        discoveryEntry.discovery_payload.fan_mode_command_topic = true;
-                        discoveryEntry.discovery_payload.fan_mode_state_template =
-                            `{{ value_json.${fanMode.property} }}`;
-                        discoveryEntry.discovery_payload.fan_mode_state_topic = true;
-                    }
-
-                    const preset = expose.features.find((f) => f.name === 'preset');
-                    if (preset) {
-                        discoveryEntry.discovery_payload.hold_modes = preset.values;
-                        discoveryEntry.discovery_payload.hold_command_topic = true;
-                        discoveryEntry.discovery_payload.hold_state_template =
-                            `{{ value_json.${preset.property} }}`;
-                        discoveryEntry.discovery_payload.hold_state_topic = true;
-                    }
-
-                    const awayMode = expose.features.find((f) => f.name === 'away_mode');
-                    if (awayMode) {
-                        discoveryEntry.discovery_payload.away_mode_command_topic = true;
-                        discoveryEntry.discovery_payload.away_mode_state_topic = true;
-                        discoveryEntry.discovery_payload.away_mode_state_template =
-                            `{{ value_json.${awayMode.property} }}`;
-                    }
-
-                    if (expose.endpoint) {
-                        discoveryEntry.discovery_payload.state_topic_postfix = expose.endpoint;
-                    }
-                } else if (expose.type === 'lock') {
-                    assert(!expose.endpoint, `Endpoint not supported for lock type`);
-                    const state = expose.features.find((f) => f.name === 'state');
-                    assert(state, 'No state found');
-                    discoveryEntry = {
-                        type: 'lock',
-                        object_id: 'lock',
-                        discovery_payload: {
-                            command_topic: true,
-                            value_template: `{{ value_json.${state.property} }}`,
-                        },
-                    };
-
-                    if (state.property === 'keypad_lockout') {
-                        // deprecated: keypad_lockout is messy, but changing is breaking
-                        discoveryEntry.discovery_payload.payload_lock = state.value_on;
-                        discoveryEntry.discovery_payload.payload_unlock = state.value_off;
-                        discoveryEntry.discovery_payload.state_topic = true;
-                        discoveryEntry.object_id = 'keypad_lock';
-                    } else if (state.property === 'child_lock') {
-                        // deprecated: child_lock is messy, but changing is breaking
-                        discoveryEntry.discovery_payload.payload_lock = state.value_on;
-                        discoveryEntry.discovery_payload.payload_unlock = state.value_off;
-                        discoveryEntry.discovery_payload.state_locked = 'LOCK';
-                        discoveryEntry.discovery_payload.state_unlocked = 'UNLOCK';
-                        discoveryEntry.discovery_payload.state_topic = true;
-                        discoveryEntry.object_id = 'child_lock';
-                    } else {
-                        discoveryEntry.discovery_payload.state_locked = state.value_on;
-                        discoveryEntry.discovery_payload.state_unlocked = state.value_off;
-                    }
-
-                    if (state.property !== 'state') {
-                        discoveryEntry.discovery_payload.command_topic_postfix = state.property;
-                    }
-                } else if (expose.type === 'cover') {
-                    assert(!expose.endpoint, `Endpoint not supported for cover type`);
-                    const hasPosition = expose.features.find((e) => e.name === 'position');
-                    const hasTilt = expose.features.find((e) => e.name === 'tilt');
-
-                    discoveryEntry = {
-                        type: 'cover',
-                        object_id: 'cover',
-                        discovery_payload: {
-                            command_topic: true,
-                            state_topic: !hasPosition,
-                        },
-                    };
-
-                    if (!hasPosition && !hasTilt) {
-                        discoveryEntry.discovery_payload.optimistic = true;
-                    }
-
-                    if (hasPosition) {
-                        discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
-                            position_template: '{{ value_json.position }}',
-                            set_position_template: '{ "position": {{ position }} }',
-                            set_position_topic: true,
-                            position_topic: true,
-                        };
-                    }
-
-                    if (hasTilt) {
-                        discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
-                            tilt_command_topic: true,
-                            tilt_status_topic: true,
-                            tilt_status_template: '{{ value_json.tilt }}',
-                        };
-                    }
-                } else if (expose.type === 'fan') {
-                    assert(!expose.endpoint, `Endpoint not supported for fan type`);
-                    discoveryEntry = {
-                        type: 'fan',
-                        object_id: 'fan',
-                        discovery_payload: {
-                            state_topic: true,
-                            state_value_template: '{{ value_json.fan_state }}',
-                            command_topic: true,
-                            command_topic_postfix: 'fan_state',
-                        },
-                    };
-
-                    const speed = expose.features.find((e) => e.name === 'mode');
-                    if (speed) {
-                        discoveryEntry.discovery_payload.speed_state_topic = true;
-                        discoveryEntry.discovery_payload.speed_command_topic = true;
-                        discoveryEntry.discovery_payload.speed_value_template = '{{ value_json.fan_mode }}';
-                        discoveryEntry.discovery_payload.speeds = speed.values;
-                    }
-                } else if (expose.type === 'binary') {
-                    const lookup = {
-                        occupancy: {device_class: 'motion'},
-                        battery_low: {device_class: 'battery'},
-                        water_leak: {device_class: 'moisture'},
-                        vibration: {device_class: 'vibration'},
-                        contact: {device_class: 'door'},
-                        smoke: {device_class: 'smoke'},
-                        gas: {device_class: 'gas'},
-                        carbon_monoxide: {device_class: 'safety'},
-                        presence: {device_class: 'presence'},
-                    };
-
-                    discoveryEntry = {
-                        type: 'binary_sensor',
-                        object_id: expose.endpoint ? `${expose.name}_${expose.endpoint}` : `${expose.name}`,
-                        discovery_payload: {
-                            value_template: `{{ value_json.${expose.property} }}`,
-                            payload_on: expose.value_on,
-                            payload_off: expose.value_off,
-                            ...(lookup[expose.name] || {}),
-                        },
-                    };
-                } else if (expose.type === 'numeric') {
-                    const lookup = {
-                        battery: {device_class: 'battery'},
-                        temperature: {device_class: 'temperature'},
-                        humidity: {device_class: 'humidity'},
-                        illuminance_lux: {device_class: 'illuminance'},
-                        illuminance: {device_class: 'illuminance'},
-                        soil_moisture: {icon: 'mdi:water-percent'},
-                        position: {icon: 'mdi:valve'},
-                        pressure: {device_class: 'pressure'},
-                        power: {device_class: 'power'},
-                        linkquality: {icon: 'mdi:signal'},
-                        current: {device_class: 'current'},
-                        voltage: {device_class: 'voltage'},
-                        current_phase_b: {device_class: 'current'},
-                        voltage_phase_b: {device_class: 'voltage'},
-                        current_phase_c: {device_class: 'current'},
-                        voltage_phase_c: {device_class: 'voltage'},
-                        energy: {device_class: 'energy'},
-                        smoke_density: {icon: 'mdi:google-circles-communities'},
-                        gas_density: {icon: 'mdi:google-circles-communities'},
-                        pm25: {icon: 'mdi:air-filter'},
-                        pm10: {icon: 'mdi:air-filter'},
-                        voc: {icon: 'mdi:air-filter'},
-                        aqi: {icon: 'mdi:air-filter'},
-                        hcho: {icon: 'mdi:air-filter'},
-                        requested_brightness_level: {icon: 'mdi:brightness-5'},
-                        requested_brightness_percent: {icon: 'mdi:brightness-5'},
-                        eco2: {icon: 'mdi:molecule-co2'},
-                        co2: {icon: 'mdi:molecule-co2'},
-                        local_temperature: {device_class: 'temperature'},
-                        x_axis: {icon: 'mdi:axis-x-arrow'},
-                        y_axis: {icon: 'mdi:axis-y-arrow'},
-                        z_axis: {icon: 'mdi:axis-z-arrow'},
-                    };
-
-                    discoveryEntry = {
-                        type: 'sensor',
-                        object_id: expose.endpoint ? `${expose.name}_${expose.endpoint}` : `${expose.name}`,
-                        discovery_payload: {
-                            value_template: `{{ value_json.${expose.property} }}`,
-                            ...(expose.unit && {unit_of_measurement: expose.unit}),
-                            ...lookup[expose.name],
-                        },
-                    };
-                } else if (expose.type === 'enum' || expose.type === 'text' || expose.type === 'composite') {
-                    if (expose.access & ACCESS_STATE) {
-                        const lookup = {
-                            action: {icon: 'mdi:gesture-double-tap'},
-                        };
-
-                        discoveryEntry = {
-                            type: 'sensor',
-                            object_id: expose.property,
-                            discovery_payload: {
-                                value_template: `{{ value_json.${expose.property} }}`,
-                                ...lookup[expose.name],
-                            },
-                        };
-                    }
-                } else {
-                    throw new Error(`Unsupported exposes type: '${expose.type}'`);
-                }
-
-                if (discoveryEntry) {
-                    this.mapping[def.model].push(discoveryEntry);
-                }
-            }
-        }
-
-        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
-            if (definition.hasOwnProperty('homeassistant')) {
-                this.mapping[definition.model] = definition.homeassistant;
-            }
-        }
-    }
-
-    onDeviceRemoved(resolvedEntity) {
-        logger.debug(`Clearing Home Assistant discovery topic for '${resolvedEntity.name}'`);
-        delete this.discovered[resolvedEntity.device.ieeeAddr];
-        for (const config of this.getConfigs(resolvedEntity)) {
-            const topic = this.getDiscoveryTopic(config, resolvedEntity.device);
-            this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
-        }
-    }
-
-    async onPublishEntityState(data) {
-        /**
-         * In case we deal with a lightEndpoint configuration Zigbee2MQTT publishes
-         * e.g. {state_l1: ON, brightness_l1: 250} to zigbee2mqtt/mydevice.
-         * As the Home Assistant MQTT JSON light cannot be configured to use state_l1/brightness_l1
-         * as the state variables, the state topic is set to zigbee2mqtt/mydevice/l1.
-         * Here we retrieve all the attributes with the _l1 values and republish them on
-         * zigbee2mqtt/mydevice/l1.
-         */
-        if (data.entity.definition && this.mapping[data.entity.definition.model]) {
-            for (const config of this.mapping[data.entity.definition.model]) {
-                const match = /light_(.*)/.exec(config['object_id']);
-                if (match) {
-                    const endpoint = match[1];
-                    const endpointRegExp = new RegExp(`(.*)_${endpoint}`);
-                    const payload = {};
-                    for (const key of Object.keys(data.messagePayload)) {
-                        const keyMatch = endpointRegExp.exec(key);
-                        if (keyMatch) {
-                            payload[keyMatch[1]] = data.messagePayload[key];
-                        }
-                    }
-
-                    await this.mqtt.publish(
-                        `${data.entity.name}/${endpoint}`, stringify(payload), {},
-                    );
-                }
-            }
-        }
-
-        /**
-         * Publish an empty value for click and action payload, in this way Home Assistant
-         * can use Home Assistant entities in automations.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-         */
-        if (settings.get().advanced.homeassistant_legacy_triggers) {
-            const keys = ['action', 'click'].filter((k) => data.messagePayload[k]);
-            for (const key of keys) {
-                this.publishEntityState(data.entity.device.ieeeAddr, {[key]: ''});
-            }
-        }
-
-        /**
-         * Implements the MQTT device trigger (https://www.home-assistant.io/integrations/device_trigger.mqtt/)
-         * The MQTT device trigger does not support JSON parsing, so it cannot listen to zigbee2mqtt/my_device
-         * Whenever a device publish an {action: *} we discover an MQTT device trigger sensor
-         * and republish it to zigbee2mqtt/my_devic/action
-         */
-        if (data.entity.definition) {
-            const keys = ['action', 'click'].filter((k) => data.messagePayload[k]);
-            for (const key of keys) {
-                const value = data.messagePayload[key].toString();
-                await this.publishDeviceTriggerDiscover(data.entity, key, value);
-                await this.mqtt.publish(`${data.entity.name}/${key}`, value, {});
-            }
-        }
-
-        /**
-         * Publish a value for update_available (if not there yet) to prevent Home Assistant generating warnings of
-         * this value not being available.
-         */
-        const supportsOTA = data.entity.definition && data.entity.definition.hasOwnProperty('ota');
-        const mockedValues = [
-            {
-                property: 'update_available',
-                condition: supportsOTA && this.legacyApi,
-                value: false,
-            },
-            {
-                property: 'update',
-                condition: supportsOTA,
-                value: {state: 'idle'},
-            },
-            {
-                property: 'water_leak',
-                condition: data.entity.device && data.entity.definition && this.mapping[data.entity.definition.model] &&
-                    this.mapping[data.entity.definition.model].filter((c) => c.object_id === 'water_leak').length === 1,
-                value: false,
-            },
-        ];
-
-        for (const entry of mockedValues) {
-            if (entry.condition && !data.messagePayload.hasOwnProperty(entry.property)) {
-                logger.debug(`Mocking '${entry.property}' value for Home Assistant`);
-                this.publishEntityState(data.entity.device.ieeeAddr, {[entry.property]: entry.value});
-            }
-        }
-    }
-
-    onDeviceRenamed(device, homeAssisantRename) {
-        logger.debug(`Refreshing Home Assistant discovery topic for '${device.ieeeAddr}'`);
-        const resolvedEntity = this.zigbee.resolveEntity(device);
-
-        // Clear before rename so Home Assistant uses new friendly_name
-        // https://github.com/Koenkk/zigbee2mqtt/issues/4096#issuecomment-674044916
-        if (homeAssisantRename) {
-            for (const config of this.getConfigs(resolvedEntity)) {
-                const topic = this.getDiscoveryTopic(config, device);
-                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
-            }
-        }
-
-        this.discover(resolvedEntity, true);
-
-        if (this.discoveredTriggers[device.ieeeAddr]) {
-            for (const config of this.discoveredTriggers[device.ieeeAddr]) {
-                const key = config.substring(0, config.indexOf('_'));
-                const value = config.substring(config.indexOf('_') + 1);
-                this.publishDeviceTriggerDiscover(resolvedEntity, key, value, true);
-            }
-        }
-    }
-
-    async onMQTTConnected() {
-        this.mqtt.subscribe(this.statusTopic);
-        this.mqtt.subscribe(defaultStatusTopic);
-        this.mqtt.subscribe(`${this.discoveryTopic}/#`);
-
-        // MQTT discovery of all paired devices on startup.
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            this.discover(resolvedEntity, true);
-        }
-    }
-
-    getConfigs(resolvedEntity) {
-        if (!resolvedEntity || !resolvedEntity.definition || !this.mapping[resolvedEntity.definition.model]) return [];
-
-        let configs = this.mapping[resolvedEntity.definition.model].slice();
-        if (resolvedEntity.definition.hasOwnProperty('ota')) {
-            const updateStateSensor = {
-                type: 'sensor',
-                object_id: 'update_state',
-                discovery_payload: {
-                    icon: 'mdi:update',
-                    value_template: `{{ value_json['update']['state'] }}`,
-                },
-            };
-
-            configs.push(updateStateSensor);
-            if (this.legacyApi) {
-                const updateAvailableSensor = {
-                    type: 'binary_sensor',
-                    object_id: 'update_available',
-                    discovery_payload: {
-                        payload_on: true,
-                        payload_off: false,
-                        value_template: '{{ value_json.update_available}}',
-                    },
-                };
-                configs.push(updateAvailableSensor);
-            }
-        }
-
-        if (resolvedEntity.settings.hasOwnProperty('legacy') && !resolvedEntity.settings.legacy) {
-            configs = configs.filter((c) => c !== sensorClick);
-        }
-
-        if (!settings.get().advanced.homeassistant_legacy_triggers) {
-            configs = configs.filter((c) => c.object_id !== 'action' && c.object_id !== 'click');
-        }
-
-        // deep clone of the config objects
-        configs = JSON.parse(JSON.stringify(configs));
-
-        if (resolvedEntity.settings.homeassistant) {
-            const s = resolvedEntity.settings.homeassistant;
-            configs = configs.filter((config) => !s.hasOwnProperty(config.object_id) || s[config.object_id] != null);
-            configs.forEach((config) => {
-                const configOverride = s[config.object_id];
-                if (configOverride) {
-                    config.object_id = configOverride.object_id || config.object_id;
-                    config.type = configOverride.type || config.type;
-                }
-            });
-        }
-
-        return configs;
-    }
-
-    discover(resolvedEntity, force=false) {
-        // Check if already discoverd and check if there are configs.
-        const {device, definition} = resolvedEntity;
-        const discover = force || !this.discovered[device.ieeeAddr];
-        if (!discover || !device || !definition || !this.mapping[definition.model] || device.interviewing ||
-            (resolvedEntity.settings.hasOwnProperty('homeassistant') && !resolvedEntity.settings.homeassistant)) {
-            return;
-        }
-
-        const friendlyName = resolvedEntity.settings.friendlyName;
-        this.getConfigs(resolvedEntity).forEach((config) => {
-            const payload = {...config.discovery_payload};
-            let stateTopic = `${settings.get().mqtt.base_topic}/${friendlyName}`;
-            if (payload.state_topic_postfix) {
-                stateTopic += `/${payload.state_topic_postfix}`;
-                delete payload.state_topic_postfix;
-            }
-
-            if (!payload.hasOwnProperty('state_topic') || payload.state_topic) {
-                payload.state_topic = stateTopic;
-            } else {
-                /* istanbul ignore else */
-                if (payload.hasOwnProperty('state_topic')) {
-                    delete payload.state_topic;
-                }
-            }
-
-            if (payload.position_topic) {
-                payload.position_topic = stateTopic;
-            }
-
-            if (payload.tilt_status_topic) {
-                payload.tilt_status_topic = stateTopic;
-            }
-
-            payload.json_attributes_topic = stateTopic;
-
-            // Set (unique) name, separate by space if friendlyName contains space.
-            const nameSeparator = friendlyName.includes('_') ? '_' : ' ';
-            payload.name = friendlyName;
-            if (config.object_id.startsWith(config.type) && config.object_id.includes('_')) {
-                payload.name += `${nameSeparator}${config.object_id.split(/_(.+)/)[1]}`;
-            } else if (!config.object_id.startsWith(config.type)) {
-                payload.name += `${nameSeparator}${config.object_id.replace(/_/g, nameSeparator)}`;
-            }
-
-            // Set unique_id
-            payload.unique_id = `${resolvedEntity.settings.ID}_${config.object_id}_${settings.get().mqtt.base_topic}`;
-
-            // Attributes for device registry
-            payload.device = this.getDevicePayload(resolvedEntity);
-
-            // Availability payload
-            payload.availability = [{topic: `${settings.get().mqtt.base_topic}/bridge/state`}];
-            if (settings.get().advanced.availability_timeout) {
-                payload.availability.push({topic: `${settings.get().mqtt.base_topic}/${friendlyName}/availability`});
-            }
-
-            if (payload.command_topic) {
-                payload.command_topic = `${settings.get().mqtt.base_topic}/${friendlyName}/`;
-
-                if (payload.command_topic_prefix) {
-                    payload.command_topic += `${payload.command_topic_prefix}/`;
-                    delete payload.command_topic_prefix;
-                }
-
-                payload.command_topic += 'set';
-
-                if (payload.command_topic_postfix) {
-                    payload.command_topic += `/${payload.command_topic_postfix}`;
-                    delete payload.command_topic_postfix;
-                }
-            }
-
-            if (payload.set_position_topic && payload.command_topic) {
-                payload.set_position_topic = payload.command_topic;
-            }
-
-            if (payload.tilt_command_topic && payload.command_topic) {
-                // Home Assistant does not support templates to set tilt (as of 2019-08-17),
-                // so we (have to) use a subtopic.
-                payload.tilt_command_topic = payload.command_topic + '/tilt';
-            }
-
-            if (payload.mode_state_topic) {
-                payload.mode_state_topic = stateTopic;
-            }
-
-            if (payload.mode_command_topic) {
-                payload.mode_command_topic = `${stateTopic}/set/system_mode`;
-            }
-
-            if (payload.hold_command_topic) {
-                payload.hold_command_topic = `${stateTopic}/set/preset`;
-            }
-
-            if (payload.hold_state_topic) {
-                payload.hold_state_topic = stateTopic;
-            }
-
-            if (payload.away_mode_state_topic) {
-                payload.away_mode_state_topic = stateTopic;
-            }
-
-            if (payload.away_mode_command_topic) {
-                payload.away_mode_command_topic = `${stateTopic}/set/away_mode`;
-            }
-
-            if (payload.current_temperature_topic) {
-                payload.current_temperature_topic = stateTopic;
-            }
-
-            if (payload.temperature_state_topic) {
-                payload.temperature_state_topic = stateTopic;
-            }
-
-            if (payload.temperature_low_state_topic) {
-                payload.temperature_low_state_topic = stateTopic;
-            }
-
-            if (payload.temperature_high_state_topic) {
-                payload.temperature_high_state_topic = stateTopic;
-            }
-
-            if (payload.speed_state_topic) {
-                payload.speed_state_topic = stateTopic;
-            }
-
-            if (payload.temperature_command_topic) {
-                payload.temperature_command_topic = `${stateTopic}/set/${payload.temperature_command_topic}`;
-            }
-
-            if (payload.temperature_low_command_topic) {
-                payload.temperature_low_command_topic = `${stateTopic}/set/${payload.temperature_low_command_topic}`;
-            }
-
-            if (payload.temperature_high_command_topic) {
-                payload.temperature_high_command_topic = `${stateTopic}/set/${payload.temperature_high_command_topic}`;
-            }
-
-            if (payload.fan_mode_state_topic) {
-                payload.fan_mode_state_topic = stateTopic;
-            }
-
-            if (payload.fan_mode_command_topic) {
-                payload.fan_mode_command_topic = `${stateTopic}/set/fan_mode`;
-            }
-
-            if (payload.speed_command_topic) {
-                payload.speed_command_topic = `${stateTopic}/set/fan_mode`;
-            }
-
-            if (payload.action_topic) {
-                payload.action_topic = stateTopic;
-            }
-
-            // Override configuration with user settings.
-            if (resolvedEntity.settings.hasOwnProperty('homeassistant')) {
-                const add = (obj) => {
-                    Object.keys(obj).forEach((key) => {
-                        if (['type', 'object_id'].includes(key)) {
-                            return;
-                        } else if (['number', 'string', 'boolean'].includes(typeof obj[key])) {
-                            payload[key] = obj[key];
-                        } else if (obj[key] === null) {
-                            delete payload[key];
-                        } else if (key === 'device' && typeof obj[key] === 'object') {
-                            Object.keys(obj['device']).forEach((key) => {
-                                payload['device'][key] = obj['device'][key];
-                            });
-                        }
-                    });
-                };
-
-                add(resolvedEntity.settings.homeassistant);
-
-                if (resolvedEntity.settings.homeassistant.hasOwnProperty(config.object_id)) {
-                    add(resolvedEntity.settings.homeassistant[config.object_id]);
-                }
-            }
-
-            const topic = this.getDiscoveryTopic(config, device);
-            this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic, false, false);
-        });
-
-        this.discovered[device.ieeeAddr] = true;
-    }
-
-    onMQTTMessage(topic, message) {
-        const discoveryRegex = new RegExp(`${this.discoveryTopic}/(.*)/(.*)/(.*)/config`);
-        const discoveryMatch = topic.match(discoveryRegex);
-        const isDeviceAutomation = discoveryMatch && discoveryMatch[1] === 'device_automation';
-        if (discoveryMatch) {
-            // Clear outdated discovery configs and remember already discoverd device_automations
-            try {
-                message = JSON.parse(message);
-                const baseTopic = settings.get().mqtt.base_topic + '/';
-                if (isDeviceAutomation && (!message.topic || !message.topic.startsWith(baseTopic))) {
-                    return;
-                }
-
-                if (!isDeviceAutomation &&
-                    (!message.availability || !message.availability[0].topic.startsWith(baseTopic))) {
-                    return;
-                }
-            } catch (e) {
-                return;
-            }
-
-            const ieeeAddr = discoveryMatch[2];
-            const resolvedEntity = this.zigbee.resolveEntity(ieeeAddr);
-            let clear = !resolvedEntity || !resolvedEntity.definition;
-
-            // Only save when topic matches otherwise config is not updated when renamed by editing configuration.yaml
-            if (resolvedEntity) {
-                const key = `${discoveryMatch[3].substring(0, discoveryMatch[3].indexOf('_'))}`;
-                const triggerTopic = `${settings.get().mqtt.base_topic}/${resolvedEntity.name}/${key}`;
-                if (isDeviceAutomation && message.topic === triggerTopic) {
-                    if (!this.discoveredTriggers[ieeeAddr]) {
-                        this.discoveredTriggers[ieeeAddr] = new Set();
-                    }
-                    this.discoveredTriggers[ieeeAddr].add(discoveryMatch[3]);
-                }
-            }
-
-            if (!clear && !isDeviceAutomation) {
-                const type = discoveryMatch[1];
-                const objectID = discoveryMatch[3];
-                clear = !this.getConfigs(resolvedEntity).find((c) => c.type === type && c.object_id === objectID);
-            }
-            // Device was flagged to be excluded from homeassistant discovery
-            clear = clear || (resolvedEntity.settings.hasOwnProperty('homeassistant') &&
-                                !resolvedEntity.settings.homeassistant);
-
-            if (clear) {
-                logger.debug(`Clearing Home Assistant config '${topic}'`);
-                topic = topic.substring(this.discoveryTopic.length + 1);
-                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
-            }
-        } else if ((topic === this.statusTopic || topic === defaultStatusTopic) && message.toLowerCase() === 'online') {
-            const timer = setTimeout(async () => {
-                // Publish all device states.
-                for (const device of this.zigbee.getClients()) {
-                    if (this.state.exists(device.ieeeAddr)) {
-                        this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
-                    }
-                }
-
-                clearTimeout(timer);
-            }, 30000);
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        if (resolvedEntity && type !== 'deviceLeave' && this.mqtt.isConnected()) {
-            this.discover(resolvedEntity);
-        }
-    }
-
-    getDevicePayload(resolvedEntity) {
-        return {
-            identifiers: [`zigbee2mqtt_${resolvedEntity.settings.ID}`],
-            name: resolvedEntity.settings.friendlyName,
-            sw_version: `Zigbee2MQTT ${zigbee2mqttVersion}`,
-            model: `${resolvedEntity.definition.description} (${resolvedEntity.definition.model})`,
-            manufacturer: resolvedEntity.definition.vendor,
-        };
-    }
-
-    adjustMessagePayloadBeforePublish(resolvedEntity, messagePayload) {
-        // Set missing values of state to 'null': https://github.com/Koenkk/zigbee2mqtt/issues/6987
-        if (!resolvedEntity || !resolvedEntity.definition) return null;
-
-        const add = (expose) => {
-            if (!messagePayload.hasOwnProperty(expose.property) && expose.access & ACCESS_STATE) {
-                messagePayload[expose.property] = null;
-            }
-        };
-
-        for (const expose of resolvedEntity.definition.exposes) {
-            if (expose.hasOwnProperty('features')) {
-                for (const feature of expose.features) {
-                    if (feature.name === 'state') {
-                        add(feature);
-                    }
-                }
-            } else {
-                add(expose);
-            }
-        }
-    }
-
-    getDiscoveryTopic(config, device) {
-        return `${config.type}/${device.ieeeAddr}/${config.object_id}/config`;
-    }
-
-    async publishDeviceTriggerDiscover(entity, key, value, force=false) {
-        const haConfig = entity.settings.homeassistant;
-        if (entity.settings.hasOwnProperty('homeassistant') && (haConfig == null ||
-                (haConfig.hasOwnProperty('device_automation') && haConfig.device_automation == null))) {
-            return;
-        }
-
-        const device = entity.device;
-        if (!this.discoveredTriggers[device.ieeeAddr]) {
-            this.discoveredTriggers[device.ieeeAddr] = new Set();
-        }
-
-        const discoveredKey = `${key}_${value}`;
-        if (this.discoveredTriggers[device.ieeeAddr].has(discoveredKey) && !force) {
-            return;
-        }
-
-        const config = {
-            type: 'device_automation',
-            object_id: `${key}_${value}`,
-            discovery_payload: {
-                automation_type: 'trigger',
-                type: key,
-            },
-        };
-
-        const topic = this.getDiscoveryTopic(config, device);
-        const payload = {
-            ...config.discovery_payload,
-            subtype: value,
-            payload: value,
-            topic: `${settings.get().mqtt.base_topic}/${entity.name}/${key}`,
-            device: this.getDevicePayload(entity),
-        };
-
-        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic, false, false);
-        this.discoveredTriggers[device.ieeeAddr].add(discoveredKey);
-    }
-
-    // Only for homeassistant.test.js
-    _getMapping() {
-        return this.mapping;
-    }
-
-    _clearDiscoveredTrigger() {
-        this.discoveredTriggers = new Set();
-    }
-}
-
-module.exports = HomeAssistant;

lib/extension/legacy/availability.js

@@ -1,8 +1,8 @@
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-const utils = require('../util/utils');
+const logger = require('../../util/logger');
+const settings = require('../../util/settings');
+const utils = require('../../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const Extension = require('./extension');
+const Extension = require('../extension');
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/(.*)/availability`);
 
 // Pingable end devices, some end devices should be pinged
@@ -25,7 +25,7 @@ function timeoutLag(timeout, ratio) {
 /**
  * This extensions pings devices to check if they are online.
  */
-class Availability extends Extension {
+class AvailabilityLegacy extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -58,7 +58,7 @@ class Availability extends Extension {
     inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings.friendlyName;
+        const name = deviceSettings && deviceSettings.friendlyName;
 
         // Passlist is not empty and device is in it, enable availability
         if (this.passlist.length > 0) {
@@ -83,7 +83,7 @@ class Availability extends Extension {
     }
 
     onMQTTConnected() {
-        for (const device of this.zigbee.getClients()) {
+        for (const device of this.zigbee.getClientsLegacy()) {
             // Mark all devices as online on start
             const ieeeAddr = device.ieeeAddr;
             this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
@@ -94,7 +94,7 @@ class Availability extends Extension {
                 } else {
                     this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
-                    }, utils.secondsToMilliseconds(300));
+                    }, utils.seconds(300));
                 }
             }
         }
@@ -103,14 +103,15 @@ class Availability extends Extension {
     async onMQTTMessage(topic, message) {
         // Clear topics for non-existing devices
         const match = topic.match(topicRegex);
-        if (match && (!this.zigbee.resolveEntity(match[1]) || this.zigbee.resolveEntity(match[1]).name !== match[1])) {
+        if (match && (!this.zigbee.resolveEntityLegacy(match[1]) ||
+            this.zigbee.resolveEntityLegacy(match[1]).name !== match[1])) {
             this.mqtt.publish(`${match[1]}/availability`, null, {retain: true, qos: 0});
         }
     }
 
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device.ieeeAddr);
         if (!resolvedEntity) {
             logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
             return;
@@ -130,7 +131,7 @@ class Availability extends Extension {
     }
 
     async handleIntervalNotPingable(device) {
-        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device.ieeeAddr);
         if (!resolvedEntity || !device.lastSeen) {
             return;
         }
@@ -148,7 +149,7 @@ class Availability extends Extension {
         clearTimeout(this.timers[device.ieeeAddr]);
         this.timers[device.ieeeAddr] = setTimeout(async () => {
             await this.handleIntervalPingable(device);
-        }, utils.secondsToMilliseconds(timeout));
+        }, utils.seconds(timeout));
     }
 
     async stop() {
@@ -157,11 +158,11 @@ class Availability extends Extension {
             clearTimeout(timer);
         }
 
-        this.zigbee.getClients().forEach((device) => this.publishAvailability(device, false));
+        this.zigbee.getClientsLegacy().forEach((device) => this.publishAvailability(device, false));
     }
 
     async onReconnect(device) {
-        const resolvedEntity = this.zigbee.resolveEntity(device);
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device);
         if (resolvedEntity && resolvedEntity.definition) {
             try {
                 for (const key of ['state', 'brightness', 'color', 'color_temp']) {
@@ -182,9 +183,8 @@ class Availability extends Extension {
             this.onReconnect(device);
         }
 
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
-        const topic = `${name}/availability`;
+        const resolvedEntity = this.zigbee.resolveEntityLegacy(device);
+        const topic = `${resolvedEntity.name}/availability`;
         const payload = available ? 'online' : 'offline';
         if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;
@@ -224,4 +224,4 @@ class Availability extends Extension {
     }
 }
 
-module.exports = Availability;
+module.exports = AvailabilityLegacy;

lib/extension/legacy/bridgeLegacy.js

@@ -95,7 +95,7 @@ class BridgeLegacy extends Extension {
     }
 
     async permitJoin(topic, message) {
-        await this.zigbee.permitJoin(message.toLowerCase() === 'true');
+        await this.zigbee.permitJoinLegacy(message.toLowerCase() === 'true');
         this.publish();
     }
 
@@ -144,7 +144,7 @@ class BridgeLegacy extends Extension {
 
     async devices(topic, message) {
         const coordinator = await this.zigbee.getCoordinatorVersion();
-        const devices = this.zigbee.getDevices().map((device) => {
+        const devices = this.zigbee.getDevicesLegacy().map((device) => {
             const payload = {
                 ieeeAddr: device.ieeeAddr,
                 type: device.type,
@@ -230,7 +230,7 @@ class BridgeLegacy extends Extension {
             const isGroup = settings.getGroup(from) !== null;
             settings.changeFriendlyName(from, to);
             logger.info(`Successfully renamed - ${from} to ${to} `);
-            const entity = this.zigbee.resolveEntity(to);
+            const entity = this.zigbee.resolveEntityLegacy(to);
             const eventData = isGroup ? {group: entity.group} : {device: entity.device, from, to};
             eventData.homeAssisantRename = false;
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
@@ -268,14 +268,14 @@ class BridgeLegacy extends Extension {
         }
 
         const group = settings.addGroup(name, id);
-        this.zigbee.createGroup(group.ID);
+        this.zigbee.createGroupLegacy(group.ID);
         this.mqtt.publish('bridge/log', stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
     removeGroup(topic, message) {
         const name = message;
-        const entity = this.zigbee.resolveEntity(message);
+        const entity = this.zigbee.resolveEntityLegacy(message);
         assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
 
         if (topic.includes('force')) {
@@ -302,7 +302,7 @@ class BridgeLegacy extends Extension {
     }
 
     async removeForceRemoveOrBan(action, message) {
-        const entity = this.zigbee.resolveEntity(message.trim());
+        const entity = this.zigbee.resolveEntityLegacy(message.trim());
         const lookup = {
             ban: ['banned', 'Banning', 'ban'],
             force_remove: ['force_removed', 'Force removing', 'force remove'],
@@ -373,7 +373,7 @@ class BridgeLegacy extends Extension {
     }
 
     async publish() {
-        const info = await utils.getZigbee2mqttVersion();
+        const info = await utils.getZigbee2MQTTVersion();
         const coordinator = await this.zigbee.getCoordinatorVersion();
         const topic = `bridge/config`;
         const payload = {

lib/extension/legacy/deviceGroupMembership.js

@@ -12,7 +12,7 @@ class DeviceGroupMembership extends Extension {
             return null;
         }
 
-        const entity = this.zigbee.resolveEntity(match[1]);
+        const entity = this.zigbee.resolveEntityLegacy(match[1]);
         if (!entity || entity.type !== 'device') {
             logger.error(`Device '${match[1]}' does not exist`);
             return;

lib/extension/legacy/report.js

@@ -173,10 +173,10 @@ class Report extends Extension {
     }
 
     async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+        this.coordinatorEndpoint = this.zigbee.getDevicesByTypeLegacy('Coordinator')[0].getEndpoint(1);
 
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
+        for (const device of this.zigbee.getClientsLegacy()) {
+            const resolvedEntity = this.zigbee.resolveEntityLegacy(device);
             if (this.shouldSetupReporting(resolvedEntity, null)) {
                 await this.setupReporting(resolvedEntity);
             }

lib/extension/legacy/softReset.js

@@ -12,11 +12,11 @@ class SoftReset extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;
-        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
+        this.timeout = utils.seconds(settings.get().advanced.soft_reset_timeout);
     }
 
     onZigbeeStarted() {
-        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+        logger.debug(`Soft reset timeout set to ${this.timeout / 1000} seconds`);
         this.resetTimer();
     }
 

lib/extension/networkMap.ts

@@ -1,27 +1,37 @@
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const logger = require('../util/logger');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
+import * as settings from '../util/settings';
+import * as utils from '../util/utils';
+import logger from '../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+
+interface Link {
+    source: {ieeeAddr: string, networkAddress: number}, target: {ieeeAddr: string, networkAddress: number},
+    linkquality: number, depth: number, routes: RoutingTableEntry[],
+    sourceIeeeAddr: string, targetIeeeAddr: string, sourceNwkAddr: number, lqi: number, relationship: number,
+}
+
+interface Topology {
+    nodes: {
+        ieeeAddr: string, friendlyName: string, type: string, networkAddress: number, manufacturerName: string,
+        modelID: string, failed: string[], lastSeen: number,
+        definition: {model: string, vendor: string, supports: string, description: string}}[],
+    links: Link[],
+}
 
 /**
  * This extension creates a network map
  */
-class NetworkMap extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
+class NetworkMap extends ExtensionTS {
+    private legacyApi = settings.get().advanced.legacy_api;
+    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+    private legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+    private topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
+    private supportedFormats: {[s: string]: (topology: Topology) => KeyValue | string};
 
-        this.legacyApi = settings.get().advanced.legacy_api;
-        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
-
-        // Bind
-        this.raw = this.raw.bind(this);
-        this.graphviz = this.graphviz.bind(this);
-        this.plantuml = this.plantuml.bind(this);
-
-        // Set supported formats
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
         this.supportedFormats = {
             'raw': this.raw,
             'graphviz': this.graphviz,
@@ -29,22 +39,22 @@ class NetworkMap extends Extension {
         };
     }
 
-    async onMQTTMessage(topic, message) {
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
         /* istanbul ignore else */
         if (this.legacyApi) {
-            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
-                this.supportedFormats.hasOwnProperty(message)) {
-                const includeRoutes = topic === this.legacyTopicRoutes;
+            if ((data.topic === this.legacyTopic || data.topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(data.message)) {
+                const includeRoutes = data.topic === this.legacyTopicRoutes;
                 const topology = await this.networkScan(includeRoutes);
-                let converted = this.supportedFormats[message](topology);
-                converted = message === 'raw' ? stringify(converted) : converted;
-                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+                let converted = this.supportedFormats[data.message](topology);
+                converted = data.message === 'raw' ? stringify(converted) : converted;
+                this.mqtt.publish(`bridge/networkmap/${data.message}`, converted as string, {});
             }
         }
 
-        if (topic === this.topic) {
+        if (data.topic === this.topic) {
+            const message = utils.parseJSON(data.message, data.message);
             try {
-                message = utils.parseJSON(message, message);
                 const type = typeof message === 'object' ? message.type : message;
                 if (!this.supportedFormats.hasOwnProperty(type)) {
                     throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
@@ -66,11 +76,11 @@ class NetworkMap extends Extension {
         }
     }
 
-    raw(topology) {
+    @bind raw(topology: Topology): KeyValue {
         return topology;
     }
 
-    graphviz(topology) {
+    @bind graphviz(topology: Topology): string {
         const colors = settings.get().map_options.graphviz.colors;
 
         let text = 'digraph G {\nnode[shape=record];\n';
@@ -84,7 +94,7 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
@@ -102,7 +112,7 @@ class NetworkMap extends Extension {
             let lastSeen = 'unknown';
             const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'relative');
+                lastSeen = utils.formatDate(date, 'relative') as string;
             }
 
             labels.push(lastSeen);
@@ -132,7 +142,7 @@ class NetworkMap extends Extension {
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
-                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const textRoutes = e.routes.map((r) => utils.toNetworkAddressHex(r.destinationAddress));
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
                 text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
@@ -145,7 +155,7 @@ class NetworkMap extends Extension {
         return text.replace(/\0/g, '');
     }
 
-    plantuml(topology) {
+    @bind plantuml(topology: Topology): string {
         const text = [];
 
         text.push(`' paste into: https://www.planttext.com/`);
@@ -160,14 +170,14 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             text.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
             if (node.type !== 'Coordinator') {
                 text.push(`---`);
-                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                const definition = this.zigbee.resolveEntityLegacy(node.ieeeAddr).definition;
                 if (definition) {
                     text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
@@ -180,7 +190,7 @@ class NetworkMap extends Extension {
             let lastSeen = 'unknown';
             const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'relative');
+                lastSeen = utils.formatDate(date, 'relative') as string;
             }
             text.push(`---`);
             text.push(lastSeen);
@@ -203,57 +213,69 @@ class NetworkMap extends Extension {
         return text.join(`\n`);
     }
 
-    async networkScan(includeRoutes) {
+    async networkScan(includeRoutes: boolean): Promise<Topology> {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
         const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
-        const lqis = new Map();
-        const routingTables = new Map();
-        const failed = new Map();
+        const lqis: Map<Device, LQI> = new Map();
+        const routingTables: Map<Device, RoutingTable> = new Map();
+        const failed: Map<Device, string[]> = new Map();
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
             await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
-            const resolvedEntity = this.zigbee.resolveEntity(device);
+
+            const doRequest = async <T>(request: () => Promise<T>, firstAttempt = true): Promise<T> => {
+                try {
+                    return await request();
+                } catch (error) {
+                    if (!firstAttempt) {
+                        throw error;
+                    } else {
+                        // Network is possibly congested, sleep 5 seconds to let the network settle.
+                        await utils.sleep(5);
+                        return doRequest(request, false);
+                    }
+                }
+            };
+
             try {
-                const result = await device.lqi();
+                const result = await doRequest<LQI>(async () => device.lqi());
                 lqis.set(device, result);
-                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
+                logger.debug(`LQI succeeded for '${device.name}'`);
             } catch (error) {
                 failed.get(device).push('lqi');
-                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
+                logger.error(`Failed to execute LQI for '${device.name}'`);
             }
 
             if (includeRoutes) {
                 try {
-                    const result = await device.routingTable();
+                    const result = await doRequest(async () => device.routingTable());
                     routingTables.set(device, result);
-                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
+                    logger.debug(`Routing table succeeded for '${device.name}'`);
                 } catch (error) {
                     failed.get(device).push('routingTable');
-                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
+                    logger.error(`Failed to execute routing table for '${device.name}'`);
                 }
             }
         }
 
         logger.info(`Network scan finished`);
 
-        const networkMap = {nodes: [], links: []};
+        const topology: Topology = {nodes: [], links: []};
         // Add nodes
         for (const device of devices) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            const definition = resolvedEntity.definition ? {
-                model: resolvedEntity.definition.model,
-                vendor: resolvedEntity.definition.vendor,
-                description: resolvedEntity.definition.description,
-                supports: Array.from(new Set((resolvedEntity.definition.exposes).map((e) => {
-                    return e.hasOwnProperty('name') ?
-                        e.name :
+            const definition = device.definition ? {
+                model: device.definition.model,
+                vendor: device.definition.vendor,
+                description: device.definition.description,
+                supports: Array.from(new Set((device.definition.exposes).map((e) => {
+                    return e.hasOwnProperty('name') ? e.name :
                         `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
                 }))).join(', '),
             } : null;
 
-            networkMap.nodes.push({
-                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
+            topology.nodes.push({
+                ieeeAddr: device.ieeeAddr, friendlyName: device.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
                 definition,
@@ -270,12 +292,12 @@ class NetworkMap extends Extension {
 
                 // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
                 // Determine the correct ieeeAddr based on the networkAddress.
-                const neighborDevice = this.zigbee.getDeviceByNetworkAddress(neighbor.networkAddress);
+                const neighborDevice = this.zigbee.deviceByNetworkAddress(neighbor.networkAddress);
                 if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
                     neighbor.ieeeAddr = neighborDevice.ieeeAddr;
                 }
 
-                const link = {
+                const link: Link = {
                     source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
                     target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},
                     linkquality: neighbor.linkquality, depth: neighbor.depth, routes: [],
@@ -291,11 +313,11 @@ class NetworkMap extends Extension {
                         .filter((t) => t.status === 'ACTIVE' && t.nextHop === neighbor.networkAddress);
                 }
 
-                networkMap.links.push(link);
+                topology.links.push(link);
             }
         });
 
-        return networkMap;
+        return topology;
     }
 }
 

lib/extension/onEvent.js

@@ -1,38 +0,0 @@
-const Extension = require('./extension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-
-/**
- * This extension calls the zigbee-herdsman-converters onEvent.
- */
-class OnEvent extends Extension {
-    async onZigbeeStarted() {
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            this.callOnEvent(resolvedEntity, 'start', {});
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        if (resolvedEntity && resolvedEntity.type === 'device') {
-            this.callOnEvent(resolvedEntity, type, data);
-        }
-    }
-
-    async stop() {
-        super.stop();
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            this.callOnEvent(resolvedEntity, 'stop', {});
-        }
-    }
-
-    callOnEvent(resolvedEntity, type, data) {
-        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
-
-        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
-            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
-        }
-    }
-}
-
-module.exports = OnEvent;

lib/extension/onEvent.ts

@@ -0,0 +1,45 @@
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import ExtensionTS from './extensionts';
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends ExtensionTS {
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.getClients()) {
+            await this.callOnEvent(device, 'start', {});
+        }
+
+        this.eventBus.onDeviceMessage(this, (data) => this.callOnEvent(data.device, 'message', this.convertData(data)));
+        this.eventBus.onDeviceJoined(this,
+            (data) => this.callOnEvent(data.device, 'deviceJoined', this.convertData(data)));
+        this.eventBus.onDeviceInterview(this,
+            (data) => this.callOnEvent(data.device, 'deviceInterview', this.convertData(data)));
+        this.eventBus.onDeviceAnnounce(this,
+            (data) => this.callOnEvent(data.device, 'deviceAnnounce', this.convertData(data)));
+        this.eventBus.onDeviceNetworkAddressChanged(this,
+            (data) => this.callOnEvent(data.device, 'deviceNetworkAddressChanged', this.convertData(data)));
+    }
+
+    private convertData(data: KeyValue): KeyValue {
+        return {...data, device: data.device.zhDevice};
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        for (const device of this.zigbee.getClients()) {
+            await this.callOnEvent(device, 'stop', {});
+        }
+    }
+
+    private async callOnEvent(device: Device, type: string, data: KeyValue): Promise<void> {
+        zhc.onEvent(type, data, device.zhDevice, device.settings);
+
+        if (device.definition?.onEvent) {
+            await device.definition.onEvent(type, data, device.zhDevice, device.settings);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.ts

@@ -1,27 +1,35 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const stringify = require('json-stable-stringify-without-jsonify');
-const utils = require('../util/utils');
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import * as utils from '../util/utils';
+// @ts-ignore
+import tradfriOTA from 'zigbee-herdsman-converters/lib/ota/tradfri';
+import ExtensionTS from './extensionts';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+
+type UpdateState = 'updating' | 'idle' | 'available';
+interface UpdatePayload {
+    // eslint-disable-next-line camelcase
+    update: {progress?: number, remaining?: number, state: UpdateState}, update_available?: boolean}
+
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
-const tradfriOTA = require('zigbee-herdsman-converters/lib/ota/tradfri');
 const topicRegex =
     new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
 
-const Extension = require('./extension');
-
-class OTAUpdate extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.inProgress = new Set();
-        this.lastChecked = {};
-        this.legacyApi = settings.get().advanced.legacy_api;
+class OTAUpdate extends ExtensionTS {
+    private inProgress = new Set();
+    private lastChecked: {[s: string]: number} = {};
+    private legacyApi = settings.get().advanced.legacy_api;
 
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+        this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
         if (settings.get().advanced.ikea_ota_use_test_url) {
             tradfriOTA.useTestURL();
         }
-    }
 
-    onMQTTConnected() {
         for (const device of this.zigbee.getClients()) {
             // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
             // remove them.
@@ -29,16 +37,17 @@ class OTAUpdate extends Extension {
         }
     }
 
-    removeProgressAndRemainingFromState(device) {
+    private removeProgressAndRemainingFromState(device: Device): void {
         this.state.removeKey(device.ieeeAddr, ['update', 'progress']);
         this.state.removeKey(device.ieeeAddr, ['update', 'remaining']);
     }
 
-    async onZigbeeEvent(type, data, resolvedEntity) {
+    @bind private async onZigbeeEvent(data: EventDeviceMessage): Promise<void> {
         if (settings.get().ota.disable_automatic_update_check) return;
-        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition) return;
+        logger.debug(`Device '${data.device.name}' requested OTA`);
 
-        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
+        const supportsOTA = data.device.definition.hasOwnProperty('ota');
         if (supportsOTA) {
             // When a device does a next image request, it will usually do it a few times after each other
             // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
@@ -49,17 +58,18 @@ class OTAUpdate extends Extension {
             if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
-            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
+            const available = await data.device.definition.ota.isUpdateAvailable(
+                data.device.zhDevice, logger, data.data);
             const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
             this.publishEntityState(data.device.ieeeAddr, payload);
 
             if (available) {
-                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
+                const message = `Update available for '${data.device.name}'`;
                 logger.info(message);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    const meta = {status: 'available', device: data.device.name};
                     this.mqtt.publish(
                         'bridge/log',
                         stringify({type: `ota_update`, message, meta}),
@@ -78,15 +88,16 @@ class OTAUpdate extends Extension {
         }
     }
 
-    async readSoftwareBuildIDAndDateCode(device, update) {
+    private async readSoftwareBuildIDAndDateCode(device: Device, update: boolean):
+        Promise<{softwareBuildID: string, dateCode: string}> {
         try {
             const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
             const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
 
             if (update) {
-                device.softwareBuildID = result.swBuildId;
-                device.dateCode = result.dateCode;
-                device.save();
+                device.zhDevice.softwareBuildID = result.swBuildId;
+                device.zhDevice.dateCode = result.dateCode;
+                device.zhDevice.save();
             }
 
             return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
@@ -95,8 +106,8 @@ class OTAUpdate extends Extension {
         }
     }
 
-    getEntityPublishPayload(state, progress=null, remaining=null) {
-        const payload = {update: {state}};
+    private getEntityPublishPayload(state: UpdateState, progress: number=null, remaining: number=null): UpdatePayload {
+        const payload: UpdatePayload = {update: {state}};
         if (progress !== null) payload.update.progress = progress;
         if (remaining !== null) payload.update.remaining = Math.round(remaining);
 
@@ -108,43 +119,44 @@ class OTAUpdate extends Extension {
         return payload;
     }
 
-    async onMQTTMessage(topic, message) {
-        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        if ((!this.legacyApi || !data.topic.match(legacyTopicRegex)) && !data.topic.match(topicRegex)) {
             return null;
         }
 
-        message = utils.parseJSON(message, message);
-        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
-        const resolvedEntity = this.zigbee.resolveEntity(ID);
-        const type = topic.substring(topic.lastIndexOf('/') + 1);
-        const responseData = {id: ID};
+        const message = utils.parseJSON(data.message, data.message);
+        const ID = (typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message) as string;
+        const device = this.zigbee.resolveEntity(ID);
+        const type = data.topic.substring(data.topic.lastIndexOf('/') + 1);
+        const responseData: {id: string, updateAvailable?: boolean, from?: string, to?: string}= {id: ID};
         let error = null;
+        let errorStack = null;
 
-        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+        if (!(device instanceof Device)) {
             error = `Device '${ID}' does not exist`;
-        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
-            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
+        } else if (!device.definition || !device.definition.ota) {
+            error = `Device '${device.name}' does not support OTA updates`;
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                const meta = {status: `not_supported`, device: device.name};
                 this.mqtt.publish(
                     'bridge/log',
                     stringify({type: `ota_update`, message: error, meta}),
                 );
             }
-        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
-            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else if (this.inProgress.has(device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${device.name}'`;
         } else {
-            this.inProgress.add(resolvedEntity.device.ieeeAddr);
+            this.inProgress.add(device.ieeeAddr);
 
             if (type === 'check') {
-                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                const msg = `Checking if update available for '${device.name}'`;
                 logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                    const meta = {status: `checking_if_available`, device: device.name};
                     this.mqtt.publish(
                         'bridge/log',
                         stringify({type: `ota_update`, message: msg, meta}),
@@ -152,15 +164,13 @@ class OTAUpdate extends Extension {
                 }
 
                 try {
-                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
-                        resolvedEntity.device, logger,
-                    );
-                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    const available = await device.definition.ota.isUpdateAvailable(device.zhDevice, logger);
+                    const msg = `${available ? 'Update' : 'No update'} available for '${device.name}'`;
                     logger.info(msg);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        const meta = {status: available ? 'available' : 'not_available', device: device.name};
                         this.mqtt.publish(
                             'bridge/log',
                             stringify({type: `ota_update`, message: msg, meta}),
@@ -168,15 +178,16 @@ class OTAUpdate extends Extension {
                     }
 
                     const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
-                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
-                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    this.publishEntityState(device.ieeeAddr, payload);
+                    this.lastChecked[device.ieeeAddr] = Date.now();
                     responseData.updateAvailable = available;
                 } catch (e) {
-                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
+                    error = `Failed to check if update available for '${device.name}' (${e.message})`;
+                    errorStack = e.stack;
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        const meta = {status: `check_failed`, device: device.name};
                         this.mqtt.publish(
                             'bridge/log',
                             stringify({type: `ota_update`, message: error, meta}),
@@ -184,12 +195,12 @@ class OTAUpdate extends Extension {
                     }
                 }
             } else { // type === 'update'
-                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                const msg = `Updating '${device.name}' to latest firmware`;
                 logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    const meta = {status: `update_in_progress`, device: device.name};
                     this.mqtt.publish(
                         'bridge/log',
                         stringify({type: `ota_update`, message: msg, meta}),
@@ -197,8 +208,8 @@ class OTAUpdate extends Extension {
                 }
 
                 try {
-                    const onProgress = (progress, remaining) => {
-                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                    const onProgress = (progress: number, remaining: number): void => {
+                        let msg = `Update of '${device.name}' at ${progress.toFixed(2)}%`;
                         if (remaining) {
                             msg += `, ≈ ${Math.round(remaining / 60)} minutes remaining`;
                         }
@@ -206,53 +217,55 @@ class OTAUpdate extends Extension {
                         logger.info(msg);
 
                         const payload = this.getEntityPublishPayload('updating', progress, remaining);
-                        this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                        this.publishEntityState(device.ieeeAddr, payload);
 
                         /* istanbul ignore else */
                         if (settings.get().advanced.legacy_api) {
-                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            const meta = {status: `update_progress`, device: device.name, progress};
                             this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
                         }
                     };
 
-                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
-                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
-                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, false);
+                    await device.definition.ota.updateToLatest(device.zhDevice, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(device, true);
                     const [fromS, toS] = [stringify(from_), stringify(to)];
-                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                    const msg = `Finished update of '${device.name}'` +
                         (to ? `, from '${fromS}' to '${toS}'` : ``);
                     logger.info(msg);
-                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    this.removeProgressAndRemainingFromState(device);
                     const payload = this.getEntityPublishPayload('idle');
-                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.publishEntityState(device.ieeeAddr, payload);
                     responseData.from = from_ ? utils.toSnakeCase(from_) : null;
                     responseData.to = to ? utils.toSnakeCase(to) : null;
                     this.eventBus.emit(`devicesChanged`);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                        const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
                     }
                 } catch (e) {
-                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+                    logger.debug(`Update of '${device.name}' failed (${e})`);
+                    error = `Update of '${device.name}' failed (${e.message})`;
+                    errorStack = e.stack;
 
-                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    this.removeProgressAndRemainingFromState(device);
                     const payload = this.getEntityPublishPayload('available');
-                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.publishEntityState(device.ieeeAddr, payload);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        const meta = {status: `update_failed`, device: device.name};
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
                     }
                 }
             }
 
-            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+            this.inProgress.delete(device.ieeeAddr);
         }
 
-        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        const triggeredViaLegacyApi = data.topic.match(legacyTopicRegex);
         if (!triggeredViaLegacyApi) {
             const response = utils.getResponse(message, responseData, error);
             await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
@@ -260,6 +273,7 @@ class OTAUpdate extends Extension {
 
         if (error) {
             logger.error(error);
+            errorStack && logger.debug(errorStack);
         }
     }
 }

lib/extension/publish.js

@@ -1,293 +0,0 @@
-
-const settings = require('../util/settings');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const assert = require('assert');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
-const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
-
-const groupConverters = [
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
-    zigbeeHerdsmanConverters.toZigbeeConverters.effect,
-    zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
-    zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
-    zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
-    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
-];
-
-class EntityPublish extends Extension {
-    parseTopic(topic) {
-        const match = topic.match(topicRegex);
-        if (!match) {
-            return null;
-        }
-
-        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
-        // If we didn't replace base_topic we received something we don't care about
-        if (ID === match[1] || ID.match(/bridge/)) {
-            return null;
-        }
-
-        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
-    }
-
-    async onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-        if (!topic) {
-            return false;
-        }
-
-        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
-        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
-
-        if (!resolvedEntity) {
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const message = {friendly_name: entityKey};
-                this.mqtt.publish(
-                    'bridge/log',
-                    stringify({type: `entity_not_found`, message}),
-                );
-            }
-
-            logger.error(`Entity '${entityKey}' is unknown`);
-            return;
-        }
-
-        // Get entity details
-        let converters = null;
-        let target = null;
-        let options = {};
-        let device = null;
-        let definition = null;
-        let membersState = null;
-
-        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
-        if (resolvedEntity.type === 'device') {
-            if (!resolvedEntity.definition) {
-                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-                return;
-            }
-
-            device = resolvedEntity.device;
-            definition = resolvedEntity.definition;
-            target = resolvedEntity.endpoint;
-            converters = resolvedEntity.definition.toZigbee;
-            options = resolvedEntity.settings;
-        } else {
-            target = resolvedEntity.group;
-            options = resolvedEntity.settings;
-            definition = resolvedEntity.group.members
-                .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
-            converters = new Set(groupConverters);
-            for (const d of definition) {
-                d.toZigbee.forEach(converters.add, converters);
-            }
-            converters = [...converters];
-            membersState = {};
-            for (const member of resolvedEntity.group.members) {
-                const ieeeAddr = member.getDevice().ieeeAddr;
-                membersState[ieeeAddr] = this.state.get(ieeeAddr);
-            }
-        }
-
-        // Convert the MQTT message to a Zigbee message.
-        let json = {};
-        if (topic.hasOwnProperty('attribute') && topic.attribute) {
-            try {
-                json[topic.attribute] = JSON.parse(message);
-            } catch (e) {
-                json[topic.attribute] = message;
-            }
-        } else {
-            try {
-                json = JSON.parse(message);
-            } catch (e) {
-                if (stateValues.includes(message.toLowerCase())) {
-                    json = {state: message};
-                } else {
-                    logger.error(`Invalid JSON '${message}', skipping...`);
-                    return false;
-                }
-            }
-        }
-        if (!json) {
-            logger.error(`Invalid JSON '${message}', skipping...`);
-            return;
-        }
-
-        /**
-         * Home Assistant always publishes 'state', even when e.g. only setting
-         * the color temperature. This would lead to 2 zigbee publishes, where the first one
-         * (state) is probably unecessary.
-         */
-        const entityState = this.state.get(resolvedEntity.settings.ID) || {};
-        if (settings.get().homeassistant) {
-            const hasColorTemp = json.hasOwnProperty('color_temp');
-            const hasColor = json.hasOwnProperty('color');
-            const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = entityState && entityState.state === 'ON' ? true : false;
-            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
-                delete json.state;
-                logger.debug('Skipping state because of Home Assistant');
-            }
-        }
-
-        /**
-         * Order state & brightness based on current bulb state
-         *
-         * Not all bulbs support setting the color/color_temp while it is off
-         * this results in inconsistant behavior between different vendors.
-         *
-         * bulb on => move state & brightness to the back
-         * bulb off => move state & brightness to the front
-         */
-        const entries = Object.entries(json);
-        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
-        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
-
-        // For each attribute call the corresponding converter
-        const usedConverters = {};
-        for (let [key, value] of entries) {
-            let endpointName = topic.endpointName;
-            let actualTarget = target;
-
-            // When the key has a endpointName included (e.g. state_right), this will override the target.
-            if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
-                if (endpointNameMatch) {
-                    endpointName = endpointNameMatch;
-                    key = key.replace(`_${endpointNameMatch}`, '');
-
-                    const device = target.getDevice();
-                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
-                    if (!actualTarget) {
-                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
-                        continue;
-                    }
-                }
-            }
-
-            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
-            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
-            const converter = converters.find((c) => c.key.includes(key));
-
-            if (topic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
-                // Use a converter for set only once
-                // (e.g. light_onoff_brightness converters can convert state and brightness)
-                continue;
-            }
-
-            if (!converter) {
-                logger.error(`No converter available for '${key}' (${json[key]})`);
-                continue;
-            }
-
-            // Converter didn't return a result, skip
-            const meta = {
-                endpoint_name: endpointName,
-                options,
-                message: {...json},
-                logger,
-                device,
-                state: entityState,
-                membersState,
-                mapped: definition,
-            };
-
-            // Strip endpoint name from meta.message properties.
-            if (meta.endpoint_name) {
-                for (const [key, value] of Object.entries(meta.message)) {
-                    if (key.endsWith(meta.endpoint_name)) {
-                        delete meta.message[key];
-                        const keyWithoutEndpoint = key.substring(0, key.length - meta.endpoint_name.length - 1);
-                        meta.message[keyWithoutEndpoint] = value;
-                    }
-                }
-            }
-
-            try {
-                if (topic.type === 'set' && converter.convertSet) {
-                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
-                    const result = await converter.convertSet(actualTarget, key, value, meta);
-                    const isGroup = actualTarget.constructor.name == 'Group';
-                    // Optimistic behaviour of groups and devices is different.
-                    const publishOptimistic = isGroup || (!options.hasOwnProperty('optimistic') || options.optimistic);
-                    if (result && result.state && publishOptimistic) {
-                        const msg = result.state;
-
-                        if (endpointName) {
-                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
-                                if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${endpointName}`] = msg[key];
-                                    delete msg[key];
-                                }
-                            }
-                        }
-
-                        this.publishEntityState(resolvedEntity.settings.ID, msg);
-                    }
-
-                    if (result && result.membersState && publishOptimistic) {
-                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
-                            this.publishEntityState(ieeeAddr, state);
-                        }
-                    }
-
-                    // It's possible for devices to get out of sync when writing an attribute that's not reportable.
-                    // So here we re-read the value after a specified timeout, this timeout could for example be the
-                    // transition time of a color change or for forcing a state read for devices that don't
-                    // automatically report a new state when set.
-                    // When reporting is requested for a device (report: true in device-specific settings) we won't
-                    // ever issue a read here, as we assume the device will properly report changes.
-                    // Only do this when the retrieve_state option is enabled for this device.
-                    // retrieve_state == decprecated
-                    if (
-                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
-                        resolvedEntity.settings.retrieve_state
-                    ) {
-                        setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
-                    }
-                } else if (topic.type === 'get' && converter.convertGet) {
-                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
-                    await converter.convertGet(actualTarget, key, meta);
-                } else {
-                    logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
-                    continue;
-                }
-            } catch (error) {
-                const message =
-                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
-                logger.error(message);
-                logger.debug(error.stack);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {friendly_name: resolvedEntity.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        stringify({type: `zigbee_publish_error`, message, meta}),
-                    );
-                }
-            }
-
-            usedConverters[endpointOrGroupID].push(converter);
-        }
-
-        return true;
-    }
-}
-
-module.exports = EntityPublish;

lib/extension/publish.ts

@@ -0,0 +1,289 @@
+
+import * as settings from '../util/settings';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+import logger from '../util/logger';
+import * as utils from '../util/utils';
+import ExtensionTS from './extensionts';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import Group from '../model/group';
+import Device from '../model/device';
+import bind from 'bind-decorator';
+
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}))?/(get|set)(?:/(.+))?`);
+const propertyEndpointRegex = new RegExp(`^(.*)_(${utils.endpointNames.join('|')})$`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
+
+// Legacy: don't provide default converters anymore, this is required by older z2m installs not saving group members
+const defaultGroupConverters = [
+    zhc.toZigbeeConverters.light_onoff_brightness,
+    zhc.toZigbeeConverters.light_color_colortemp,
+    zhc.toZigbeeConverters.effect,
+    zhc.toZigbeeConverters.ignore_transition,
+    zhc.toZigbeeConverters.cover_position_tilt,
+    zhc.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zhc.toZigbeeConverters.tint_scene,
+    zhc.toZigbeeConverters.light_brightness_move,
+    zhc.toZigbeeConverters.light_brightness_step,
+    zhc.toZigbeeConverters.light_colortemp_step,
+    zhc.toZigbeeConverters.light_colortemp_move,
+    zhc.toZigbeeConverters.light_hue_saturation_move,
+    zhc.toZigbeeConverters.light_hue_saturation_step,
+];
+
+interface ParsedTopic {ID: string, endpoint: string, attribute: string, type: 'get' | 'set'}
+
+class Publish extends ExtensionTS {
+    async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage_);
+    }
+
+    parseTopic(topic: string): ParsedTopic | null {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't replace base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge/)) {
+            return null;
+        }
+
+        return {ID: ID, endpoint: match[2], type: match[3] as 'get' | 'set', attribute: match[4]};
+    }
+
+    parseMessage(parsedTopic: ParsedTopic, data: EventMQTTMessage): KeyValue | null {
+        if (parsedTopic.attribute) {
+            try {
+                return {[parsedTopic.attribute]: JSON.parse(data.message)};
+            } catch (e) {
+                return {[parsedTopic.attribute]: data.message};
+            }
+        } else {
+            try {
+                return JSON.parse(data.message);
+            } catch (e) {
+                if (stateValues.includes(data.message.toLowerCase())) {
+                    return {state: data.message};
+                } else {
+                    return null;
+                }
+            }
+        }
+    }
+
+    legacyLog(payload: KeyValue): void {
+        /* istanbul ignore else */
+        if (settings.get().advanced.legacy_api) {
+            this.mqtt.publish('bridge/log', stringify(payload));
+        }
+    }
+
+    legacyRetrieveState(re: Device | Group, converter: ToZigbeeConverter, result: ToZigbeeConverterResult,
+        target: ZHEndpoint | ZHGroup, key: string, meta: ToZigbeeConverterGetMeta): void {
+        // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+        // So here we re-read the value after a specified timeout, this timeout could for example be the
+        // transition time of a color change or for forcing a state read for devices that don't
+        // automatically report a new state when set.
+        // When reporting is requested for a device (report: true in device-specific settings) we won't
+        // ever issue a read here, as we assume the device will properly report changes.
+        // Only do this when the retrieve_state option is enabled for this device.
+        // retrieve_state == decprecated
+        if (re instanceof Device && result && result.hasOwnProperty('readAfterWriteTime') &&
+            re.settings.retrieve_state
+        ) {
+            setTimeout(() => converter.convertGet(target, key, meta), result.readAfterWriteTime);
+        }
+    }
+
+    updateMessageHomeAssistant(message: KeyValue, entityState: KeyValue): void {
+        /**
+         * Home Assistant always publishes 'state', even when e.g. only setting
+         * the color temperature. This would lead to 2 zigbee publishes, where the first one
+         * (state) is probably unecessary.
+         */
+        if (settings.get().homeassistant) {
+            const hasColorTemp = message.hasOwnProperty('color_temp');
+            const hasColor = message.hasOwnProperty('color');
+            const hasBrightness = message.hasOwnProperty('brightness');
+            const isOn = entityState.state === 'ON' ? true : false;
+            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
+                delete message.state;
+                logger.debug('Skipping state because of Home Assistant');
+            }
+        }
+    }
+
+    // TODO remove trailing _
+    @bind async onMQTTMessage_(data: EventMQTTMessage): Promise<void> {
+        const parsedTopic = this.parseTopic(data.topic);
+        if (!parsedTopic) return;
+
+        const re = this.zigbee.resolveEntity(parsedTopic.ID);
+        if (re == null) {
+            this.legacyLog({type: `entity_not_found`, message: {friendly_name: parsedTopic.ID}});
+            logger.error(`Entity '${parsedTopic.ID}' is unknown`);
+            return;
+        }
+
+        // Get entity details
+        const definition = re instanceof Device ? re.definition : re.membersDefinitions();
+        if (definition == null) {
+            logger.error(`Cannot publish to unsupported device '${re.name}'`);
+            return;
+        }
+        const target = re instanceof Group ? re.zhGroup : re.endpoint(parsedTopic.endpoint);
+        if (target == null) {
+            logger.error(`Device '${re.name}' has no endpoint '${parsedTopic.endpoint}'`);
+            return;
+        }
+        const device = re instanceof Device ? re.zhDevice : null;
+        const entitySettings = re.settings;
+        const entityState = this.state.get(re.ID) || {};
+        const membersState = re instanceof Group ?
+            Object.fromEntries(re.membersIeeeAddr().map((e) => [e, this.state.get(e)])) : null;
+        let converters: ToZigbeeConverter[];
+        {
+            if (Array.isArray(definition)) {
+                const c = new Set(definition.map((d) => d.toZigbee).flat());
+                if (c.size == 0) converters = defaultGroupConverters;
+                else converters = Array.from(c);
+            } else {
+                converters = definition.toZigbee;
+            }
+        }
+
+        // Convert the MQTT message to a Zigbee message.
+        const message = this.parseMessage(parsedTopic, data);
+        if (message == null) {
+            logger.error(`Invalid message '${message}', skipping...`);
+            return;
+        }
+        this.updateMessageHomeAssistant(message, entityState);
+
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
+        const entries = Object.entries(message);
+        const sorter = typeof message.state === 'string' && message.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
+
+        // For each attribute call the corresponding converter
+        const usedConverters: {[s: number]: ToZigbeeConverter[]} = {};
+        const toPublish: {[s: number | string]: KeyValue} = {};
+        const addToToPublish = (ID: number | string, payload: KeyValue): void => {
+            if (!(ID in toPublish)) toPublish[ID] = {};
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
+        for (let [key, value] of entries) {
+            let endpointName = parsedTopic.endpoint;
+            let localTarget = target;
+            // @ts-ignore TODO fix
+            let endpointOrGroupID = target.constructor.name === 'Endpoint' ? target.ID : target.groupID;
+
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            const propertyEndpointMatch = key.match(propertyEndpointRegex);
+            if (re instanceof Device && propertyEndpointMatch) {
+                endpointName = propertyEndpointMatch[2];
+                key = propertyEndpointMatch[1];
+                localTarget = re.endpoint(endpointName);
+                if (localTarget == null) {
+                    logger.error(`Device '${re.name}' has no endpoint '${endpointName}'`);
+                    continue;
+                }
+                endpointOrGroupID = localTarget.ID;
+            }
+
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
+            const converter = converters.find((c) => c.key.includes(key));
+
+            if (parsedTopic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
+                continue;
+            }
+
+            if (!converter) {
+                logger.error(`No converter available for '${key}' (${message[key]})`);
+                continue;
+            }
+
+            // Converter didn't return a result, skip
+            const meta = {endpoint_name: endpointName, options: entitySettings, message: {...message}, logger, device,
+                state: entityState, membersState, mapped: definition};
+
+            // Strip endpoint name from meta.message properties.
+            if (endpointName) {
+                for (const [key, value] of Object.entries(meta.message)) {
+                    if (key.endsWith(endpointName)) {
+                        delete meta.message[key];
+                        const keyWithoutEndpoint = key.substring(0, key.length - endpointName.length - 1);
+                        meta.message[keyWithoutEndpoint] = value;
+                    }
+                }
+            }
+
+            try {
+                if (parsedTopic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${parsedTopic.type}' '${key}' to '${re.name}'`);
+                    const result = await converter.convertSet(localTarget, key, value, meta);
+                    const optimistic = !entitySettings.hasOwnProperty('optimistic') || entitySettings.optimistic;
+                    if (result && result.state && optimistic) {
+                        const msg = result.state;
+
+                        if (endpointName) {
+                            for (const key of Object.keys(msg)) {
+                                msg[`${key}_${endpointName}`] = msg[key];
+                                delete msg[key];
+                            }
+                        }
+
+                        // filter out attribute listed in filtered_optimistic
+                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        addToToPublish(re.ID, msg);
+                    }
+
+                    if (result && result.membersState && optimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            addToToPublish(ieeeAddr, state);
+                        }
+                    }
+
+                    this.legacyRetrieveState(re, converter, result, localTarget, key, meta);
+                } else if (parsedTopic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${parsedTopic.type}' '${key}' to '${re.name}'`);
+                    await converter.convertGet(localTarget, key, meta);
+                } else {
+                    logger.error(`No converter available for '${parsedTopic.type}' '${key}' (${message[key]})`);
+                    continue;
+                }
+            } catch (error) {
+                const message =
+                    `Publish '${parsedTopic.type}' '${key}' to '${re.name}' failed: '${error}'`;
+                logger.error(message);
+                logger.debug(error.stack);
+                this.legacyLog({type: `zigbee_publish_error`, message, meta: {friendly_name: re.name}});
+            }
+
+            usedConverters[endpointOrGroupID].push(converter);
+        }
+
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            if (Object.keys(payload).length != 0) {
+                this.publishEntityState(ID, payload);
+            }
+        }
+    }
+}
+
+// TODO_finished: : change class to export default
+module.exports = Publish;

lib/extension/receive.ts

@@ -1,37 +1,35 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const debounce = require('debounce');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+import debounce from 'debounce';
+import ExtensionTS from './extensionts';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
 
-class Receive extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.elapsed = {};
-        this.debouncers = {};
-        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
+class Receive extends ExtensionTS {
+    private elapsed: {[s: string]: number} = {};
+    // eslint-disable-next-line
+    private debouncers: {[s: string]: {payload: KeyValue, publish: any}} = {}; //TODO fix type
+
+    async start(): Promise<void> {
+        this.eventBus.onPublishEntityState(this, this.onPublishEntityState);
+        this.eventBus.onDeviceMessage(this, this.onDeviceMessage);
     }
 
-    async onZigbeeStarted() {
-        this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
-    }
-
-    async onPublishEntityState(data) {
+    @bind async onPublishEntityState(data: EventPublishEntityState): Promise<void> {
         /**
          * Prevent that outdated properties are being published.
          * In case that e.g. the state is currently held back by a debounce and a new state is published
          * remove it from the to be send debounced message.
          */
-        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
-            data.stateChangeReason !== 'publishDebounce') {
+        if (data.ieeeAddr && this.debouncers[data.ieeeAddr] && data.stateChangeReason !== 'publishDebounce') {
             for (const key of Object.keys(data.payload)) {
-                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+                delete this.debouncers[data.ieeeAddr].payload[key];
             }
         }
     }
 
-    publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
+    publishDebounce(ieeeAddr: string, payload: KeyValue, time: number, debounceIgnore: string[]): void {
         if (!this.debouncers[ieeeAddr]) {
             this.debouncers[ieeeAddr] = {
                 payload: {},
@@ -56,7 +54,7 @@ class Receive extends Extension {
     // then all newPayload values with key present in debounce_ignore
     // should equal or be undefined in oldPayload
     // otherwise payload is conflicted
-    isPayloadConflicted(newPayload, oldPayload, debounceIgnore) {
+    isPayloadConflicted(newPayload: KeyValue, oldPayload: KeyValue, debounceIgnore: string[] | null): boolean {
         let result = false;
         Object.keys(oldPayload)
             .filter((key) => (debounceIgnore || []).includes(key))
@@ -69,12 +67,8 @@ class Receive extends Extension {
         return result;
     }
 
-    shouldProcess(type, data, resolvedEntity) {
-        if (type !== 'message' || !resolvedEntity) {
-            return false;
-        }
-
-        if (!resolvedEntity.definition) {
+    shouldProcess(data: EventDeviceMessage): boolean {
+        if (!data.device.definition) {
             if (data.device.interviewing) {
                 logger.debug(`Skipping message, definition is undefined and still interviewing`);
             } else {
@@ -90,7 +84,10 @@ class Receive extends Extension {
         return true;
     }
 
-    onZigbeeEvent(type, data, resolvedEntity) {
+    @bind onDeviceMessage(data: EventDeviceMessage): void {
+        /* istanbul ignore next */
+        if (!data.device) return;
+
         /**
          * Handling of re-transmitted Xiaomi messages.
          * https://github.com/Koenkk/zigbee2mqtt/issues/1238
@@ -103,27 +100,23 @@ class Receive extends Extension {
          * Handling these message would result in false state updates.
          * The group ID attribute of these message defines the network address of the end device.
          */
-        if (type === 'message' && utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && data.groupID) {
+        if (data.device.isXiaomiDevice() && data.device.isRouter() && data.groupID) {
             logger.debug('Handling re-transmitted Xiaomi message');
-            data.device = this.zigbee.getDeviceByNetworkAddress(data.groupID);
-            resolvedEntity = this.zigbee.resolveEntity(data.device);
+            data = {...data, device: this.zigbee.deviceByNetworkAddress(data.groupID)};
         }
 
-        if (!this.shouldProcess(type, data, resolvedEntity)) {
-            return;
-        }
+        if (!this.shouldProcess(data)) return;
 
-        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
+        const converters = data.device.definition.fromZigbee.filter((c) => {
             const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
             return c.cluster === data.cluster && type;
         });
 
         // Check if there is an available converter, genOta messages are not interesting.
-        if (!converters.length && !['genOta', 'genTime', 'genBasic'].includes(data.cluster)) {
-            logger.debug(
-                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
-                `and type '${data.type}' and data '${stringify(data.data)}'`,
-            );
+        const ignoreClusters: (string | number)[] = ['genOta', 'genTime', 'genBasic'];
+        if (converters.length == 0 && !ignoreClusters.includes(data.cluster)) {
+            logger.debug(`No converter available for '${data.device.definition.model}' with ` +
+                `cluster '${data.cluster}' and type '${data.type}' and data '${stringify(data.data)}'`);
             return;
         }
 
@@ -132,7 +125,7 @@ class Receive extends Extension {
         // - If a payload is returned publish it to the MQTT broker
         // - If NO payload is returned do nothing. This is for non-standard behaviour
         //   for e.g. click switches where we need to count number of clicks and detect long presses.
-        const publish = (payload) => {
+        const publish = (payload: KeyValue): void => {
             if (settings.get().advanced.elapsed) {
                 const now = Date.now();
                 if (this.elapsed[data.device.ieeeAddr]) {
@@ -143,22 +136,18 @@ class Receive extends Extension {
             }
 
             // Check if we have to debounce
-            if (resolvedEntity.settings.debounce) {
-                this.publishDebounce(
-                    data.device.ieeeAddr, payload, resolvedEntity.settings.debounce,
-                    resolvedEntity.settings.debounce_ignore,
-                );
+            if (data.device.settings.debounce) {
+                this.publishDebounce(data.device.ieeeAddr, payload, data.device.settings.debounce,
+                    data.device.settings.debounce_ignore);
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
             }
         };
 
-        const meta = {device: data.device, logger, state: this.state.get(data.device.ieeeAddr)};
+        const meta = {device: data.device.zhDevice, logger, state: this.state.get(data.device.ieeeAddr)};
         let payload = {};
         converters.forEach((converter) => {
-            const converted = converter.convert(
-                resolvedEntity.definition, data, publish, resolvedEntity.settings, meta,
-            );
+            const converted = converter.convert(data.device.definition, data, publish, data.device.settings, meta);
             if (converted) {
                 payload = {...payload, ...converted};
             }

lib/model/device.ts

@@ -0,0 +1,71 @@
+/* eslint-disable brace-style */
+import * as settings from '../util/settings';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+
+export default class Device {
+    private device: ZHDevice;
+    private _definition: Definition;
+
+    get endpoints(): ZHEndpoint[] {return this.device.endpoints;}
+    get zhDevice(): ZHDevice {return this.device;}
+    get ieeeAddr(): string {return this.device.ieeeAddr;}
+    get ID(): string {return this.device.ieeeAddr;}
+    get settings(): DeviceSettings {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
+    get name(): string {
+        return this.type === 'Coordinator' ? 'Coordinator' : this.settings?.friendlyName || this.ieeeAddr;}
+    get lastSeen(): number {return this.device.lastSeen;}
+    get modelID(): string {return this.device.modelID;}
+    get softwareBuildID(): string {return this.device.softwareBuildID;}
+    get dateCode(): string {return this.device.dateCode;}
+    get interviewCompleted(): boolean {return this.device.interviewCompleted;}
+    get networkAddress(): number {return this.device.networkAddress;}
+    get manufacturerName(): string {return this.device.manufacturerName;}
+    get interviewing(): boolean {return this.device.interviewing;}
+    get type(): 'Coordinator' | 'Router' | 'EndDevice' | 'Unknown' | 'GreenPower' {return this.device.type;}
+    get powerSource(): string {return this.device.powerSource;}
+    get definition(): Definition | undefined {
+        if (!this._definition && !this.device.interviewing) {
+            this._definition = zhc.findByDevice(this.device);
+        }
+        return this._definition;
+    }
+
+    constructor(device: ZHDevice) {
+        this.device = device;
+    }
+
+    async ping(disableRecovery: boolean): Promise<void> {await this.device.ping(disableRecovery);}
+    async removeFromNetwork(): Promise<void> {await this.device.removeFromNetwork();}
+
+    endpoint(key?: string): ZHEndpoint {
+        let endpoint: ZHEndpoint;
+        if (key == null) key = 'default';
+
+        if (!isNaN(Number(key))) {
+            endpoint = this.device.getEndpoint(Number(key));
+        } else if (this.definition?.endpoint) {
+            const ID = this.definition?.endpoint?.(this.device)[key];
+            if (ID) endpoint = this.device.getEndpoint(ID);
+            else if (key === 'default') endpoint = this.device.endpoints[0];
+            else return null;
+        } else {
+            /* istanbul ignore next */
+            if (key !== 'default') return null;
+            endpoint = this.device.endpoints[0];
+        }
+
+        return endpoint;
+    }
+
+    isXiaomiDevice(): boolean {
+        const xiaomiManufacturerID = [4151, 4447];
+        /* istanbul ignore next */
+        return this.zhDevice.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(this.zhDevice.manufacturerID) &&
+            (!this.zhDevice.manufacturerName || !this.zhDevice.manufacturerName.startsWith('Trust'));
+    }
+
+    isRouter(): boolean {return this.zhDevice.type === 'Router';}
+    lqi(): Promise<LQI> {return this.zhDevice.lqi();}
+    routingTable(): Promise<RoutingTable> {return this.zhDevice.routingTable();}
+}

lib/model/group.ts

@@ -0,0 +1,26 @@
+/* eslint-disable brace-style */
+import * as settings from '../util/settings';
+// @ts-ignore
+import zhc from 'zigbee-herdsman-converters';
+
+export default class Group {
+    private group: ZHGroup;
+
+    get zhGroup(): ZHGroup {return this.group;}
+    get ID(): number {return this.group.groupID;}
+    get settings(): GroupSettings {return settings.getGroup(this.ID);}
+    get name(): string {return this.settings?.friendlyName || this.ID.toString();}
+    get members(): ZHEndpoint[] {return this.group.members;}
+
+    constructor(group: ZHGroup) {
+        this.group = group;
+    }
+
+    membersDefinitions(): Definition[] {
+        return this.members.map((m) => zhc.findByDevice(m.getDevice())).filter((d) => d) as Definition[];
+    }
+
+    membersIeeeAddr(): string[] {
+        return this.members.map((m) => m.getDevice().ieeeAddr);
+    }
+}

lib/mqtt.ts

@@ -1,25 +1,31 @@
-const mqtt = require('mqtt');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const fs = require('fs');
-const events = require('events');
+import mqtt from 'mqtt';
+import logger from './util/logger';
+import * as settings from './util/settings';
+import {seconds} from './util/utils';
+import fs from 'fs';
 
-class MQTT extends events.EventEmitter {
-    constructor() {
-        super();
+export default class MQTT {
+    private publishedTopics: Set<string> = new Set();
+    private connectionTimer: NodeJS.Timeout;
+    private client: mqtt.MqttClient;
+    private eventBus: EventBus;
+
+    constructor(eventBus: EventBus) {
+        this.eventBus = eventBus;
         this.onMessage = this.onMessage.bind(this);
-        this.publishedTopics = new Set();
+        this.onConnect = this.onConnect.bind(this);
     }
 
-    async connect() {
+    async connect(): Promise<void> {
         const mqttSettings = settings.get().mqtt;
         logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
 
-        const options = {
+        const options: mqtt.IClientOptions = {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
                 payload: 'offline',
                 retain: settings.get().mqtt.force_disable_retain ? false : true,
+                qos: 1,
             },
         };
 
@@ -62,22 +68,12 @@ class MQTT extends events.EventEmitter {
             options.rejectUnauthorized = false;
         }
 
-        // Set timer at interval to check if connected to MQTT server.
-        const interval = 10 * 1000; // seconds * 1000.
-        this.connectionTimer = setInterval(() => {
-            if (this.client.reconnecting) {
-                logger.error('Not connected to MQTT server!');
-            }
-        }, interval);
-
         return new Promise((resolve) => {
             this.client = mqtt.connect(mqttSettings.server, options);
 
-            const self = this;
-            this.client.on('connect', () => {
-                logger.info('Connected to MQTT server');
-                self.subscribe(`${settings.get().mqtt.base_topic}/#`);
-                self.publish('bridge/state', 'online', {retain: true, qos: 0});
+            const onConnect = this.onConnect;
+            this.client.on('connect', async () => {
+                await onConnect();
                 resolve();
             });
 
@@ -85,37 +81,54 @@ class MQTT extends events.EventEmitter {
         });
     }
 
-    async disconnect() {
+    private async onConnect(): Promise<void> {
+        // Set timer at interval to check if connected to MQTT server.
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = setInterval(() => {
+            if (this.client.reconnecting) {
+                logger.error('Not connected to MQTT server!');
+            }
+        }, seconds(10));
+
+        logger.info('Connected to MQTT server');
+        this.subscribe(`${settings.get().mqtt.base_topic}/#`);
+        await this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    }
+
+    async disconnect(): Promise<void> {
         clearTimeout(this.connectionTimer);
-        this.connectionTimer = null;
         await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
         logger.info('Disconnecting from MQTT server');
         this.client.end();
     }
 
-    subscribe(topic) {
+    subscribe(topic: string): void {
         this.client.subscribe(topic);
     }
 
-    onMessage(topic, message) {
+    public onMessage(topic: string, message: string): void {
         // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
         if (!this.publishedTopics.has(topic)) {
-            this.emit('message', {topic, message: message + ''});
+            this.eventBus.emitMQTTMessage({topic, message: message + ''});
         }
     }
 
-    isConnected() {
+    isConnected(): boolean {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true) {
+    async publish(topic: string, payload: string, options: {qos?: mqtt.QoS, retain?: boolean}={},
+        base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true,
+    ): Promise<void> {
+        const defaultOptions: {qos: mqtt.QoS, retain: boolean} = {qos: 0, retain: false};
         topic = `${base}/${topic}`;
-        options = {qos: 0, retain: false, ...options};
 
         if (skipReceive) {
             this.publishedTopics.add(topic);
         }
 
+        this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});
+
         if (!this.isConnected()) {
             if (!skipLog) {
                 logger.error(`Not connected to MQTT server!`);
@@ -128,18 +141,13 @@ class MQTT extends events.EventEmitter {
             logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
         }
 
-        const actualOptions = {...options};
+        const actualOptions: mqtt.IClientPublishOptions = {...defaultOptions, ...options};
         if (settings.get().mqtt.force_disable_retain) {
             actualOptions.retain = false;
         }
 
         return new Promise((resolve) => {
-            this.client.publish(topic, payload, actualOptions, () => {
-                this.emit('publishedMessage', {topic, payload, options});
-                resolve();
-            });
+            this.client.publish(topic, payload, actualOptions, () => resolve());
         });
     }
 }
-
-module.exports = MQTT;

lib/state.js

@@ -7,9 +7,9 @@ const objectAssignDeep = require('object-assign-deep');
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
+    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
+    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
+    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
 ];
 
 class State {

lib/types/types.ts

@@ -0,0 +1,279 @@
+import type {
+    Device as ZZHDevice,
+    Group as ZZHGroup,
+    Endpoint as ZZHEndpoint,
+} from 'zigbee-herdsman/dist/controller/model';
+
+import {
+    NetworkParameters as ZHNetworkParameters,
+    CoordinatorVersion as ZHCoordinatorVersion,
+    LQI as ZHLQI,
+    RoutingTable as ZHRoutingTable,
+    RoutingTableEntry as ZHRoutingTableEntry,
+} from 'zigbee-herdsman/dist/adapter/tstype';
+
+import * as D from 'lib/model/device';
+import * as Z from 'lib/zigbee';
+import * as E from 'lib/eventBus';
+import * as M from 'lib/mqtt';
+// TODO: check all
+
+declare global {
+    type RecursivePartial<T> = {
+        [P in keyof T]?: RecursivePartial<T[P]>;
+    };
+
+    type EventBus = E.default;
+
+    type MQTT = M.default;
+
+    type Zigbee = Z.default;
+
+    type Device = D.default;
+
+    type ZHEndpoint = ZZHEndpoint;
+
+    type ZHDevice = ZZHDevice;
+    type LQI = ZHLQI;
+    type RoutingTable = ZHRoutingTable;
+    type RoutingTableEntry = ZHRoutingTableEntry;
+
+    type ZHGroup = ZZHGroup;
+
+    type CoordinatorVersion = ZHCoordinatorVersion;
+
+    type NetworkParameters = ZHNetworkParameters;
+
+    type ZigbeeEventType = 'deviceLeave' | 'deviceAnnounce';
+
+    interface ZigbeeEventData {
+        ieeeAddr: string;
+    }
+
+    /* eslint-disable */
+    // Controller
+    interface KeyValue {
+        [s: string]: any,
+    }
+
+    interface Settings {
+        homeassistant?: boolean,
+        devices?: {[s: string]: {friendly_name: string, retention?: number}},
+        groups?: {[s: string]: {friendly_name: string, devices?: string[]}},
+        passlist: string[],
+        blocklist: string[],
+        whitelist: string[],
+        ban: string[],
+        availability?: boolean | {
+            active?: {timeout?: number},
+            passive?: {timeout?: number}
+        },
+        permit_join: boolean,
+        frontend?: {
+            auth_token?: string,
+            host?: string,
+            port?: number,
+        },
+        mqtt: {
+            base_topic: string,
+            include_device_information: boolean,
+            force_disable_retain: boolean
+            version?: number,
+            user?: string,
+            password?: string,
+            server: string,
+            ca?: string,
+            keepalive?: number,
+            key?: string,
+            cert?: string,
+            client_id?: string,
+            reject_unauthorized?: boolean,
+        },
+        serial: {
+            disable_led?: boolean,
+            port?: string,
+            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate'
+        },
+        device_options: {[s: string]: unknown},
+        map_options: {
+            graphviz: {
+                colors: {
+                    fill: {
+                        enddevice: string,
+                        coordinator: string,
+                        router: string,
+                    },
+                    font: {
+                        coordinator: string,
+                        router: string,
+                        enddevice: string,
+                    },
+                    line: {
+                        active: string,
+                        inactive: string,
+                    },
+                },
+            },
+        },
+        experimental: {
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            availability_new?: boolean,
+            transmit_power?: number,
+        },
+        advanced: {
+            legacy_api: boolean,
+            log_rotation: boolean,
+            log_symlink_current: boolean,
+            log_output: ('console' | 'file')[],
+            log_directory: string,
+            log_file: string,
+            log_level: 'debug' | 'info' | 'error' | 'warn',
+            log_syslog: {},
+            soft_reset_timeout: number,
+            pan_id: number | 'GENERATE',
+            ext_pan_id: number[],
+            channel: number,
+            adapter_concurrent: number | null,
+            adapter_delay: number | null,
+            availability_timeout: number,
+            availability_blocklist: string[],
+            availability_passlist: string[],
+            availability_blacklist: string[],
+            availability_whitelist: string[],
+            cache_state: boolean,
+            cache_state_persistent: boolean,
+            cache_state_send_on_startup: boolean,
+            last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' |  'epoch',
+            elapsed: boolean,
+            network_key: number[] | 'GENERATE',
+            report: boolean,
+            homeassistant_discovery_topic: string,
+            homeassistant_status_topic: string,
+            homeassistant_legacy_entity_attributes: boolean,
+            homeassistant_legacy_triggers: boolean,
+            timestamp_format: string,
+            baudrate?: number,
+            rtscts?: boolean,
+            ikea_ota_use_test_url?: boolean,
+        },
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean
+        },
+        external_converters: string[],
+    }
+
+    interface DeviceSettings {
+        friendlyName: string,
+        ID: string,
+        retention?: number,
+        availability?: boolean | {timeout: number},
+        optimistic?: boolean,
+        retrieve_state?: boolean,
+        debounce?: number,
+        debounce_ignore?: string[],
+        filtered_optimistic?: string[],
+        icon?: string,
+        homeassistant?: KeyValue,
+        legacy?: boolean,
+    }
+
+    interface GroupSettings {
+        friendlyName: string,
+        devices: string[],
+        ID: number,
+        optimistic?: boolean,
+        filtered_optimistic?: string[],
+        retrieve_state?: boolean,
+        homeassistant?: KeyValue,
+    }
+
+    type EntitySettings = {
+        type: 'device' | 'group'
+        ID: number | string,
+        friendlyName: string,
+    }
+
+    interface ResolvedEntity {
+        type: 'device' | 'group',
+        device: ZHDevice,
+        name: string,
+    }
+
+    interface ToZigbeeConverterGetMeta {message: {}, mapped: Definition | Definition[]}
+
+    interface ToZigbeeConverterResult {state: KeyValue, membersState: {[s: string]: KeyValue}, readAfterWriteTime?: number}
+
+    interface ToZigbeeConverter {
+        key: string[],
+        convertGet?: (entity: ZHEndpoint | ZHGroup, key: string, meta: ToZigbeeConverterGetMeta) => Promise<void>
+        convertSet?: (entity: ZHEndpoint | ZHGroup, key: string, value: any, meta: {state: KeyValue}) => Promise<ToZigbeeConverterResult>
+    }
+
+    // interface Logger {
+    //     error: (message: string) => void;
+    //     warn: (message: string) => void;
+    //     debug: (message: string) => void;
+    //     info: (message: string) => void;
+    // }
+
+    interface FromZigbeeConverter {
+        cluster: string,
+        type: string[] | string,
+        convert: (model: Definition, msg: KeyValue, publish: (payload: KeyValue) => void, options: KeyValue,
+            meta: {state: KeyValue, logger: any, device: ZHDevice}) => KeyValue,
+    }
+
+    interface DefinitionExposeFeature {name: string, endpoint?: string, property: string, value_max?: number, value_min?: number, value_off?: string, value_on?: string, value_step?: number, values: string[], access: number}
+
+    interface DefinitionExpose {type: string, name?: string, features?: DefinitionExposeFeature[], endpoint?: string, values?: string[], value_off?: string, value_on?: string, access: number, property: string, unit?: string, value_min?: number, value_max?: number}
+
+    interface Definition  {
+        model: string
+        endpoint?: (device: ZHDevice) => {[s: string]: number}
+        toZigbee: ToZigbeeConverter[]
+        fromZigbee: FromZigbeeConverter[]
+        icon?: string
+        description: string
+        vendor: string
+        exposes: DefinitionExpose[] // TODO
+        configure?: (device: ZHDevice, coordinatorEndpoint: ZZHEndpoint, logger: unknown) => Promise<void>;
+        onEvent?: (type: string, data: KeyValue, device: ZHDevice, settings: KeyValue) => Promise<void>;
+        ota?: {
+            isUpdateAvailable: (device: ZHDevice, logger: unknown, data?: KeyValue) => Promise<boolean>;
+            updateToLatest: (device: ZHDevice, logger: unknown, onProgress: (progress: number, remaining: number) => void) => Promise<void>;
+        }
+    }
+
+    interface ResolvedDevice {
+        type: 'device',
+        definition?: Definition,
+        name: string,
+        endpoint: ZHEndpoint,
+        device: ZHDevice,
+        settings: {
+            friendlyName: string,
+            availability?: {timeout?: number} | boolean,
+        }
+    }
+
+    interface TempMQTT {
+        publish: (topic: string, payload: string, options: {}, base?: string, skipLog?: boolean, skipReceive?: boolean) => Promise<void>;
+    }
+
+    interface TempState {
+        get: (ID: string | number) => KeyValue | null;
+        remove: (ID: string | number) => void;
+        removeKey: (ID: string, keys: string[]) => void;
+        exists: (ID: string) => boolean;
+    }
+
+    interface ExternalConverterClass {
+        new(zigbee: Zigbee, mqtt: MQTT, state: TempState, publishEntityState: PublishEntityState,
+            eventBus: EventBus, settings: unknown, logger: unknown): ExternalConverterClass;
+    }
+
+    interface MQTTResponse {data: KeyValue, status: string, error?: string, transaction?: string}
+
+    type PublishEntityState = (ID: string | number, payload: KeyValue, stateChangeReason?: 'publishDebounce') => void;
+}

lib/util/data.js

@@ -1,22 +0,0 @@
-const path = require('path');
-
-let dataPath = null;
-
-function load() {
-    if (process.env.ZIGBEE2MQTT_DATA) {
-        dataPath = process.env.ZIGBEE2MQTT_DATA;
-    } else {
-        dataPath = path.join(__dirname, '..', '..', 'data');
-        dataPath = path.normalize(dataPath);
-    }
-}
-
-load();
-
-module.exports = {
-    joinPath: (file) => path.join(dataPath, file),
-    getPath: () => dataPath,
-
-    // For test only.
-    _reload: () => load(),
-};

lib/util/data.ts

@@ -0,0 +1,27 @@
+import path from 'path';
+
+let dataPath: string = null;
+
+function load(): void {
+    if (process.env.ZIGBEE2MQTT_DATA) {
+        dataPath = process.env.ZIGBEE2MQTT_DATA;
+    } else {
+        dataPath = path.join(__dirname, '..', '..', 'data');
+        dataPath = path.normalize(dataPath);
+    }
+}
+
+load();
+
+export function joinPath(file: string): string {
+    return path.join(dataPath, file);
+}
+
+export function getPath(): string {
+    return dataPath;
+}
+
+// eslint-disable-next-line camelcase
+export function __testingOnly_reload(): void {
+    load();
+}

lib/util/settings.schema.json

@@ -112,7 +112,7 @@
           "requiresRestart": true,
           "description": "MQTT protocol version",
           "default": 4,
-          "examples": [4, 5]
+          "examples": [5]
         },
         "force_disable_retain": {
           "type": "boolean",
@@ -143,9 +143,10 @@
           "default": false
         },
         "adapter": {
-          "type": ["string", "null"],
-          "enum": ["deconz", "zstack", "zigate", "ezsp"],
+          "type": ["string"],
+          "enum": ["deconz", "zstack", "zigate", "ezsp", "auto"],
           "title": "Adapter",
+          "default": "auto",
           "requiresRestart": true,
           "description": "Adapter type, not needed unless you are experiencing problems"
         }
@@ -247,7 +248,7 @@
           "title": "ZigBee channel",
           "requiresRestart": true,
           "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
-          "examples": [11, 15, 20, 25]
+          "examples": [15, 20, 25]
         },
         "cache_state": {
           "type": "boolean",
@@ -422,6 +423,12 @@
           "requiresRestart": true,
           "examples": ["homeassistant"]
         },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
         "homeassistant_status_topic": {
           "type": "string",
           "title": "Home Assistant status topic",
@@ -483,7 +490,6 @@
               "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
               "default": "tcp4",
               "examples": [
-                "tcp4",
                 "udp4",
                 "tls4",
                 "unix",
@@ -496,7 +502,6 @@
               "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
               "default": "/dev/log",
               "examples": [
-                "/dev/log",
                 "/var/run/syslog"
               ]
             },
@@ -680,7 +685,7 @@
         "qos": {
           "type": "number",
           "title": "QoS",
-          "descritption": "QoS level for MQTT messages of this device"
+          "description": "QoS level for MQTT messages of this device"
         },
         "debounce": {
           "type": "number",
@@ -699,7 +704,17 @@
         "optimistic": {
           "type": "boolean",
           "title": "Optimistic",
-          "description": "Publish optimistic state after set (default true)"
+          "description": "Publish optimistic state after set",
+          "default": true
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["color_mode", "color_temp", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
         },
         "filtered_attributes": {
           "type": "array",
@@ -707,8 +722,8 @@
             "type": "string"
           },
           "examples": ["temperature", "battery", "action"],
-          "title": "Filtered attributes",
-          "description": "Allows to prevent certain attributes from being published"
+          "title": "Filtered publish attributes",
+          "description": "Filter attributes from publish payload."
         },
         "icon": {
           "type": "string",

lib/util/settings.ts

@@ -1,17 +1,21 @@
-const data = require('./data');
-const utils = require('./utils');
-// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
-const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
-const objectAssignDeep = require(`object-assign-deep`);
-const path = require('path');
-const yaml = require('./yaml');
-const Ajv = require('ajv');
-const schema = require('./settings.schema.json');
-const ajvSetting = new Ajv({allErrors: true}).compile(schema);
-const ajvRestartRequired = new Ajv({allErrors: true})
-    .addKeyword('requiresRestart', {validate: (v) => !v}).compile(schema);
+import * as data from './data';
+import * as utils from './utils';
+import objectAssignDeep from 'object-assign-deep';
+import path from 'path';
+import * as yaml from './yaml';
+import Ajv from 'ajv';
+import schemaJson from './settings.schema.json';
+export const schema = schemaJson;
 
-const defaults = {
+// TODO: check all
+
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+
+const defaults: RecursivePartial<Settings> = {
     passlist: [],
     blocklist: [],
     // Deprecated: use block/passlist
@@ -125,6 +129,15 @@ const defaults = {
          */
         homeassistant_status_topic: 'hass/status',
 
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
         /**
          * Home Assistant legacy triggers, when enabled:
          * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
@@ -152,11 +165,12 @@ const defaults = {
     external_converters: [],
 };
 
-let _settings;
-let _settingsWithDefaults;
+let _settings: Partial<Settings>;
+let _settingsWithDefaults: Settings;
 
-function write() {
-    const settings = get();
+function write(): void {
+    const settings = getInternalSettings();
+    /* eslint-disable-line */ // @ts-ignore
     const toWrite = objectAssignDeep.noMutate(settings);
 
     // Read settings to check if we have to split devices/groups into separate file.
@@ -179,17 +193,18 @@ function write() {
     }
 
     // Write devices/groups to separate file if required.
-    const writeDevicesOrGroups = (type) => {
+    const writeDevicesOrGroups = (type: 'devices' | 'groups'): void => {
         if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
             const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            /* eslint-disable-line */ // @ts-ignore
             const content = objectAssignDeep.noMutate(settings[type]);
 
             // If an array, only write to first file and only devices which are not in the other files.
             if (Array.isArray(actual[type])) {
-                actual[type].filter((f, i) => i !== 0)
-                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
-                    .map((c) => Object.keys(c))
-                    .forEach((k) => delete content[k]);
+                actual[type].filter((f: string, i: number) => i !== 0)
+                    .map((f: string) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c: KeyValue) => Object.keys(c))
+                    .forEach((k: string) => delete content[k]);
             }
 
             yaml.writeIfChanged(data.joinPath(fileToWrite), content);
@@ -203,12 +218,13 @@ function write() {
     yaml.writeIfChanged(file, toWrite);
 
     _settings = read();
-    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    /* eslint-disable-line */ // @ts-ignore
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
 }
 
-function validate() {
+export function validate(): string[] {
     try {
-        get();
+        getInternalSettings();
     } catch (error) {
         if (error.name === 'YAMLException') {
             return [
@@ -221,7 +237,7 @@ function validate() {
     }
 
     if (!ajvSetting(_settings)) {
-        return ajvSetting.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+        return ajvSetting.errors.map((v) => `${v.instancePath.substring(1)} ${v.message}`);
     }
 
     const errors = [];
@@ -236,14 +252,14 @@ function validate() {
     }
 
     // Verify that all friendly names are unique
-    const names = [];
-    const check = (name) => {
+    const names: string[] = [];
+    const check = (name: string): void => {
         if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
         errors.push(...utils.validateFriendlyName(name));
         names.push(name);
     };
 
-    const settingsWithDefaults = getWithDefaults();
+    const settingsWithDefaults = get();
     Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
     Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
 
@@ -255,7 +271,7 @@ function validate() {
         }
     }
 
-    const checkAvailabilityList = (list, type) => {
+    const checkAvailabilityList = (list: string[], type: string): void => {
         list.forEach((e) => {
             if (!getEntity(e)) {
                 errors.push(`Non-existing entity '${e}' specified in '${type}'`);
@@ -271,11 +287,12 @@ function validate() {
     return errors;
 }
 
-function read() {
-    const s = yaml.read(file);
+function read(): Settings {
+    const s = yaml.read(file) as Settings;
 
     // Read !secret MQTT username and password if set
-    const interpetValue = (value) => {
+    // eslint-disable-next-line
+    const interpetValue = (value: any): any => {
         const re = /!(.*) (.*)/g;
         const match = re.exec(value);
         if (match) {
@@ -287,26 +304,28 @@ function read() {
         }
     };
 
-    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+    if (s.mqtt?.user && s.mqtt?.password) {
         s.mqtt.user = interpetValue(s.mqtt.user);
         s.mqtt.password = interpetValue(s.mqtt.password);
     }
 
-    if (s.advanced && s.advanced.network_key) {
+    if (s.advanced?.network_key) {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    if (s.frontend && s.frontend.auth_token) {
+    if (s.frontend?.auth_token) {
         s.frontend.auth_token = interpetValue(s.frontend.auth_token);
     }
 
     // Read devices/groups configuration from separate file if specified.
-    const readDevicesOrGroups = (type) => {
+    const readDevicesOrGroups = (type: 'devices' | 'groups'): void => {
         if (typeof s[type] === 'string' || Array.isArray(s[type])) {
-            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            /* eslint-disable-line */ // @ts-ignore
+            const files: string[] = Array.isArray(s[type]) ? s[type] : [s[type]];
             s[type] = {};
             for (const file of files) {
                 const content = yaml.readIfExists(data.joinPath(file), {});
+                /* eslint-disable-line */ // @ts-ignore
                 s[type] = objectAssignDeep.noMutate(s[type], content);
             }
         }
@@ -318,23 +337,26 @@ function read() {
     return s;
 }
 
-function applyEnvironmentVariables(settings) {
-    const iterate = (obj, path) => {
+function applyEnvironmentVariables(settings: Partial<Settings>): void {
+    const iterate = (obj: KeyValue, path: string[]): void => {
         Object.keys(obj).forEach((key) => {
             if (key !== 'type') {
-                if (key !== 'properties') {
+                if (key !== 'properties' && obj[key]) {
                     const type = (obj[key].type || 'object').toString();
                     const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
                     const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
                     if (process.env[envVariableName]) {
-                        const setting = path.reduce((acc, val, index) => {
+                        const setting = path.reduce((acc, val) => {
+                            /* eslint-disable-line */ // @ts-ignore
                             acc[val] = acc[val] || {};
+                            /* eslint-disable-line */ // @ts-ignore
                             return acc[val];
                         }, settings);
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
                             setting[key] = JSON.parse(process.env[envVariableName]);
                         } else if (type.indexOf('number') >= 0) {
+                            /* eslint-disable-line */ // @ts-ignore
                             setting[key] = process.env[envVariableName] * 1;
                         } else if (type.indexOf('boolean') >= 0) {
                             setting[key] = process.env[envVariableName].toLowerCase() === 'true';
@@ -347,7 +369,7 @@ function applyEnvironmentVariables(settings) {
                     }
                 }
 
-                if (typeof obj[key] === 'object') {
+                if (typeof obj[key] === 'object' && obj[key]) {
                     const newPath = [...path];
                     if (key !== 'properties') {
                         newPath.push(key);
@@ -360,7 +382,7 @@ function applyEnvironmentVariables(settings) {
     iterate(schema.properties, []);
 }
 
-function get() {
+function getInternalSettings(): Partial<Settings> {
     if (!_settings) {
         _settings = read();
         applyEnvironmentVariables(_settings);
@@ -369,9 +391,10 @@ function get() {
     return _settings;
 }
 
-function getWithDefaults() {
+export function get(): Settings {
     if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+        /* eslint-disable-line */ // @ts-ignore
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
     }
 
     if (!_settingsWithDefaults.devices) {
@@ -385,8 +408,9 @@ function getWithDefaults() {
     return _settingsWithDefaults;
 }
 
-function set(path, value) {
-    let settings = get();
+export function set(path: string[], value: string | number | boolean | KeyValue): void {
+    /* eslint-disable-next-line */
+    let settings: any = getInternalSettings();
 
     for (let i = 0; i < path.length; i++) {
         const key = path[i];
@@ -404,15 +428,16 @@ function set(path, value) {
     write();
 }
 
-function apply(newSettings) {
+export function apply(newSettings: Record<string, unknown>): boolean {
     ajvSetting(newSettings);
     const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
     if (errors.length) {
         const error = errors[0];
-        throw new Error(`${error.dataPath.substring(1)} ${error.message}`);
+        throw new Error(`${error.instancePath.substring(1)} ${error.message}`);
     }
 
-    get(); // Ensure _settings is intialized.
+    getInternalSettings(); // Ensure _settings is intialized.
+    /* eslint-disable-line */ // @ts-ignore
     _settings = objectAssignDeep.noMutate(_settings, newSettings);
     write();
 
@@ -422,30 +447,30 @@ function apply(newSettings) {
     return restartRequired;
 }
 
-function getGroup(IDorName) {
-    const settings = getWithDefaults();
+export function getGroup(IDorName: string | number): GroupSettings {
+    const settings = get();
     const byID = settings.groups[IDorName];
     if (byID) {
-        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+        return {devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
     }
 
     for (const [ID, group] of Object.entries(settings.groups)) {
         if (group.friendly_name === IDorName) {
-            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+            return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
         }
     }
 
     return null;
 }
 
-function getGroups() {
-    const settings = getWithDefaults();
+export function getGroups(): GroupSettings[] {
+    const settings = get();
     return Object.entries(settings.groups).map(([ID, group]) => {
-        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
     });
 }
 
-function getGroupThrowIfNotExists(IDorName) {
+function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
     const group = getGroup(IDorName);
     if (!group) {
         throw new Error(`Group '${IDorName}' does not exist`);
@@ -454,8 +479,8 @@ function getGroupThrowIfNotExists(IDorName) {
     return group;
 }
 
-function getDevice(IDorName) {
-    const settings = getWithDefaults();
+export function getDevice(IDorName: string): DeviceSettings {
+    const settings = get();
     const byID = settings.devices[IDorName];
     if (byID) {
         return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
@@ -470,7 +495,7 @@ function getDevice(IDorName) {
     return null;
 }
 
-function getDeviceThrowIfNotExists(IDorName) {
+function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
     const device = getDevice(IDorName);
     if (!device) {
         throw new Error(`Device '${IDorName}' does not exist`);
@@ -479,7 +504,7 @@ function getDeviceThrowIfNotExists(IDorName) {
     return device;
 }
 
-function getEntity(IDorName) {
+export function getEntity(IDorName: string): EntitySettings {
     const device = getDevice(IDorName);
     if (device) {
         return {...device, type: 'device'};
@@ -493,12 +518,12 @@ function getEntity(IDorName) {
     return null;
 }
 
-function addDevice(ID) {
+export function addDevice(ID: string): DeviceSettings {
     if (getDevice(ID)) {
         throw new Error(`Device '${ID}' already exists`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
 
     if (!settings.devices) {
         settings.devices = {};
@@ -509,9 +534,8 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
-// Legacy: can be removed after bridgeLegacy has been removed
-function whitelistDevice(ID) {
-    const settings = get();
+export function whitelistDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.whitelist) {
         settings.whitelist = [];
     }
@@ -524,8 +548,8 @@ function whitelistDevice(ID) {
     write();
 }
 
-function blockDevice(ID) {
-    const settings = get();
+export function blockDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.blocklist) {
         settings.blocklist = [];
     }
@@ -534,8 +558,8 @@ function blockDevice(ID) {
     write();
 }
 
-function banDevice(ID) {
-    const settings = get();
+export function banDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.ban) {
         settings.ban = [];
     }
@@ -544,20 +568,30 @@ function banDevice(ID) {
     write();
 }
 
-function removeDevice(IDorName) {
+export function removeDevice(IDorName: string): void {
     const device = getDeviceThrowIfNotExists(IDorName);
-    const settings = get();
+    const settings = getInternalSettings();
     delete settings.devices[device.ID];
+
+    // Remove device from groups
+    if (settings.groups) {
+        const regex =
+            new RegExp(`^(${device.friendlyName}|${device.ID})(/(\\d|${utils.endpointNames.join('|')}))?$`);
+        for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
+            group.devices = group.devices.filter((device) => !device.match(regex));
+        }
+    }
+
     write();
 }
 
-function addGroup(name, ID=null) {
+export function addGroup(name: string, ID?: string): GroupSettings {
     utils.validateFriendlyName(name, true);
     if (getGroup(name) || getDevice(name)) {
         throw new Error(`friendly_name '${name}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (!settings.groups) {
         settings.groups = {};
     }
@@ -572,7 +606,7 @@ function addGroup(name, ID=null) {
         // ensure provided ID is not in use
         ID = ID.toString();
         if (settings.groups.hasOwnProperty(ID)) {
-            throw new Error(`group id '${ID}' is already in use`);
+            throw new Error(`Group ID '${ID}' is already in use`);
         }
     }
 
@@ -582,56 +616,50 @@ function addGroup(name, ID=null) {
     return getGroup(ID);
 }
 
-function groupHasDevice(group, keys) {
-    for (const device of group.devices) {
-        const index = keys.indexOf(device);
-        if (index != -1) {
-            return keys[index];
-        }
+function groupGetDevice(group: {devices?: string[]}, keys: string[]): string {
+    for (const device of group.devices ?? []) {
+        if (keys.includes(device)) return device;
     }
 
-    return false;
+    return null;
 }
 
-function addDeviceToGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function addDeviceToGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
 
     const group = settings.groups[groupID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    if (!groupHasDevice(group, keys)) {
+    if (!groupGetDevice(group, keys)) {
+        if (!group.devices) group.devices = [];
         group.devices.push(keys[0]);
         write();
     }
 }
 
-function removeDeviceFromGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeDeviceFromGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
     const group = settings.groups[groupID];
     if (!group.devices) {
-        group.devices = [];
+        return;
     }
 
-    const key = groupHasDevice(group, keys);
+    const key = groupGetDevice(group, keys);
     if (key) {
         group.devices = group.devices.filter((d) => d != key);
         write();
     }
 }
 
-function removeGroup(groupIDorName) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeGroup(IDorName: string | number): void {
+    const groupID = getGroupThrowIfNotExists(IDorName.toString()).ID;
+    const settings = getInternalSettings();
     delete settings.groups[groupID];
     write();
 }
 
-function changeEntityOptions(IDorName, newOptions) {
-    const settings = get();
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+    const settings = getInternalSettings();
     delete newOptions.friendly_name;
     delete newOptions.devices;
     if (getDevice(IDorName)) {
@@ -647,13 +675,13 @@ function changeEntityOptions(IDorName, newOptions) {
     write();
 }
 
-function changeFriendlyName(IDorName, newName) {
+export function changeFriendlyName(IDorName: string, newName: string): void {
     utils.validateFriendlyName(newName, true);
     if (getGroup(newName) || getDevice(newName)) {
         throw new Error(`friendly_name '${newName}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (getDevice(IDorName)) {
         settings.devices[getDevice(IDorName).ID].friendly_name = newName;
     } else if (getGroup(IDorName)) {
@@ -665,39 +693,18 @@ function changeFriendlyName(IDorName, newName) {
     write();
 }
 
-module.exports = {
-    validate,
-    get: getWithDefaults,
-    set,
-    apply,
-    getDevice,
-    getGroup,
-    getGroups,
-    getEntity,
-    whitelistDevice,
-    banDevice,
-    blockDevice,
-    addDevice,
-    removeDevice,
-    addGroup,
-    removeGroup,
-    addDeviceToGroup,
-    removeDeviceFromGroup,
-    changeEntityOptions,
-    changeFriendlyName,
-    schema,
-    reRead: () => {
-        _settings = null;
-        get();
-        _settingsWithDefaults = null;
-        getWithDefaults();
-    },
+export function reRead(): void {
+    _settings = null;
+    getInternalSettings();
+    _settingsWithDefaults = null;
+    get();
+}
 
-    // For tests only
-    _write: write,
-    _clear: () => {
+export const testing = {
+    write,
+    clear: (): void => {
         _settings = null;
         _settingsWithDefaults = null;
     },
-    _getDefaults: () => defaults,
+    defaults,
 };

lib/util/utils.js

@@ -1,291 +0,0 @@
-const equals = require('fast-deep-equal/es6');
-const humanizeDuration = require('humanize-duration');
-const data = require('./data');
-const vm = require('vm');
-const fs = require('fs');
-const path = require('path');
-
-// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
-const xiaomiManufacturerID = [4151, 4447];
-const ikeaTradfriManufacturerID = [4476];
-
-// construct a local ISO8601 string (instead of UTC-based)
-// Example:
-//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
-//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
-function toLocalISOString(dDate) {
-    const tzOffset = -dDate.getTimezoneOffset();
-    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
-    const pad = function(num) {
-        const norm = Math.floor(Math.abs(num));
-        return (norm < 10 ? '0' : '') + norm;
-    };
-
-    return dDate.getFullYear() +
-        '-' + pad(dDate.getMonth() + 1) +
-        '-' + pad(dDate.getDate()) +
-        'T' + pad(dDate.getHours()) +
-        ':' + pad(dDate.getMinutes()) +
-        ':' + pad(dDate.getSeconds()) +
-        plusOrMinus + pad(tzOffset / 60) +
-        ':' + pad(tzOffset % 60);
-}
-
-const endpointNames = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
-    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
-    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
-    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
-    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
-    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
-    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
-    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
-    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-    'heat', 'cool', 'water', 'meter', 'wifi',
-];
-
-function capitalize(s) {
-    return s[0].toUpperCase() + s.slice(1);
-}
-
-async function getZigbee2mqttVersion() {
-    return new Promise((resolve, reject) => {
-        const git = require('git-last-commit');
-        const packageJSON = require('../../package.json');
-        const version = packageJSON.version;
-
-        git.getLastCommit((err, commit) => {
-            let commitHash = null;
-
-            if (err) {
-                try {
-                    commitHash = require('../../.hash.json').hash;
-                } catch (error) {
-                    commitHash = 'unknown';
-                }
-            } else {
-                commitHash = commit.shortHash;
-            }
-
-            resolve({commitHash, version});
-        });
-    });
-}
-
-async function getDependencyVersion(depend) {
-    return new Promise((resolve, reject) => {
-        const packageJSON = require('../../node_modules/'+depend+'/package.json');
-        const version = packageJSON.version;
-        resolve({version});
-    });
-}
-
-function formatDate(date, type) {
-    let result;
-
-    switch (type) {
-    case 'ISO_8601':
-        result = new Date(date).toISOString();
-        break;
-    case 'ISO_8601_local':
-        result = toLocalISOString(new Date(date));
-        break;
-    case 'epoch':
-        result = date;
-        break;
-    case 'relative':
-        // https://github.com/EvanHahn/HumanizeDuration.js#options
-        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
-        break;
-    default:
-        throw new Error(`Unsupported type '${type}'`);
-    }
-
-    return result;
-}
-
-function objectHasProperties(object, properties) {
-    for (const property of properties) {
-        if (!object.hasOwnProperty(property)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-function equalsPartial(object, expected) {
-    for (const [key, value] of Object.entries(expected)) {
-        if (!equals(object[key], value)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-function getObjectProperty(object, key, defaultValue) {
-    return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
-}
-
-function getResponse(request, data, error) {
-    const response = {data, status: error ? 'error' : 'ok'};
-    if (error) response.error = error;
-    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
-        response.transaction = request.transaction;
-    }
-    return response;
-}
-
-function parseJSON(value, failedReturnValue) {
-    try {
-        return JSON.parse(value);
-    } catch (e) {
-        return failedReturnValue;
-    }
-}
-
-function loadModuleFromText(moduleCode) {
-    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
-    const sandbox = {
-        require: require,
-        module: {},
-        console,
-        setTimeout,
-        clearTimeout,
-        setInterval,
-        clearInterval,
-        setImmediate,
-        clearImmediate,
-    };
-    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
-    return sandbox.module.exports;
-}
-
-function loadModuleFromFile(modulePath) {
-    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
-    return loadModuleFromText(moduleCode);
-}
-
-function* getExternalConvertersDefinitions(settings) {
-    const externalConverters = settings.get().external_converters;
-
-    for (const moduleName of externalConverters) {
-        let converter;
-
-        if (moduleName.endsWith('.js')) {
-            converter = loadModuleFromFile(data.joinPath(moduleName));
-        } else {
-            converter = require(moduleName);
-        }
-
-        if (Array.isArray(converter)) {
-            for (const item of converter) {
-                yield item;
-            }
-        } else {
-            yield converter;
-        }
-    }
-}
-
-function removeNullPropertiesFromObject(obj) {
-    for (const key of Object.keys(obj)) {
-        const value = obj[key];
-        if (value == null) {
-            delete obj[key];
-        } else if (typeof value === 'object') {
-            removeNullPropertiesFromObject(value);
-        }
-    }
-}
-
-function getKey(object, value, fallback, convertTo) {
-    for (const key in object) {
-        if (object[key]===value) {
-            return convertTo ? convertTo(key) : key;
-        }
-    }
-
-    return fallback;
-}
-
-function toSnakeCase(value) {
-    if (typeof value === 'object') {
-        value = {...value};
-        for (const key of Object.keys(value)) {
-            const keySnakeCase = toSnakeCase(key);
-            if (key !== keySnakeCase) {
-                value[keySnakeCase] = value[key];
-                delete value[key];
-            }
-        }
-        return value;
-    } else {
-        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
-    }
-}
-
-function validateFriendlyName(name, throwFirstError=false) {
-    const errors = [];
-    for (const endpointName of endpointNames) {
-        if (name.toLowerCase().endsWith('/' + endpointName)) {
-            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
-        }
-    }
-
-    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
-    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
-    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
-    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
-    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
-    if (name.includes('#') || name.includes('+')) {
-        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
-    }
-
-    if (throwFirstError && errors.length) {
-        throw new Error(errors[0]);
-    }
-
-    return errors;
-}
-function sleep(seconds) {
-    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
-}
-
-function sanitizeImageParameter(parameter) {
-    const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
-    let sanitized = parameter;
-    replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
-    return sanitized;
-}
-
-module.exports = {
-    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
-    secondsToMilliseconds: (seconds) => seconds * 1000,
-    getZigbee2mqttVersion,
-    getDependencyVersion,
-    objectHasProperties,
-    toSnakeCase,
-    sleep,
-    getObjectProperty,
-    getEndpointNames: () => endpointNames,
-    isXiaomiDevice: (device) => {
-        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
-            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
-    },
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
-    formatDate: (date, type) => formatDate(date, type),
-    equalsPartial,
-    getResponse,
-    capitalize,
-    parseJSON,
-    getExternalConvertersDefinitions,
-    validateFriendlyName,
-    loadModuleFromFile,
-    loadModuleFromText,
-    getKey,
-    sanitizeImageParameter,
-    removeNullPropertiesFromObject,
-};

lib/util/utils.ts

@@ -0,0 +1,328 @@
+import equals from 'fast-deep-equal/es6';
+import humanizeDuration from 'humanize-duration';
+import * as data from './data';
+import vm from 'vm';
+import fs from 'fs';
+import path from 'path';
+import {Endpoint} from 'zigbee-herdsman/dist/controller/model';
+
+// TODO: check all
+
+// construct a local ISO8601 string (instead of UTC-based)
+// Example:
+//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
+//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
+function toLocalISOString(date: Date): string {
+    const tzOffset = -date.getTimezoneOffset();
+    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
+    const pad = (num: number): string => {
+        const norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+    };
+
+    return date.getFullYear() +
+        '-' + pad(date.getMonth() + 1) +
+        '-' + pad(date.getDate()) +
+        'T' + pad(date.getHours()) +
+        ':' + pad(date.getMinutes()) +
+        ':' + pad(date.getSeconds()) +
+        plusOrMinus + pad(tzOffset / 60) +
+        ':' + pad(tzOffset % 60);
+}
+
+export const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay', 'usb',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool', 'water', 'meter', 'wifi',
+];
+
+export function capitalize(s: string): string {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
+export async function getZigbee2MQTTVersionSimple(): Promise<string> {
+    const packageJSON = await import('../..' + '/package.json');
+    return packageJSON.version;
+}
+
+export async function getZigbee2MQTTVersion(): Promise<{commitHash: string, version: string}> {
+    const git = await import('git-last-commit');
+    const packageJSON = await import('../..' + '/package.json');
+
+    return new Promise((resolve) => {
+        const version = packageJSON.version;
+
+        git.getLastCommit((err: Error, commit: {shortHash: string}) => {
+            let commitHash = null;
+
+            if (err) {
+                try {
+                    commitHash = fs.readFileSync(path.join(__dirname, '..', '..', 'dist', '.hash'), 'utf-8');
+                } catch (error) {
+                    /* istanbul ignore next */
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
+            }
+
+            resolve({commitHash, version});
+        });
+    });
+}
+
+export async function getDependencyVersion(depend: string): Promise<{version: string}> {
+    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const version = packageJSON.version;
+    return {version};
+}
+
+export function formatDate(time: number, type: 'ISO_8601' | 'ISO_8601_local' | 'epoch' | 'relative'): string | number {
+    if (type === 'ISO_8601') return new Date(time).toISOString();
+    else if (type === 'ISO_8601_local') return toLocalISOString(new Date(time));
+    else if (type === 'epoch') return time;
+    else { // relative
+        return humanizeDuration(Date.now() - time, {language: 'en', largest: 2, round: true}) + ' ago';
+    }
+}
+
+export function objectHasProperties(object: {[s: string]: unknown}, properties: string[]): boolean {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+export function equalsPartial(object: {[s: string]: unknown}, expected: {[s: string]: unknown}): boolean {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+export function getObjectProperty(object: {[s: string]: unknown}, key: string, defaultValue: unknown): unknown {
+    return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
+}
+
+export function getResponse(request: KeyValue | string, data: KeyValue, error: string): MQTTResponse {
+    const response: {data: unknown, status: string, error?: string, transaction?: string} =
+        {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+export function parseJSON(value: string, failedReturnValue: string): KeyValue | string {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+export function loadModuleFromText(moduleCode: string): unknown {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+        setTimeout,
+        clearTimeout,
+        setInterval,
+        clearInterval,
+        setImmediate,
+        clearImmediate,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    /* eslint-disable-line */ // @ts-ignore
+    return sandbox.module.exports;
+}
+
+export function loadModuleFromFile(modulePath: string): unknown {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
+/* eslint-disable-next-line */
+export function* getExternalConvertersDefinitions(settings: any): any {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converter;
+
+        if (moduleName.endsWith('.js')) {
+            converter = loadModuleFromFile(data.joinPath(moduleName));
+        } else {
+            converter = require(moduleName);
+        }
+
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+export function removeNullPropertiesFromObject(obj: KeyValue): void {
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value == null) {
+            delete obj[key];
+        } else if (typeof value === 'object') {
+            removeNullPropertiesFromObject(value);
+        }
+    }
+}
+
+export function getKey(
+    object: KeyValue, value: unknown, fallback: unknown, convertTo: (v: unknown) => unknown): unknown {
+    for (const key in object) {
+        if (object[key]===value) {
+            return convertTo ? convertTo(key) : key;
+        }
+    }
+
+    return fallback;
+}
+
+export function toNetworkAddressHex(value: number): string {
+    const hex = value.toString(16);
+    return `0x${'0'.repeat(4 - hex.length)}${hex}`;
+}
+
+// eslint-disable-next-line
+export function toSnakeCase(value: string | KeyValue): any {
+    if (typeof value === 'object') {
+        value = {...value};
+        for (const key of Object.keys(value)) {
+            const keySnakeCase = toSnakeCase(key);
+            if (key !== keySnakeCase) {
+                value[keySnakeCase] = value[key];
+                delete value[key];
+            }
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
+export function validateFriendlyName(name: string, throwFirstError=false): string[] {
+    const errors = [];
+    for (const endpointName of endpointNames) {
+        if (name.toLowerCase().endsWith('/' + endpointName)) {
+            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
+        }
+    }
+
+    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
+    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
+    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+
+    if (throwFirstError && errors.length) {
+        throw new Error(errors[0]);
+    }
+
+    return errors;
+}
+
+export function sleep(seconds: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+export function sanitizeImageParameter(parameter: string): string {
+    const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
+    let sanitized = parameter;
+    replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
+    return sanitized;
+}
+
+export function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boolean {
+    /* istanbul ignore next */
+    if (!settings.experimental.availability_new) return false;
+
+    if (device.settings.hasOwnProperty('availability')) {
+        return !!device.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(device.name) || passlist.includes(device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(device.name) && !blocklist.includes(device.ieeeAddr);
+}
+
+/* istanbul ignore next */
+export function isAvailabilityEnabledForDeviceLegacy(rd: ResolvedDevice, settings: Settings): boolean {
+    if (!settings.experimental.availability_new) return false;
+
+    if (rd.settings.hasOwnProperty('availability')) {
+        return !!rd.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(rd.name) || passlist.includes(rd.device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(rd.name) && !blocklist.includes(rd.device.ieeeAddr);
+}
+
+export function isXiaomiDevice(device: ZHDevice): boolean {
+    const xiaomiManufacturerID = [4151, 4447];
+    return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+        (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+}
+
+export function isIkeaTradfriDevice(device: ZHDevice): boolean {
+    return [4476].includes(device.manufacturerID);
+}
+
+const entityIDRegex = new RegExp(`^(.+?)(?:/(${endpointNames.join('|')}|\\d+))?$`);
+export function parseEntityID(ID: string): {ID: string, endpoint: string} {
+    const match = ID.match(entityIDRegex);
+    return match && {ID: match[1], endpoint: match[2]};
+}
+
+export function isEndpoint(obj: unknown): obj is Endpoint {
+    return obj.constructor.name.toLowerCase() === 'endpoint';
+}
+
+export const hours = (hours: number): number => 1000 * 60 * 60 * hours;
+export const minutes = (minutes: number): number => 1000 * 60 * minutes;
+export const seconds = (seconds: number): number => 1000 * seconds;

lib/util/yaml.js

@@ -1,36 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-const equals = require('fast-deep-equal/es6');
-
-function read(file) {
-    try {
-        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-    } catch (error) {
-        if (error.name === 'YAMLException') {
-            error.file = file;
-        }
-
-        throw error;
-    }
-}
-
-function readIfExists(file, default_=null) {
-    return fs.existsSync(file) ? read(file) : default_;
-}
-
-function writeIfChanged(file, content) {
-    const before = readIfExists(file);
-    if (!equals(before, content)) {
-        fs.writeFileSync(file, yaml.safeDump(content));
-    }
-}
-
-function updateIfChanged(file, key, value) {
-    const content = read(file);
-    if (content[key] !== value) {
-        content[key] = value;
-        writeIfChanged(file, content);
-    }
-}
-
-module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/util/yaml.ts

@@ -0,0 +1,36 @@
+import yaml from 'js-yaml';
+import fs from 'fs';
+import equals from 'fast-deep-equal/es6';
+
+// TODO: check all
+export function read(file: string): KeyValue {
+    try {
+        /* eslint-disable-line */ // @ts-ignore
+        return yaml.load(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+export function readIfExists(file: string, default_?: KeyValue): KeyValue {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+export function writeIfChanged(file: string, content: KeyValue): void {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.dump(content));
+    }
+}
+
+export function updateIfChanged(file: string, key: string, value: KeyValue): void {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}

lib/zigbee.ts

@@ -1,32 +1,39 @@
-const ZigbeeHerdsman = require('zigbee-herdsman');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const data = require('./util/data');
-const assert = require('assert');
-const utils = require('./util/utils');
-const events = require('events');
-const objectAssignDeep = require('object-assign-deep');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const stringify = require('json-stable-stringify-without-jsonify');
+import {Controller} from 'zigbee-herdsman';
+import logger from './util/logger';
+import * as settings from './util/settings';
+import * as data from './util/data';
+import assert from 'assert';
+import * as utils from './util/utils';
+import objectAssignDeep from 'object-assign-deep';
+// @ts-ignore
+import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
+// @ts-ignore
+import stringify from 'json-stable-stringify-without-jsonify';
+import Device from './model/device';
+import Group from './model/group';
+import * as ZHEvents from 'zigbee-herdsman/dist/controller/events';
 
-const endpointNames = utils.getEndpointNames();
-const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
+export default class Zigbee {
+    private herdsman: Controller;
+    private eventBus: EventBus;
+    private resolvedEntitiesLookup: {[s: string]: Device | Group} = {};
 
-class Zigbee extends events.EventEmitter {
-    constructor() {
-        super();
+    constructor(eventBus: EventBus) {
         this.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler.bind(this);
+        this.eventBus = eventBus;
     }
 
-    async start() {
+    async start(): Promise<'reset' | 'resumed' | 'restored'> {
         const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
         logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
         const herdsmanSettings = {
             network: {
-                panID: settings.get().advanced.pan_id,
+                panID: settings.get().advanced.pan_id === 'GENERATE' ?
+                    this.generatePanID() : settings.get().advanced.pan_id as number,
                 extendedPanID: settings.get().advanced.ext_pan_id,
                 channelList: [settings.get().advanced.channel],
-                networkKey: settings.get().advanced.network_key,
+                networkKey: settings.get().advanced.network_key === 'GENERATE' ?
+                    this.generateNetworkKey() : settings.get().advanced.network_key as number[],
             },
             databasePath: data.joinPath('database.db'),
             databaseBackupPath: data.joinPath('database.db.backup'),
@@ -40,43 +47,58 @@ class Zigbee extends events.EventEmitter {
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
                 delay: settings.get().advanced.adapter_delay,
+                disableLED: settings.get().serial.disable_led,
             },
+            acceptJoiningDeviceHandler: this.acceptJoiningDeviceHandler,
         };
 
-        const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
+        const herdsmanSettingsLog = objectAssignDeep({}, herdsmanSettings);
+        // @ts-ignore
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
         logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
 
-        if (herdsmanSettings.network.networkKey === 'GENERATE') {
-            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
-            settings.set(['advanced', 'network_key'], newKey);
-            herdsmanSettings.network.networkKey = newKey;
-        }
-
-        if (herdsmanSettings.network.panID === 'GENERATE') {
-            const newPanID = Math.floor(Math.random() * (0xFFFF - 2)) + 1;
-            settings.set(['advanced', 'pan_id'], newPanID);
-            herdsmanSettings.network.panID = newPanID;
-        }
-
+        let startResult;
         try {
-            herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
-            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, logger);
-            await this.herdsman.start();
+            this.herdsman = new Controller(herdsmanSettings, logger);
+            startResult = await this.herdsman.start();
         } catch (error) {
             logger.error(`Error while starting zigbee-herdsman`);
             throw error;
         }
 
-        this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
-        this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
-        this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
-        this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
-        this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
-        this.herdsman.on('message', (data) => this.emit('event', 'message', data));
-        this.herdsman.on('permitJoinChanged', (data) => this.emit('permitJoinChanged', data));
+        this.herdsman.on('adapterDisconnected', () => this.eventBus.emitAdapterDisconnected());
+        this.herdsman.on('lastSeenChanged', (data: ZHEvents.LastSeenChangedPayload) => {
+            this.eventBus.emitLastSeenChanged({device: this.resolveEntity(data.device) as Device});
+        });
+        this.herdsman.on('permitJoinChanged', (data: ZHEvents.PermitJoinChangedPayload) => {
+            this.eventBus.emitPermitJoinChanged(data);
+        });
+        this.herdsman.on('deviceNetworkAddressChanged', (data: ZHEvents.DeviceNetworkAddressChangedPayload) => {
+            this.eventBus.emit('event', 'deviceNetworkAddressChanged', data); // TODO remove this event
+            this.eventBus.emitDeviceNetworkAddressChanged({device: this.resolveEntity(data.device) as Device});
+        });
+        this.herdsman.on('deviceAnnounce', (data: ZHEvents.DeviceAnnouncePayload) => {
+            this.eventBus.emit('event', 'deviceAnnounce', data); // TODO remove this event
+            this.eventBus.emitDeviceAnnounce({device: this.resolveEntity(data.device) as Device});
+        });
+        this.herdsman.on('deviceInterview', (data: ZHEvents.DeviceInterviewPayload) => {
+            this.eventBus.emit('event', 'deviceInterview', data); // TODO remove this event
+            this.eventBus.emitDeviceInterview({device: this.resolveEntity(data.device) as Device, status: data.status});
+        });
+        this.herdsman.on('deviceJoined', (data: ZHEvents.DeviceJoinedPayload) => {
+            this.eventBus.emit('event', 'deviceJoined', data); // TODO remove this event
+            this.eventBus.emitDeviceJoined({device: this.resolveEntity(data.device) as Device});
+        });
+        this.herdsman.on('deviceLeave', (data: ZHEvents.DeviceLeavePayload) => {
+            this.eventBus.emit('event', 'deviceLeave', data); // TODO remove this event
+            this.eventBus.emitDeviceLeave(data);
+        });
+        this.herdsman.on('message', (data: ZHEvents.MessagePayload) => {
+            this.eventBus.emit('event', 'message', data); // TODO remove this event
+            this.eventBus.emitDeviceMessage({...data, device: this.resolveEntity(data.device) as Device});
+        });
 
-        logger.info('zigbee-herdsman started');
+        logger.info(`zigbee-herdsman started (${startResult})`);
         logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
         logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
@@ -106,82 +128,208 @@ class Zigbee extends events.EventEmitter {
             await this.herdsman.setTransmitPower(transmitPower);
             logger.info(`Set transmit power to '${transmitPower}'`);
         }
+
+        return startResult;
     }
 
-    async getCoordinatorVersion() {
+    private generateNetworkKey(): number[] {
+        const key = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+        settings.set(['advanced', 'network_key'], key);
+        return key;
+    }
+
+    private generatePanID(): number {
+        const panID = Math.floor(Math.random() * (0xFFFF - 2)) + 1;
+        settings.set(['advanced', 'pan_id'], panID);
+        return panID;
+    }
+
+    async getCoordinatorVersion(): Promise<CoordinatorVersion> {
         return this.herdsman.getCoordinatorVersion();
     }
 
-    isStopping() {
+    isStopping(): boolean {
         return this.herdsman.isStopping();
     }
 
-    async getNetworkParameters() {
+    async getNetworkParameters(): Promise<NetworkParameters> {
         return this.herdsman.getNetworkParameters();
     }
 
-    async reset(type) {
+    async reset(type: 'soft' | 'hard'): Promise<void> {
         await this.herdsman.reset(type);
     }
 
-    async stop() {
+    async stop(): Promise<void> {
         logger.info('Stopping zigbee-herdsman...');
         await this.herdsman.stop();
         logger.info('Stopped zigbee-herdsman');
     }
 
-    async permitJoin(permit, resolvedEntity, time=undefined) {
-        permit ?
-            logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`) :
-            logger.info('Zigbee: disabling joining new devices.');
+    getPermitJoin(): boolean {
+        return this.herdsman.getPermitJoin();
+    }
 
-        if (resolvedEntity && permit) {
-            await this.herdsman.permitJoin(permit, resolvedEntity.device, time);
+    getPermitJoinTimeout(): number {
+        return this.herdsman.getPermitJoinTimeout();
+    }
+
+    async permitJoin(permit: boolean, device?: Device, time: number=undefined): Promise<void> {
+        if (permit) {
+            logger.info(`Zigbee: allowing new devices to join${device ? ` via ${device.name}` : ''}.`);
+        } else {
+            logger.info('Zigbee: disabling joining new devices.');
+        }
+
+        if (device && permit) {
+            await this.herdsman.permitJoin(permit, device.zhDevice, time);
         } else {
             await this.herdsman.permitJoin(permit, undefined, time);
         }
     }
 
-    getPermitJoin() {
-        return this.herdsman.getPermitJoin();
+    private addDeviceToResolvedEntitiesLookup(ieeeAddr: string): Device {
+        if (!this.resolvedEntitiesLookup[ieeeAddr]) {
+            const device = this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
+            /* istanbul ignore else */
+            if (device) this.resolvedEntitiesLookup[ieeeAddr] = new Device(device);
+        }
+
+        return this.resolvedEntitiesLookup[ieeeAddr] as Device;
     }
 
-    getPermitJoinTimeout() {
-        return this.herdsman.getPermitJoinTimeout();
+    private addGroupToResolvedEntitiesLookup(groupID: number): Group {
+        if (!this.resolvedEntitiesLookup[groupID]) {
+            let group = this.herdsman.getGroupByID(groupID);
+            // Legacy: previously zigbee-herdsman did not keep track of groups, therefore create it when published to
+            // note that the group is in the configuration.yaml already.
+            if (group == null) group = this.herdsman.createGroup(groupID);
+            this.resolvedEntitiesLookup[groupID] = new Group(group);
+        }
+
+        return this.resolvedEntitiesLookup[groupID] as Group;
     }
 
-    getClients() {
-        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
+    resolveEntity(key: ZHDevice | string): Device | Group {
+        const ID = typeof key === 'string' ? key : key.ieeeAddr;
+        const entitySettings = settings.getEntity(ID);
+        if (!entitySettings && !(typeof key === 'object' && key.type === 'Coordinator')) return undefined;
+
+        if (typeof key === 'object') {
+            return this.addDeviceToResolvedEntitiesLookup(key.ieeeAddr);
+        } else {
+            return entitySettings.type === 'device' ?
+                this.addDeviceToResolvedEntitiesLookup(entitySettings.ID as string) :
+                this.addGroupToResolvedEntitiesLookup(entitySettings.ID as number);
+        }
     }
 
-    getDevices() {
-        return this.herdsman.getDevices();
+    getClients(): Device[] {
+        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator')
+            .map((d) => this.resolveEntity(d) as Device).filter((d) => d);
     }
 
-    getDeviceByIeeeAddr(ieeeAddr) {
-        return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
+    getFirstCoordinatorEndpoint(): ZHEndpoint {
+        return this.herdsman.getDevicesByType('Coordinator')[0].endpoints[0];
     }
 
-    getDeviceByNetworkAddress(networkAddress) {
-        return this.herdsman.getDeviceByNetworkAddress(networkAddress);
+    getGroups(): Group[] {
+        return this.herdsman.getGroups().map((g) => this.addGroupToResolvedEntitiesLookup(g.groupID)).filter((g) => g);
     }
 
-    getDevicesByType(type) {
+    getDevices(): Device[] {
+        return this.herdsman.getDevices()
+            .map((d) => this.addDeviceToResolvedEntitiesLookup(d.ieeeAddr)).filter((d) => d);
+    }
+
+    private async acceptJoiningDeviceHandler(ieeeAddr: string): Promise<boolean> {
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
+                return true;
+            } else {
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
+                return false;
+            }
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
+                return false;
+            } else {
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    async touchlinkFactoryResetFirst(): Promise<boolean> {
+        return this.herdsman.touchlinkFactoryResetFirst();
+    }
+
+    async touchlinkFactoryReset(ieeeAddr: string, channel: number): Promise<boolean> {
+        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
+    }
+
+    async touchlinkIdentify(ieeeAddr: string, channel: number): Promise<void> {
+        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
+    }
+
+    async touchlinkScan(): Promise<{ieeeAddr: string, channel: number}[]> {
+        return this.herdsman.touchlinkScan();
+    }
+
+    createGroup(groupID: number): Group {
+        this.herdsman.createGroup(groupID);
+        return this.addGroupToResolvedEntitiesLookup(groupID);
+    }
+
+    deviceByNetworkAddress(networkAddress: number): Device {
+        const device = this.herdsman.getDeviceByNetworkAddress(networkAddress);
+        return device && this.addDeviceToResolvedEntitiesLookup(device.ieeeAddr);
+    }
+
+    // TODO remove all legacy below
+    createGroupLegacy(groupID: number): ZHGroup {
+        return this.herdsman.createGroup(groupID);
+    }
+    getGroupsLegacy(): ZHGroup[] {
+        return this.herdsman.getGroups();
+    }
+    getGroupByIDLegacy(ID: number): ZHGroup {
+        return this.herdsman.getGroupByID(ID);
+    }
+    getDevicesByTypeLegacy(type: 'Coordinator' | 'Router' | 'EndDevice'): ZHDevice[] {
         return this.herdsman.getDevicesByType(type);
     }
+    getClientsLegacy(): ZHDevice[] {
+        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
+    }
+    getDevicesLegacy(): ZHDevice[] {
+        return this.herdsman.getDevices();
+    }
+    async permitJoinLegacy(permit: boolean, resolvedEntity: ResolvedDevice, time: number=undefined): Promise<void> {
+        if (permit) {
+            /* istanbul ignore next */
+            logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`);
+        } else {
+            logger.info('Zigbee: disabling joining new devices.');
+        }
 
-    /**
-     * @param {string} key
-     * @return {object} {
-     *      type: device | coordinator
-     *      device|group: zigbee-herdsman entity
-     *      endpoint: selected endpoint (only if type === device)
-     *      settings: from configuration.yaml
-     *      name: name of the entity
-     *      definition: zigbee-herdsman-converters definition (only if type === device)
-     * }
-     */
-    resolveEntity(key) {
+        /* istanbul ignore next */
+        if (resolvedEntity && permit) {
+            /* istanbul ignore next */
+            await this.herdsman.permitJoin(permit, resolvedEntity.device, time);
+        } else {
+            await this.herdsman.permitJoin(permit, undefined, time);
+        }
+    }
+    /* eslint-disable-next-line */
+    resolveEntityLegacy(key: any): any {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
             key.constructor.name === 'Device' || key.constructor.name === 'Group' ||
@@ -189,19 +337,21 @@ class Zigbee extends events.EventEmitter {
             `Wrong type '${typeof key}'`,
         );
 
-        const getEndpointName = (endpointNames, endpoint) => {
+        /* eslint-disable-next-line */
+        const getEndpointName = (endpointNames: any, endpoint: any) => {
             return endpoint ?
                 utils.getKey(endpointNames, endpoint.ID, null, ((v) => v === 'default' ? null : v)) : null;
         };
 
         const deviceOptions = settings.get().device_options;
+        /* istanbul ignore else */
         if (typeof key === 'string' || typeof key === 'number') {
             if (typeof key === 'number') {
                 key = key.toString();
             }
 
             if (typeof key === 'string' && key.toLowerCase() === 'coordinator') {
-                const coordinator = this.getDevicesByType('Coordinator')[0];
+                const coordinator = this.getDevicesByTypeLegacy('Coordinator')[0];
                 return {
                     type: 'device',
                     device: coordinator,
@@ -212,7 +362,9 @@ class Zigbee extends events.EventEmitter {
                 };
             }
 
-            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            /* eslint-disable-next-line */
+            let endpointKey: any = utils.endpointNames.find((p) => key.endsWith(`/${p}`));
+            const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
             const endpointByNumber = key.match(keyEndpointByNumber);
             if (!endpointKey && endpointByNumber) {
                 endpointKey = Number(endpointByNumber[1]);
@@ -225,7 +377,8 @@ class Zigbee extends events.EventEmitter {
             if (!entity) {
                 return null;
             } else if (entity.type === 'device') {
-                const device = this.getDeviceByIeeeAddr(entity.ID);
+                /* eslint-disable-line */ // @ts-ignore
+                const device = this.herdsman.getDeviceByIeeeAddr(entity.ID);
                 if (!device) {
                     return null;
                 }
@@ -256,8 +409,10 @@ class Zigbee extends events.EventEmitter {
                     endpointName: getEndpointName(endpointNames, endpoint),
                 };
             } else {
-                let group = this.getGroupByID(entity.ID);
-                if (!group) group = this.createGroup(entity.ID);
+                /* eslint-disable-line */ // @ts-ignore
+                let group = this.getGroupByIDLegacy(entity.ID);
+                /* istanbul ignore if */ // @ts-ignore
+                if (!group) group = this.createGroupLegacy(entity.ID);
                 return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
             }
         } else if (key.constructor.name === 'Device' || key.constructor.name === 'Endpoint') {
@@ -287,59 +442,4 @@ class Zigbee extends events.EventEmitter {
             };
         }
     }
-
-    getGroupByID(ID) {
-        return this.herdsman.getGroupByID(ID);
-    }
-
-    getGroups() {
-        return this.herdsman.getGroups();
-    }
-
-    createGroup(groupID) {
-        return this.herdsman.createGroup(groupID);
-    }
-
-    acceptJoiningDeviceHandler(ieeeAddr) {
-        // If passlist is set, all devices not on passlist will be rejected to join the network
-        const passlist = settings.get().passlist.concat(settings.get().whitelist);
-        const blocklist = settings.get().blocklist.concat(settings.get().ban);
-        if (passlist.length > 0) {
-            if (passlist.includes(ieeeAddr)) {
-                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
-                return true;
-            } else {
-                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
-                return false;
-            }
-        } else if (blocklist.length > 0) {
-            if (blocklist.includes(ieeeAddr)) {
-                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
-                return false;
-            } else {
-                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
-                return true;
-            }
-        } else {
-            return true;
-        }
-    }
-
-    async touchlinkFactoryResetFirst() {
-        return this.herdsman.touchlinkFactoryResetFirst();
-    }
-
-    async touchlinkFactoryReset(ieeeAddr, channel) {
-        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
-    }
-
-    async touchlinkIdentify(ieeeAddr, channel) {
-        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
-    }
-
-    async touchlinkScan() {
-        return this.herdsman.touchlinkScan();
-    }
 }
-
-module.exports = Zigbee;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.18.3",
+  "version": "1.21.2",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^14 || ^15"
+    "node": "^10 || ^12 || ^14 || ^15 || ^16"
   },
   "keywords": [
     "xiaomi",
@@ -20,9 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "test-with-coverage": "jest test --coverage",
-    "eslint": "node_modules/.bin/eslint lib/",
+    "build": "tsc && node index.js writehash",
+    "build-watch": "tsc --watch",
+    "eslint": "eslint lib/ --max-warnings=0",
     "start": "node index.js",
+    "test-with-coverage": "jest test --coverage",
     "test": "jest test",
     "test-watch": "jest test --watch"
   },
@@ -33,35 +35,53 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^6.12.4",
-    "debounce": "^1.2.0",
+    "ajv": "^8.6.3",
+    "bind-decorator": "^1.0.11",
+    "debounce": "^1.2.1",
     "deep-object-diff": "^1.1.0",
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.1.2",
-    "git-last-commit": "^1.0.0",
-    "humanize-duration": "^3.23.1",
-    "js-yaml": "^3.14.0",
-    "json-stable-stringify-without-jsonify": "=1.0.1",
+    "git-last-commit": "^1.0.1",
+    "humanize-duration": "^3.27.0",
+    "js-yaml": "^4.1.0",
+    "json-stable-stringify-without-jsonify": "^1.0.1",
     "mkdir-recursive": "^0.4.0",
-    "moment": "^2.27.0",
-    "mqtt": "4.2.1",
+    "moment": "^2.29.1",
+    "mqtt": "4.2.8",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
-    "semver": "^7.3.2",
+    "semver": "^7.3.5",
     "serve-static": "^1.14.1",
+    "source-map-support": "^0.5.20",
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
-    "ws": "^7.3.1",
-    "zigbee-herdsman": "0.13.92",
-    "zigbee-herdsman-converters": "14.0.131",
-    "zigbee2mqtt-frontend": "0.4.0"
+    "ws": "^8.2.2",
+    "zigbee-herdsman": "0.13.149",
+    "zigbee-herdsman-converters": "14.0.261",
+    "zigbee2mqtt-frontend": "0.6.12"
   },
   "devDependencies": {
-    "eslint": "*",
-    "eslint-config-google": "*",
-    "eslint-plugin-jest": "*",
-    "jest": "*",
-    "tmp": "*"
+    "@babel/core": "^7.15.5",
+    "@babel/plugin-proposal-decorators": "^7.15.4",
+    "@babel/preset-env": "^7.15.6",
+    "@babel/preset-typescript": "^7.15.0",
+    "@types/debounce": "^1.2.1",
+    "@types/finalhandler": "^1.1.1",
+    "@types/humanize-duration": "^3.25.1",
+    "@types/jest": "^27.0.2",
+    "@types/js-yaml": "^4.0.3",
+    "@types/object-assign-deep": "^0.4.0",
+    "@types/serve-static": "^1.13.10",
+    "@types/ws": "^8.2.0",
+    "@typescript-eslint/eslint-plugin": "^4.31.2",
+    "@typescript-eslint/parser": "^4.31.2",
+    "babel-jest": "^27.2.2",
+    "eslint": "^7.32.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-jest": "^24.4.2",
+    "jest": "^27.2.2",
+    "tmp": "^0.2.1",
+    "typescript": "^4.4.3"
   },
   "jest": {
     "coverageThreshold": {
@@ -73,7 +93,8 @@
       }
     },
     "collectCoverageFrom": [
-      "lib/**/*.js"
+      "lib/**/*.js",
+      "lib/**/*.ts"
     ]
   },
   "bin": {

scripts/zStackEraseAllNvMem.js

@@ -64,11 +64,11 @@ class ZStackNvMemEraser {
                 if (needOsal) {
                     await this.znp.request(Subsystem.SYS, 'osalNvDelete',
                         {id: id, len: len},
-                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                        null, null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
                 } else {
                     await this.znp.request(Subsystem.SYS, 'nvDelete',
                         {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0},
-                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                        null, null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
                 }
                 deletedCount++;
             }

scripts/zigbee2socat_installer.sh

@@ -53,7 +53,7 @@ function handle_opts {
 
         -v|--verbose)
             verbose=1; shift; continue ;;
-        
+
         -m|--man)
             makemanual ;;
 
@@ -84,7 +84,6 @@ port=""
 function main {
     handle_opts "$@"
 
-    # TODO: Check IP-address valid?
     if [[ "$ip" = "" ]]; then
         echo "IP-address not specified"
         exit 5
@@ -161,7 +160,7 @@ function usage {
     -v, --verbose       Print more information
     -m, --man           Make and display manual
     -h, --help          Display this help message
-    
+
     \rOriginal concept by JFLN\
     "
     exit 0

test/availability.test.js

@@ -6,38 +6,45 @@ zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
-
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45535');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45521');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45525');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-
-const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const mocks = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Availability', () => {
     let controller;
+    let extension;
 
-    function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'AvailabilityLegacy');
+        await controller.enableDisableExtension(true, 'AvailabilityLegacy');
+        extension = controller.extensions.find((e) => e.constructor.name === 'AvailabilityLegacy');
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
         jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10);
         controller = new Controller(jest.fn(), jest.fn());
-        mocksClear.forEach((m) => m.mockClear());
         await controller.start();
         await flushPromises();
     });
 
-    afterEach(async () => {
-        await controller.stop();
-        await flushPromises();
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['advanced', 'availability_timeout'], 10);
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     })
 
     it('Should publish availabilty on startup', async () => {
@@ -69,7 +76,6 @@ describe('Availability', () => {
         device.ping.mockImplementationOnce(() => {throw new Error('failed')});
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
-        expect(logger.debug).toHaveBeenCalledTimes(3);
         expect(logger.debug).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
@@ -153,7 +159,7 @@ describe('Availability', () => {
 
     it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
         MQTT.publish.mockClear();
-        getExtension().state = {};
+        extension.state = {};
         const payload = {device: zigbeeHerdsman.devices.bulb_color};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
@@ -199,11 +205,7 @@ describe('Availability', () => {
     it('Should not ping devices on blocklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -213,11 +215,7 @@ describe('Availability', () => {
     it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -227,11 +225,7 @@ describe('Availability', () => {
     it('Should not ping devices on blocklist by IEEE address', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -254,11 +248,7 @@ describe('Availability', () => {
     it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -268,11 +258,7 @@ describe('Availability', () => {
     it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -282,11 +268,7 @@ describe('Availability', () => {
     it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -295,13 +277,10 @@ describe('Availability', () => {
 
     it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        getExtension().state[device.ieeeAddr] = false;
+        device.ping.mockClear();
+        extension.state[device.ieeeAddr] = false;
         settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(0);
@@ -311,7 +290,7 @@ describe('Availability', () => {
 
     it('Should not read when device has no modelID and reconnects', async () => {
         const device = zigbeeHerdsman.devices.nomodel;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -320,7 +299,7 @@ describe('Availability', () => {
 
     it('Should not read when device has is unsupported', async () => {
         const device = zigbeeHerdsman.devices.unsupported_router;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -341,6 +320,7 @@ describe('Availability', () => {
     });
 
     it('Should publish availability when end device joins', async () => {
+        delete extension.state[zigbeeHerdsman.devices.WXKG02LM_rev1.ieeeAddr];
         const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const payload = {device};
         MQTT.publish.mockClear();
@@ -361,10 +341,9 @@ describe('Availability', () => {
         device.lastSeen = Date.now();
         MQTT.publish.mockClear();
         jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
         device.lastSeen = device.lastSeen - (1000 * 60 * 60 * 25);
         jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote/availability',
           'offline',
@@ -375,20 +354,16 @@ describe('Availability', () => {
         // Shouldn't do anything more when device is removed
         settings.removeDevice(device.ieeeAddr);
         jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
 
         device.lastSeen = defaultLastSeen;
     });
 
     it('Should republish existing state on MQTT connected', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        await controller.stop();
-        await flushPromises();
         MQTT.publish.mockClear();
-        controller = new Controller(jest.fn(), jest.fn());
-        getExtension().state[device.ieeeAddr] = false;
-        await controller.start();
-        await flushPromises();
+        extension.state[device.ieeeAddr] = false;
+        await extension.onMQTTConnected();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color/availability',
           'offline',

test/availabilityNew.test.ts

@@ -0,0 +1,290 @@
+import data from './stub/data';
+import logger from './stub/logger';
+import MQTT from './stub/mqtt';
+import zigbeeHerdsman from './stub/zigbeeHerdsman';
+
+import * as utils from '../lib/util/utils';
+import * as settings from '../lib/util/settings';
+import Controller from '../lib/controller';
+import flushPromises from './lib/flushPromises';
+import stringify from 'json-stable-stringify-without-jsonify';
+
+const mocks = [MQTT.publish, logger.warn, logger.debug];
+const devices = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(
+    ...[devices.bulb_color.ieeeAddr, devices.bulb_color_2.ieeeAddr, devices.coordinator.ieeeAddr, devices.remote.ieeeAddr])
+
+describe('Availability', () => {
+    let controller;
+
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Availability');
+        await controller.enableDisableExtension(true, 'Availability');
+    }
+
+    const advancedTime = async (value) => {
+        jest.setSystemTime(Date.now() + value);
+        jest.advanceTimersByTime(value);
+        await flushPromises();
+    }
+
+    beforeAll(async () => {
+        jest.spyOn(utils, 'sleep').mockImplementation(async (seconds: number) => {});
+        jest.useFakeTimers('modern');
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['experimental', 'availability_new'], true);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        jest.useFakeTimers('modern').setSystemTime(utils.minutes(1));
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['experimental', 'availability_new'], true);
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], false);
+        // @ts-ignore
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+        // @ts-ignore
+        Object.values(devices).forEach((d) => d.ping.mockClear());
+    });
+
+    afterEach(async () => {
+        // @ts-ignore
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+    })
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should publish availabilty on startup for device where it is enabled for', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bulb_color_2/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish offline for active device when not seen for 10 minutes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Shouldnt do anything for a device when availability: false is set for device', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2}); // Coverage satisfaction
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish offline for passive device when not seen for 25 hours', async () => {
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for active device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should ping again when first ping fails', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        devices.bulb_color.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        devices.bulb_color.lastSeen = Date.now() + utils.minutes(10);
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(2);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(2, false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for passive device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.hours(24));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.remote});
+
+        await advancedTime(utils.hours(25));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        devices.remote.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        await advancedTime(utils.hours(3));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should immediately mark device as online when it lastSeen changes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(15));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+
+        devices.bulb_color.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should allow to change availability timeout via device options', async () => {
+        settings.set(['devices', '0x000b57fffec6a5b3', 'availability'], {timeout: 40});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(17));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to change availability timeout via avaiability options', async () => {
+        settings.set(['availability'], {active: {timeout: 30}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should stop pinging device when it leaves', async () => {
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(9));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: devices.bulb_color.ieeeAddr});
+        await flushPromises();
+
+        await advancedTime(utils.minutes(3));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to be disabled', async () => {
+        settings.set(['availability'], false);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should retrieve device state when it reconnects', async () => {
+        MQTT.publish.mockClear();
+
+        const endpoint = devices.bulb_color.getEndpoint(1);
+        endpoint.read.mockClear();
+
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        await advancedTime(utils.seconds(1));
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+        await advancedTime(utils.seconds(2));
+
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
+            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);
+
+        // Should stop when one request fails
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        endpoint.read.mockImplementationOnce(() => {throw new Error('')});
+        await advancedTime(utils.seconds(3));
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should republish availability when device is renamed', async () => {
+        MQTT.publish.mockClear();
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from: 'bulb_color', to: 'bulb_new_name'}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_new_name/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        await advancedTime(utils.hours(12));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_new_name/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
+        settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Deprecated - should allow to pass certain devices via availability_passlist', async () => {
+        settings.set(['advanced', 'availability_passlist'], [devices.bulb_color_2.ieeeAddr]);
+        settings.changeEntityOptions(devices.bulb_color_2.ieeeAddr, {availability: null});
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Deprecated - should allow to enable via availability_timeout', async () => {
+        settings.set(['availability'], false);
+        settings.set(['advanced', 'availability_timeout'], 60);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+});

test/bind.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const stringify = require('json-stable-stringify-without-jsonify');
 jest.mock('debounce', () => jest.fn(fn => fn));
 const debounce = require('debounce');
@@ -12,7 +12,7 @@ const debounce = require('debounce');
 describe('Bind', () => {
     let controller;
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -23,22 +23,34 @@ describe('Bind', () => {
         }
     }
 
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Bind');
+        await controller.enableDisableExtension(true, 'Bind');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
         zigbeeHerdsman.groups.group_1.members = [];
         zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
         zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
         zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
         debounce.mockClear();
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
         MQTT.publish.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Should bind to device and configure reporting', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
@@ -229,6 +241,52 @@ describe('Bind', () => {
         );
     });
 
+    it('Should unbind from group with skip_disable_reporting=true', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set to false, we don't expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(0);
+        endpoint.binds = originalBinds;
+    });
+
+    it('Should unbind from group with skip_disable_reporting=false', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: false}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set, we expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 65535, "minimumReportInterval": 5, "reportableChange": 1}])
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 65535, "minimumReportInterval": 0, "reportableChange": 0}])
+        endpoint.binds = originalBinds;
+    });
+
     it('Should bind to group by number', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
@@ -527,6 +585,19 @@ describe('Bind', () => {
         expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
     });
 
+    it('Should poll bounded Hue bulb when receiving message from scene controller', async () => {
+        const remote = zigbeeHerdsman.devices.bj_scene_switch;
+        const data = {"action": "recall_2_row_1"};
+        const payload = {data, cluster: 'genScenes', device: remote, endpoint: remote.getEndpoint(10), type: 'commandRecall', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        // Calls to three clusters are expected in this case
+        expect(debounce).toHaveBeenCalledTimes(3);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+	expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]);
+    });
+
     it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {
         zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
         zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read.mockClear();

test/configure.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const stringify = require('json-stable-stringify-without-jsonify');
 
@@ -12,13 +12,14 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Configure', () => {
     let controller;
+    let coordinatorEndpoint;
 
-    expectRemoteConfigured = () => {
+    const expectRemoteConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(2);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
 
         const endpoint2 = device.getEndpoint(2);
         expect(endpoint2.write).toHaveBeenCalledTimes(1);
@@ -26,29 +27,27 @@ describe('Configure', () => {
         expect(device.meta.configured).toBe(1);
     }
 
-    expectBulbConfigured = () => {
+    const expectBulbConfigured = () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint1 = device.getEndpoint(1);
-        console.log(endpoint1.read.mock.calls);
         expect(endpoint1.read).toHaveBeenCalledTimes(2);
         expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities']);
         expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorTempPhysicalMin', 'colorTempPhysicalMax' ]);
     }
 
-    expectBulbNotConfigured = () => {
+    const expectBulbNotConfigured = () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint1 = device.getEndpoint(1);
-        console.log(endpoint1.read.mock.calls);
         expect(endpoint1.read).toHaveBeenCalledTimes(0);
     }
 
-    expectRemoteNotConfigured = () => {
+    const expectRemoteNotConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(0);
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -57,18 +56,30 @@ describe('Configure', () => {
         }
     }
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Configure');
+        await controller.enableDisableExtension(true, 'Configure');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
-        mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
     });
 
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+     coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Should configure Router on startup', async () => {
         expectBulbConfigured();
     });
@@ -102,10 +113,8 @@ describe('Configure', () => {
     it('Should not configure twice', async () => {
         expectBulbConfigured();
         const device = zigbeeHerdsman.devices.bulb;
-        const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expectBulbNotConfigured();
     });
@@ -113,10 +122,8 @@ describe('Configure', () => {
     it('Should configure on zigbee message when not configured yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         delete device.meta.configured;
-        const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expectBulbConfigured();
     });
@@ -156,11 +163,11 @@ describe('Configure', () => {
     });
 
     it('Fail to configure via MQTT when device has no configure', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb_enddevice", transaction: 20}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "0x90fd9ffffe4b64ax", transaction: 20}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            stringify({"data":{"id": "bulb_enddevice"},"status":"error","error": "Device 'bulb_enddevice' cannot be configured","transaction":20}),
+            stringify({"data":{"id": "0x90fd9ffffe4b64ax"},"status":"error","error": "Device 'ZNLDP12LM' cannot be configured","transaction":20}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -180,9 +187,9 @@ describe('Configure', () => {
     });
 
     it('Legacy api: Should skip reconfigure when device does not require this', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x0017880104e45553');
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x90fd9ffffe4b64ax');
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb_enddevice', device does not require this.`)
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
     it('Should not configure when interviewing', async () => {
@@ -191,8 +198,7 @@ describe('Configure', () => {
         device.interviewing = true;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expectRemoteNotConfigured();
         device.interviewing = false;
@@ -204,8 +210,7 @@ describe('Configure', () => {
         device.interviewCompleted = false;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expectRemoteConfigured();
         device.interviewCompleted = true;
@@ -217,11 +222,10 @@ describe('Configure', () => {
         const endpoint = device.getEndpoint(1);
         endpoint.bind.mockImplementationOnce(async () => await wait(500));
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
     });
@@ -233,20 +237,19 @@ describe('Configure', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
     });

test/controller.test.js

@@ -6,7 +6,7 @@ const path = require('path');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const tmp = require('tmp');
 const mocksClear = [
     zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
@@ -20,6 +20,10 @@ describe('Controller', () => {
     let controller;
     let mockExit;
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
         mockExit = jest.fn();
@@ -30,9 +34,13 @@ describe('Controller', () => {
         data.writeDefaultState();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -43,7 +51,7 @@ describe('Controller', () => {
         expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
         expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
-        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state", "qos": 1}});
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}),{ retain: true, qos: 0 }, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', stringify({"brightness":255}), { retain: true, qos: 0 }, expect.any(Function));
     });
@@ -79,7 +87,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         const expected = {
-            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"},
+            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state", "qos": 1},
             keepalive: 30,
             ca: Buffer.from([99, 97]),
             key: Buffer.from([107, 101, 121]),
@@ -132,14 +140,12 @@ describe('Controller', () => {
     });
 
     it('Log when MQTT client is unavailable', async () => {
-        jest.useFakeTimers();
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
         controller.mqtt.client.reconnecting = true;
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
         controller.mqtt.client.reconnecting = false;
     });
@@ -609,7 +615,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         const expected = {
-            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state", "qos": 1 },
         }
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
@@ -623,4 +629,13 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
     });
+
+    it('Should disable legacy options on new network start', async () => {
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        expect(settings.get().advanced.legacy_api).toBeTruthy();
+        zigbeeHerdsman.start.mockReturnValueOnce('reset');
+        await controller.start();
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().advanced.legacy_api).toBeFalsy();
+    });
 });

test/data.test.js

@@ -1,5 +1,5 @@
 const logger = require('./stub/logger');
-const data = require('../lib/util/data.js');
+const data = require('../lib/util/data');
 const path = require('path');
 const tmp = require('tmp');
 const fs = require('fs');
@@ -15,12 +15,12 @@ describe('Data', () => {
         it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
             const expected = tmp.dirSync().name;
             process.env.ZIGBEE2MQTT_DATA = expected;
-            data._reload();
+            data.__testingOnly_reload();
             const actual = data.getPath();
             expect(actual).toBe(expected);
             expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
             delete process.env.ZIGBEE2MQTT_DATA;
-            data._reload();
+            data.__testingOnly_reload();
         });
     });
 });

test/externalConverters.test.js

@@ -6,7 +6,7 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const path = require('path');
 const fs = require('fs');
 
@@ -41,20 +41,32 @@ jest.mock(
 describe('Loads external converters', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'ExternalConverters');
+        await controller.enableDisableExtension(true, 'ExternalConverters');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
         await flushPromises();
-        mocksClear.forEach((m) => m.mockClear());
+    });
+
+    beforeEach(async () => {
         data.writeDefaultConfiguration();
-        data.writeEmptyState();
         settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Does not load external converters', async () => {
         settings.set(['external_converters'], []);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
     });
 
@@ -62,9 +74,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter.js']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
             mock: true,
@@ -82,9 +92,7 @@ describe('Loads external converters', () => {
         fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
         const devicesCount = zigbeeHerdsman.devices.lenght;
         settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1,
@@ -110,9 +118,7 @@ describe('Loads external converters', () => {
 
     it('Loads external converters from package', async () => {
         settings.set(['external_converters'], ['mock-external-converter-module']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
             mock: true
@@ -121,9 +127,7 @@ describe('Loads external converters', () => {
 
     it('Loads multiple external converters from package', async () => {
         settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
         expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
             mock: 1

test/externalExtension.test.js

@@ -3,11 +3,11 @@ const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const path = require('path');
+const rimraf = require('rimraf');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
-const tmp = require('tmp');
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [
     zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
     MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
@@ -20,12 +20,24 @@ const unlinkSyncSpy = jest.spyOn(fs, 'unlinkSync');
 
 describe('User extensions', () => {
     let controller;
-    let mockExit;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
 
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
-        mockExit = jest.fn();
-        controller = new Controller(jest.fn(), mockExit);
+        controller = new Controller(jest.fn(), jest.fn());
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
         settings.reRead();
@@ -33,7 +45,7 @@ describe('User extensions', () => {
     });
     afterEach(() => {
         const extensionPath = path.join(data.mockDir, 'extension');
-        fs.rmdirSync(extensionPath, {recursive: true});
+        rimraf.sync(extensionPath);
     })
 
     it('Load user extension', async () => {

test/frontend.test.js

@@ -5,7 +5,7 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 jest.spyOn(process, 'exit').mockImplementation(() => {});
 
@@ -31,6 +31,7 @@ const mockWS = {
             cb(mockWSocket)
         }),
         emit: jest.fn(),
+        close: jest.fn(),
     },
     variables: {},
     events: {},
@@ -70,6 +71,10 @@ jest.mock('ws', () => ({
 describe('Frontend', () => {
     let controller;
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
     beforeEach(async () => {
         mockWS.implementation.clients = [];
         data.writeDefaultConfiguration();
@@ -80,6 +85,10 @@ describe('Frontend', () => {
         zigbeeHerdsman.devices.bulb.linkquality = 10;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     afterEach(async() => {
         delete zigbeeHerdsman.devices.bulb.linkquality;
     });
@@ -92,15 +101,16 @@ describe('Frontend', () => {
 
         const mockWSClient = {
             implementation: {
-                close: jest.fn(),
+                terminate: jest.fn(),
                 send: jest.fn(),
             },
             events: {},
         };
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await controller.stop();
-        expect(mockWSClient.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockWSClient.implementation.terminate).toHaveBeenCalledTimes(1);
         expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockWS.implementation.close).toHaveBeenCalledTimes(1);
     });
 
     it('Websocket interaction', async () => {
@@ -120,33 +130,33 @@ describe('Frontend', () => {
         await mockWS.events.connection(mockWSClient.implementation);
 
         expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bridge/state', payload: 'online'}));
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255}}));
 
         // Message
         MQTT.publish.mockClear();
         mockWSClient.implementation.send.mockClear();
-        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}), false)
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color',
             stringify({state: 'ON', linkquality: null}),
             { retain: false, qos: 0 },
             expect.any(Function)
         );
-        mockWSClient.events.message(undefined);
-        mockWSClient.events.message("");
-        mockWSClient.events.message(null);
+        mockWSClient.events.message(undefined, false);
+        mockWSClient.events.message("", false);
+        mockWSClient.events.message(null, false);
         await flushPromises();
 
         // Received message on socket
-        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(4);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(1);
         expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', linkquality: null}}));
 
         // Shouldnt set when not ready
         mockWSClient.implementation.send.mockClear();
         mockWSClient.implementation.readyState = 'close';
-        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}), false)
         expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(0);
 
         // Send last seen on connect
@@ -155,7 +165,7 @@ describe('Frontend', () => {
         settings.set(['advanced'], {last_seen: 'ISO_8601'});
         mockWS.implementation.clients.push(mockWSClient.implementation);
         await mockWS.events.connection(mockWSClient.implementation);
-        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z"}}));
     });
 
     it('onReques/onUpgrade', async () => {

test/group.test.js

@@ -13,28 +13,42 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45724');
 
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const settings = require('../lib/util/settings');
 
 describe('Groups', () => {
     let controller;
 
-    beforeEach(() => {
-        data.writeEmptyState();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Groups');
+        await controller.enableDisableExtension(true, 'Groups');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(() => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
         settings.reRead();
         MQTT.publish.mockClear();
         zigbeeHerdsman.groups.gledopto_group.command.mockClear();
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+        controller.state.state = {};
     })
 
     it('Apply group updates add', async () => {
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
         zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -46,8 +60,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
@@ -58,8 +71,7 @@ describe('Groups', () => {
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
         logger.error.mockClear();
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(logger.error).toHaveBeenCalledWith(`Failed to remove 'bulb_color' from 'group_1'`);
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([endpoint]);
     });
@@ -70,32 +82,28 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
     it('Add non standard endpoint to group with name', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
     });
 
     it('Add non standard endpoint to group with number', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
     });
 
     it('Shouldnt crash on non-existing devices', async () => {
         logger.error.mockClear();
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
@@ -103,8 +111,7 @@ describe('Groups', () => {
     it('Should resolve device friendly names', async () => {
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'friendly_name'], 'bulb_friendly_name');
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb_friendly_name', 'bulb_color']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -117,8 +124,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -132,8 +138,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -146,8 +151,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
@@ -159,8 +163,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
@@ -175,8 +178,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -190,8 +192,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -204,8 +205,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -218,8 +218,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -232,8 +231,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -246,8 +244,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -261,8 +258,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
@@ -270,8 +266,7 @@ describe('Groups', () => {
     });
 
     it('Legacy api: Log when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
         await flushPromises();
@@ -279,8 +274,7 @@ describe('Groups', () => {
     });
 
     it('Legacy api: Log when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
         await flushPromises();
@@ -293,8 +287,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -310,18 +303,19 @@ describe('Groups', () => {
         const device1 = zigbeeHerdsman.devices.bulb_2;
         const device2 = zigbeeHerdsman.devices.bulb_color_2;
         const group = zigbeeHerdsman.groups.group_tradfri_remote;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledTimes(6);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/ha_discovery_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/switch_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -330,8 +324,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
@@ -348,8 +341,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
@@ -375,8 +367,7 @@ describe('Groups', () => {
 
     it('Should publish state of device with endpoint name', async () => {
         const group = zigbeeHerdsman.groups.gledopto_group;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/gledopto_group/set', stringify({state: 'ON'}));
@@ -390,8 +381,7 @@ describe('Groups', () => {
 
     it('Should publish state of group when specific state of specific endpoint is changed', async () => {
         const group = zigbeeHerdsman.groups.gledopto_group;
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/GLEDOPTO_2ID/set', stringify({state_cct: 'ON'}));
@@ -408,8 +398,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
@@ -425,8 +414,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -447,8 +435,7 @@ describe('Groups', () => {
             '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
             '3': {friendly_name: 'group_3', retain: false, devices: []}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
@@ -470,8 +457,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -495,8 +481,7 @@ describe('Groups', () => {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
             '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -520,8 +505,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -547,8 +531,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
 
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF', color_temp: 200}));
@@ -559,9 +542,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 300}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode": "color_temp", "color":{"x":0.415211980162654,"y":0.395434886759171},"color_temp":300,"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode": "color_temp", "color":{"x":0.415211980162654,"y":0.395434886759171},"color_temp":300,"state":"ON"}), {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode": "color_temp", "color":{"x":0.415211980162654,"y":0.395434886759171},"color_temp":300,"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode": "color_temp","color_temp":300,"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -575,8 +558,7 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -597,8 +579,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({transaction: "123", group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -618,7 +599,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
+        await resetExtension();
         endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
         await flushPromises();
         MQTT.publish.mockClear();
@@ -640,8 +621,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
         await flushPromises();
@@ -660,8 +640,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
@@ -680,8 +659,7 @@ describe('Groups', () => {
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
         await flushPromises();
@@ -703,8 +681,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -718,14 +695,33 @@ describe('Groups', () => {
         );
     });
 
+    it('Remove from group via MQTT keeping device reporting', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
         await flushPromises();
@@ -745,8 +741,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
         await flushPromises();
@@ -766,8 +761,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
         await flushPromises();
@@ -787,8 +781,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
         await flushPromises();
@@ -808,8 +801,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
         await flushPromises();
@@ -824,8 +816,7 @@ describe('Groups', () => {
     });
 
     it('Error when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
@@ -839,8 +830,7 @@ describe('Groups', () => {
     });
 
     it('Error when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         logger.error.mockClear();
         MQTT.publish.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
@@ -852,4 +842,30 @@ describe('Groups', () => {
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
+
+    it('Should only include relevant properties when publishing member states', async () => {
+        const bulbColor = zigbeeHerdsman.devices.bulb_color;
+        const bulbColorTemp = zigbeeHerdsman.devices.bulb;
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(bulbColor.getEndpoint(1));
+        group.members.push(bulbColorTemp.getEndpoint(1));
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [bulbColor.ieeeAddr, bulbColorTemp.ieeeAddr]}});
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 50}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color: {x: 0.5, y: 0.3}}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":548}), {"retain": true, qos: 0}, expect.any(Function));
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -7,14 +7,16 @@ const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 
 
 describe('Bridge legacy', () => {
     let controller;
+    let version;
 
     beforeAll(async () => {
-        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
+        jest.useFakeTimers();
+        version = await require('../../lib/util/utils').getZigbee2MQTTVersion();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
     })
@@ -22,15 +24,18 @@ describe('Bridge legacy', () => {
     beforeEach(() => {
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeDefaultState();
         logger.info.mockClear();
         logger.warn.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should publish bridge configuration on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/config',
-          stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
+          stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -188,7 +193,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","optimistic":true,"retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","optimistic":true,"retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","optimistic":true,"retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","optimistic":true,"retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","optimistic":true,"retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group","optimistic":true},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","optimistic":true,"retain":false}],"type":"groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","retain":false},{"ID":9,"devices":["bulb_color_2","bulb_2","wall_switch_double/right"],"friendly_name":"ha_discovery_group"},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},{"ID":14,"devices":["power_plug","bulb_2"],"friendly_name":"switch_group","retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group"},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {
@@ -221,10 +226,10 @@ describe('Bridge legacy', () => {
 
     it('Should allow rename groups', async () => {
         MQTT.publish.mockClear();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", retain: false});
         MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
         await flushPromises();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", retain: false});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
             stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
@@ -270,7 +275,7 @@ describe('Bridge legacy', () => {
             {qos: 0, retain: false},
             expect.any(Function)
         );
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
     });
@@ -279,7 +284,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -294,7 +299,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
         expect(MQTT.publish).toHaveBeenCalledWith(
@@ -309,7 +314,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
         await flushPromises();
-        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
     });

test/legacy/report.test.js

@@ -13,13 +13,14 @@ zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
 const MQTT = require('../stub/mqtt');
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Report', () => {
     let controller;
+    let extension;
 
     function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
@@ -59,7 +60,7 @@ describe('Report', () => {
         }
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -69,23 +70,34 @@ describe('Report', () => {
         }
     }
 
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        settings.set(['advanced', 'report'], true);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        extension = controller.extensions.find((e) => e.constructor.name === 'Report');
+    });
+
     beforeEach(async () => {
+        extension.enabled = true;
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
-        settings.set(['advanced', 'report'], true);
         for (const device of Object.values(zigbeeHerdsman.devices)) {
             mockClear(device);
             delete device.meta.reporting;
         }
-
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
         mocksClear.forEach((m) => m.mockClear());
-        await flushPromises();
+        extension.queue = new Set();
+        extension.failed = new Set();
+        await extension.onZigbeeStarted();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should configure reporting on startup', async () => {
+        await extension.onZigbeeStarted();
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         expectOnOffBrightnessColorReport(endpoint, true);
@@ -96,10 +108,8 @@ describe('Report', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         delete device.meta.report;
-        settings.set(['advanced', 'report'], false);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        extension.enabled = false;
+        await extension.onZigbeeStarted();
         expect(device.meta.reporting).toBe(undefined);
         expect(endpoint.bind).toHaveBeenCalledTimes(0);
     });
@@ -108,11 +118,9 @@ describe('Report', () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         device.meta.reporting = 1;
         const endpoint = device.getEndpoint(1);
-        settings.set(['advanced', 'report'], false);
+        extension.enabled = false;
         mockClear(device);
-        controller = new Controller(jest.fn(), jest.fn());
-        await controller.start();
-        await flushPromises();
+        await extension.onZigbeeStarted();
         expectOnOffBrightnessColorReportDisabled(endpoint, true);
     });
 
@@ -131,7 +139,6 @@ describe('Report', () => {
     });
 
     it('Should not configure reporting when still configuring', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         endpoint.bind.mockImplementationOnce(async () => await wait(1000));
@@ -143,8 +150,6 @@ describe('Report', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
-        jest.useRealTimers();
     });
 
     it('Should not mark as configured when reporting setup fails', async () => {

test/lib/flushPromises.js

@@ -0,0 +1,2 @@
+const globalSetImmediate = setImmediate;
+module.exports = () => new Promise(globalSetImmediate);

test/networkMap.test.js

@@ -16,9 +16,7 @@ zigbeeHerdsman.returnDevices.push(external_converter_device.ieeeAddr)
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
-Date.now = jest.fn()
-Date.now.mockReturnValue(10000);
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 const setTimeoutNative = setTimeout;
 
@@ -26,6 +24,9 @@ describe('Networkmap', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
+        Date.now = jest.fn()
+        Date.now.mockReturnValue(10000);
         data.writeDefaultConfiguration();
         settings.reRead();
         data.writeEmptyState();
@@ -50,6 +51,10 @@ describe('Networkmap', () => {
         global.setTimeout = setTimeoutNative;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     function mock() {
         /**
          * Topology
@@ -97,7 +102,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -137,18 +142,18 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|0 seconds ago}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
-              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
-              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0x0000)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (0x9db1)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
+              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 0x198c)"]
+              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (0x9dcf)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (0x198a)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (0x1988)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (0x198c)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
-              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (1114)|external external (external_converter_device)|9 seconds ago}"];
+              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (0x045a)|external external (external_converter_device)|9 seconds ago}"];
               "0x0017880104e45511" -> "0x00124b00120144ae" [penwidth=1, weight=0, color="#994444", label="92"]
             }`;
 
@@ -180,7 +185,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45511 [
         0x0017880104e45511
         ---
-        0x0017880104e45511 (1114)
+        0x0017880104e45511 (0x045a)
         ---
         external external (external_converter_device)
         ---
@@ -190,7 +195,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45525 [
         0x0017880104e45525
         ---
-        0x0017880104e45525 (6536) failed: lqi,routingTable
+        0x0017880104e45525 (0x1988) failed: lqi,routingTable
         ---
         Boef notSupportedModelID
         ---
@@ -200,7 +205,7 @@ describe('Networkmap', () => {
         card 0x000b57fffec6a5b2 [
         bulb
         ---
-        0x000b57fffec6a5b2 (40369)
+        0x000b57fffec6a5b2 (0x9db1)
         ---
         IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
         ---
@@ -210,7 +215,7 @@ describe('Networkmap', () => {
         card 0x000b57fffec6a5b3 [
         bulb_color
         ---
-        0x000b57fffec6a5b3 (40399)
+        0x000b57fffec6a5b3 (0x9dcf)
         ---
         Philips Hue Go (7146060PH)
         ---
@@ -220,7 +225,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45521 [
         button_double_key
         ---
-        0x0017880104e45521 (6538)
+        0x0017880104e45521 (0x198a)
         ---
         Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)
         ---
@@ -230,7 +235,7 @@ describe('Networkmap', () => {
         card 0x0017880104e45559 [
         cc2530_router
         ---
-        0x0017880104e45559 (6540)
+        0x0017880104e45559 (0x198c)
         ---
         Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
         ---
@@ -240,7 +245,7 @@ describe('Networkmap', () => {
         card 0x00124b00120144ae [
         Coordinator
         ---
-        0x00124b00120144ae (0)
+        0x00124b00120144ae (0x0000)
         ---
         0 seconds ago
         ]
@@ -271,7 +276,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
 
-        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
         const actual = JSON.parse(call[1]);
         expect(actual).toStrictEqual(expected);
     });

test/onEvent.test.js

@@ -6,7 +6,7 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45560');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
@@ -22,14 +22,24 @@ describe('On event', () => {
     const device = zigbeeHerdsman.devices.LIVOLO;
 
     beforeEach(async () => {
+        jest.useFakeTimers();
         data.writeDefaultConfiguration();
         settings.reRead();
-        data.writeEmptyState();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         zigbeeHerdsmanConverters.onEvent.mockClear();
-        await flushPromises();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should call with start event', async () => {
@@ -56,10 +66,7 @@ describe('On event', () => {
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
         expect(mockOnEvent).toHaveBeenCalledTimes(1);
-        const call = mockOnEvent.mock.calls[0];
-        expect(call[0]).toBe('deviceAnnounce')
-        expect(call[1]).toStrictEqual({device})
-        expect(call[2]).toBe(device);
+        expect(mockOnEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr));
     });
 
     it('Should call index onEvent with zigbee event', async () => {
@@ -67,9 +74,6 @@ describe('On event', () => {
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
         expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledTimes(1);
-        const call = zigbeeHerdsmanConverters.onEvent.mock.calls[0];
-        expect(call[0]).toBe('deviceAnnounce')
-        expect(call[1]).toStrictEqual({device})
-        expect(call[2]).toBe(device);
+        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledWith('deviceAnnounce', {device}, device, settings.getDevice(device.ieeeAddr));
     });
 });

test/otaUpdate.test.js

@@ -4,26 +4,44 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('OTA update', () => {
     let controller;
 
-    mockClear = (mapped) => {
+    const mockClear = (mapped) => {
         mapped.ota.updateToLatest = jest.fn();
         mapped.ota.isUpdateAvailable = jest.fn();
     }
 
-    beforeEach(async () => {
+    beforeAll(async () => {
         data.writeDefaultConfiguration();
         settings.reRead();
+        data.writeDefaultConfiguration();
         settings.set(['advanced', 'ikea_ota_use_test_url'], true);
-        data.writeEmptyState();
+        settings.reRead();
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
+    });
+
+    afterEach(async () => {
+        settings.set(['ota', 'disable_automatic_update_check'], false);
+        jest.runOnlyPendingTimers();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(async () => {
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        extension.inProgress = new Set();
+        controller.state.state = {};
         MQTT.publish.mockClear();
     });
 
@@ -176,7 +194,6 @@ describe('OTA update', () => {
     });
 
     it('Should refuse to check/update when already in progress', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -189,7 +206,7 @@ describe('OTA update', () => {
         MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/ota_update/check',
@@ -201,7 +218,6 @@ describe('OTA update', () => {
     it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
-        let count = 0;
         endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
 
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
@@ -217,6 +233,7 @@ describe('OTA update', () => {
 
     it('Should check for update when device requests it', async () => {
         const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
         const data = {imageType: 12382};
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -236,8 +253,6 @@ describe('OTA update', () => {
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
 
-        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
-        extension.lastChecked = {};
         logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
         await zigbeeHerdsman.events.message(payload);
@@ -250,6 +265,28 @@ describe('OTA update', () => {
         );
     });
 
+    it('Should check for update when device requests it and it is not available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Should not check for update when device requests it and disable_automatic_update_check is set to true', async () => {
         settings.set(['ota', 'disable_automatic_update_check'], true);
         const device = zigbeeHerdsman.devices.bulb;

test/publish.test.js

@@ -6,7 +6,7 @@ const stringify = require('json-stable-stringify-without-jsonify');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
@@ -27,6 +27,7 @@ describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
         data.writeEmptyState();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
@@ -34,8 +35,6 @@ describe('Publish', () => {
     });
 
     beforeEach(async () => {
-        jest.useRealTimers();
-        await flushPromises();
         data.writeDefaultConfiguration();
         controller.state.state = {};
         settings.reRead();
@@ -54,6 +53,11 @@ describe('Publish', () => {
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
+    afterAll(async () => {
+        jest.runOnlyPendingTimers();
+        jest.useRealTimers();
+    });
+
     it('Should publish messages to zigbee devices', async () => {
         const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
@@ -139,7 +143,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 222, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 222, color: {x: 0.360786471097048, y: 0.363543544699483}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 222});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -152,7 +156,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 500, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 500, color: {x: 0.526676280311873, y: 0.41329727450763}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'color_temp', color_temp: 500});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -176,10 +180,10 @@ describe('Publish', () => {
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_right: 'OFF'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/wall_switch_double');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state_right: 'OFF'});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should publish messages to zigbee devices to non default-ep with state_[EP]', async () => {
@@ -219,7 +223,7 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color_temp: 62, color: {x: 100, y: 50}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -231,13 +235,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, color_temp: 62, state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON'}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy and brightness', async () => {
@@ -248,13 +251,12 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', color_temp: 62, brightness: 20});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'ON', brightness: 20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state on', async () => {
@@ -265,13 +267,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, state: 'ON', color_temp: 62, brightness: 20, color_mode: 'xy'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color: {x: 100, y: 50}, state: 'ON', brightness: 20, color_mode: 'xy'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Should publish messages to zigbee devices with color xy, brightness and state off', async () => {
@@ -282,40 +284,40 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
         expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, color_temp: 62});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color_mode: 'xy', color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({color_mode: 'xy', color: {x: 100, y: 50}, state: 'OFF'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with color rgb object', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}, color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y:  0.6585}, color_mode: 'xy'});
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with color rgb string', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y:  0.6584}, color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y:  0.6585}, color_mode: 'xy'});
     });
 
-    it('Should publish messages to zigbee devices with color rgb', async () => {
+    it('Should publish messages to zigbee devices with brightness', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON', brightness: '50'}));
@@ -387,7 +389,7 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 24248, colory: 18350, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.37, y: 0.28}, color_temp: 249, color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.37, y: 0.28}, color_mode: 'xy'});
     });
 
     it('Should publish messages to groups color temperature', async () => {
@@ -398,17 +400,18 @@ describe('Publish', () => {
         expect(group.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100, color_mode: 'color_temp'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 100, color_mode: 'color_temp'});
     });
 
     it('Should create and publish to group which is in configuration.yaml but not in zigbee-herdsman', async () => {
-        delete zigbeeHerdsman.groups.group_2;
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(8);
-        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'ON'}));
+        settings.addGroup('group_12312', 12312);
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(10);
+        await MQTT.events.message('zigbee2mqtt/group_12312/set', stringify({state: 'ON'}));
         await flushPromises();
-        expect(Object.values(zigbeeHerdsman.groups).length).toBe(9);
-        expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.groups.group_2.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        expect(Object.values(zigbeeHerdsman.groups).length).toBe(11);
+        expect(zigbeeHerdsman.groups.group_12312.command).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.groups.group_12312.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        delete zigbeeHerdsman.groups.group_12312;
     });
 
     it('Shouldnt publish new state when optimistic = false', async () => {
@@ -422,6 +425,25 @@ describe('Publish', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Shouldnt publish new brightness state when filtered_optimistic is used', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        settings.set(['devices', device.ieeeAddr, 'filtered_optimistic'], ["brightness"]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {"level": 200, "transtime": 0}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+    });
+
+    it('Shouldnt publish new state when optimistic = false for group', async () => {
+        settings.set(['groups', '2', 'optimistic'], false);
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({brightness: '200'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
     it('Should handle non-valid topics', async () => {
         await MQTT.events.message('zigbee2mqtt1/bulb_color/set', stringify({state: 'ON'}));
         await flushPromises();
@@ -474,7 +496,17 @@ describe('Publish', () => {
         expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
-    it('Should log error when device has no such endpoint', async () => {
+    it('Should log error when device has no such endpoint (via topic)', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        logger.error.mockClear();
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/center/get', stringify({state: ''}));
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'wall_switch_double' has no endpoint 'center'`);
+        expect(endpoint2.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should log error when device has no such endpoint (via property)', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint2 = device.getEndpoint(2);
         const endpoint3 = device.getEndpoint(3);
@@ -551,10 +583,10 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17721, colory: 43148, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 157, color: {x: 0.2704, y: 0.6584}, color_mode: "xy"});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.2717, y: 0.6585}, color_mode: "xy"});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -665,20 +697,19 @@ describe('Publish', () => {
     });
 
     it('Should read after write when enabled', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'retrieve_state'], true);
         const endpoint = device.getEndpoint(1);
         const payload = {'state': 'ON', 'color': {'x': 0.701, 'y': 0.299}};
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify(payload));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColor", {"colorx": 45940, "colory": 19595, "transtime": 0}, {}]);
         expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read.mock.calls[0]).toEqual(["genOnOff", ["onOff"]]);
-        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["currentX", "currentY"]]);
+        expect(endpoint.read.mock.calls[1]).toEqual(["lightingColorCtrl", ["colorMode", "currentX", "currentY", "colorTemperature"]]);
     });
 
     it('Should also use on/off cluster when controlling group with switch', async () => {
@@ -699,7 +730,7 @@ describe('Publish', () => {
 
         MQTT.publish.mockClear();
         group.command.mockClear();
-        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF', brightness: 100}));
+        await MQTT.events.message('zigbee2mqtt/switch_group/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
@@ -910,7 +941,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891, "saturation": 199, "transtime": 0,}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44877, "saturation": 200, "transtime": 0,}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}, "color_mode": "hs"});
@@ -1074,7 +1105,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100, color_mode: 'color_temp'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', color_temp: 100, color_mode: 'color_temp'});
     });
 
     it('Home Assistant: should set state when color temperature is also set and device is off', async () => {
@@ -1089,11 +1120,13 @@ describe('Publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
         expect(endpoint.command.mock.calls[1]).toEqual(["lightingColorCtrl", "moveToColorTemp", {colortemp: 100, transtime: 0}, {}]);
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', color: {x: 0.280632719756407, y: 0.288286029784579}, color_temp: 100, color_mode: 'color_temp'});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON', color_temp: 100, color_mode: 'color_temp'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
     });
 
     it('Home Assistant: should not set state when color is also set', async () => {
@@ -1109,7 +1142,7 @@ describe('Publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "moveToColor", {colorx: 26869, colory: 16384, transtime: 0}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.41, y: 0.25}, color_temp: 150, state: 'ON', color_mode: 'xy'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 0.41, y: 0.25}, state: 'ON', color_mode: 'xy'});
     });
 
     it('Should publish correct state on toggle command to zigbee bulb', async () => {
@@ -1195,7 +1228,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 22, "warningduration": 100}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 22, "warningduration": 100, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E emergency warning', async () => {
@@ -1204,7 +1237,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 51, "warningduration": 10}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 51, "warningduration": 10, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E emergency without level', async () => {
@@ -1213,7 +1246,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 49, "warningduration": 10}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 49, "warningduration": 10, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('HS2WD-E wrong payload (should use defaults)', async () => {
@@ -1222,7 +1255,7 @@ describe('Publish', () => {
         await MQTT.events.message('zigbee2mqtt/siren/set', stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 53, "warningduration": 10}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("ssIasWd", "startWarning", {"startwarninginfo": 53, "warningduration": 10, "strobedutycycle": 0, "strobelevel": 1}, {disableDefaultResponse: true});
     });
 
     it('Shouldnt do anythign when device is not supported', async () => {
@@ -1259,19 +1292,17 @@ describe('Publish', () => {
 
     it('Should publish separate genOnOff to GL-S-007ZS when setting state and brightness as bulb doesnt turn on with moveToLevelWithOnOff', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/2757
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices['GL-S-007ZS'];
         const endpoint = device.getEndpoint(1);
         await MQTT.events.message('zigbee2mqtt/GL-S-007ZS/set', stringify({state: 'ON', brightness: 20}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
         expect(endpoint.command.mock.calls[0]).toEqual([ 'genOnOff', 'on', {}, {} ]);
         expect(endpoint.command.mock.calls[1]).toEqual([ 'genLevelCtrl', 'moveToLevelWithOnOff', { level: 20, transtime: 0 }, {} ]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/GL-S-007ZS', stringify({"state":"ON","brightness":20}), { qos: 0, retain: false }, expect.any(Function)]);
-        jest.useRealTimers();
     });
 
     it('Should log as error when setting property with no defined converter', async () => {
@@ -1381,30 +1412,70 @@ describe('Publish', () => {
         await flushPromises();
         expect(group.command).toHaveBeenCalledTimes(1);
         expect(group.command).toHaveBeenCalledWith('genScenes', 'recall', { groupid: 15071, sceneid: 1 }, {});
-        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledTimes(8);
         expect(MQTT.publish).toHaveBeenNthCalledWith(1,
             'zigbee2mqtt/group_tradfri_remote',
-            stringify({"brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON","color_mode": "color_temp"}),
+            stringify({"brightness":50,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(2,
             'zigbee2mqtt/bulb_color_2',
-            stringify({"color_mode": "color_temp", "brightness":50,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON"}),
+            stringify({"color_mode": "color_temp", "brightness":50,"color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(3,
             'zigbee2mqtt/group_tradfri_remote',
-            stringify({"brightness":100,"color":{"x":0.408707336668894,"y":0.39239142575868},"color_temp":290,"state":"ON","color_mode": "color_temp"}),
+            stringify({"brightness":100,"color_temp":290,"state":"ON","color_mode": "color_temp"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(4,
             'zigbee2mqtt/bulb_2',
-            stringify({"brightness":100,"state":"ON"}),
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenNthCalledWith(5,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":50,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(6,
             'zigbee2mqtt/group_with_tradfri',
-            stringify({"brightness":100,"state":"ON"}),
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(7,
+            'zigbee2mqtt/switch_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(8,
+            'zigbee2mqtt/ha_discovery_group',
+            stringify({"brightness":100,"color_mode":"color_temp","color_temp":290,"state":"ON"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should sync colors', async () => {
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color_temp: 100}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color: {x: 0.1, y: 0.5}}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({color_temp: 300}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color_mode":"color_temp","color_temp":100}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"x":0.1,"y":0.5},"color_mode":"xy","color_temp":79}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(3,
+            'zigbee2mqtt/bulb_color',
+            stringify({"color":{"x":0.4152,"y":0.3954},"color_mode":"color_temp","color_temp":300}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });

test/receive.test.js

@@ -5,24 +5,31 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Receive', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings.reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
+        jest.useFakeTimers();
         controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should handle a zigbee message', async () => {
         const device = zigbeeHerdsman.devices.WXKG11LM;
         device.linkquality = 10;
@@ -64,7 +71,7 @@ describe('Receive', () => {
         const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
@@ -100,7 +107,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce messages', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         const data1 = {measuredValue: 8}
@@ -115,7 +121,7 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
@@ -124,7 +130,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce and retain messages when set via device_options', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['device_options', 'debounce'], 0.1);
         settings.set(['device_options', 'retain'], true);
@@ -141,7 +146,7 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
@@ -150,7 +155,6 @@ describe('Receive', () => {
     });
 
     it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         settings.set(['devices', device.ieeeAddr, 'debounce_ignore'], ['temperature']);
@@ -166,7 +170,7 @@ describe('Receive', () => {
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2});
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03});
@@ -174,13 +178,12 @@ describe('Receive', () => {
 
     it('Shouldnt republish old state', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/3572
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON'}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
@@ -327,6 +330,20 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    // TODO
+    // it('Should publish last_seen for unhandled messages', async () => {
+    //     const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
+    //     settings.set(['advanced', 'last_seen'], 'epoch');
+    //     const data = {onOff: 1};
+    //     const payload = {data, cluster: 'genRssiLocation', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+    //     await zigbeeHerdsman.events.message(payload);
+    //     await flushPromises();
+    //     expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    //     expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+    //     expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('number')
+    //     expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    // });
+
     it('Should publish last_seen epoch', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'epoch');

test/settings.test.js

@@ -1,8 +1,8 @@
 require('./stub/logger');
 require('./stub/data');
-const data = require('../lib/util/data.js');
-const utils = require('../lib/util/utils.js');
-const settings = require('../lib/util/settings.js');
+const data = require('../lib/util/data');
+const utils = require('../lib/util/utils');
+const settings = require('../lib/util/settings.ts');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
@@ -20,12 +20,12 @@ const minimalConfig = {
 
 describe('Settings', () => {
     const write = (file, json, reread=true) => {
-        fs.writeFileSync(file, yaml.safeDump(json))
+        fs.writeFileSync(file, yaml.dump(json))
         if (reread) {
             settings.reRead();
         }
     };
-    const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const read = (file) => yaml.load(fs.readFileSync(file, 'utf8'));
     const remove = (file) => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
@@ -47,7 +47,7 @@ describe('Settings', () => {
     it('Should return default settings', () => {
         write(configurationFile, {});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expect(s).toStrictEqual(expected);
@@ -56,7 +56,7 @@ describe('Settings', () => {
     it('Should return settings', () => {
         write(configurationFile, {permit_join: true});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.permit_join = true;
@@ -73,7 +73,7 @@ describe('Settings', () => {
 
         write(configurationFile, {});
         const s = settings.get();
-        const expected = objectAssignDeep.noMutate({}, settings._getDefaults());
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.serial.disable_led = true;
@@ -163,7 +163,7 @@ describe('Settings', () => {
         expect(settings.get().mqtt).toStrictEqual(expected);
         expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
 
-        settings._write();
+        settings.testing.write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
         expect(read(secretFile)).toStrictEqual(contentSecret);
 
@@ -352,7 +352,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -378,7 +377,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -420,7 +418,6 @@ describe('Settings', () => {
 
         const group = settings.getGroup('1');
         const expectedGroup = {
-            optimistic: true,
             ID: 1,
             friendlyName: '123',
             friendly_name: '123',
@@ -496,7 +493,7 @@ describe('Settings', () => {
         settings.addGroup('test123', 123);
         expect(() => {
             settings.addGroup('test_id_123', 123);
-        }).toThrow(new Error("group id '123' is already in use"));
+        }).toThrow(new Error("Group ID '123' is already in use"));
         const expected = {
             '123': {
                 friendly_name: 'test123',
@@ -679,7 +676,7 @@ describe('Settings', () => {
 
         settings.reRead();
 
-        const error = `advanced should be object`;
+        const error = `advanced must be object`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
@@ -705,13 +702,13 @@ describe('Settings', () => {
              \t wrong
         `)
 
-        settings._clear();
+        settings.testing.clear();
         const error = `Your YAML file: '${configurationFile}' is invalid (use https://jsonformatter.org/yaml-validator to find and fix the issue)`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should throw error when yaml file does not exist', () => {
-        settings._clear();
+        settings.testing.clear();
         const error = `ENOENT: no such file or directory, open '${configurationFile}'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
@@ -797,7 +794,7 @@ describe('Settings', () => {
 
         settings.reRead();
 
-        const error = `Following friendly_name are not allowed: '${utils.getEndpointNames()}'`;
+        const error = `Following friendly_name are not allowed: '${utils.endpointNames}'`;
         expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 

test/stub/data.js

@@ -202,12 +202,16 @@ function writeDefaultConfiguration() {
             '14': {
                 friendly_name: 'switch_group',
                 retain: false,
-                devices: ['power_plug'],
+                devices: ['power_plug', 'bulb_2'],
             },
             '21': {
                 friendly_name: 'gledopto_group',
                 devices: ['GLEDOPTO_2ID/cct'],
-            }
+            },
+            '9': {
+                friendly_name: 'ha_discovery_group',
+                devices: ['bulb_color_2', 'bulb_2', 'wall_switch_double/right']
+            },
         },
         external_converters: [],
     };
@@ -229,20 +233,24 @@ function stateExists() {
     return fs.existsSync(stateFile);
 }
 
-function writeDefaultState() {
-    const state = {
-        "0x000b57fffec6a5b2": {
-            "state": "ON",
-            "brightness": 50,
-            "color_temp": 370,
-            "linkquality": 99,
-        },
-        "0x0017880104e45517": {
-            "brightness": 255
-        },
-    }
+const defaultState = {
+    "0x000b57fffec6a5b2": {
+        "state": "ON",
+        "brightness": 50,
+        "color_temp": 370,
+        "linkquality": 99,
+    },
+    "0x0017880104e45517": {
+        "brightness": 255
+    },
+}
 
-    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(state));
+function getDefaultState() {
+    return defaultState;
+}
+
+function writeDefaultState() {
+    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(defaultState));
 }
 
 jest.mock('../../lib/util/data', () => ({
@@ -261,4 +269,5 @@ module.exports = {
     removeState,
     writeEmptyState,
     stateExists,
+    getDefaultState,
 };

test/stub/zigbeeHerdsman.js

@@ -127,11 +127,12 @@ class Device {
 const returnDevices = [];
 
 const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
-const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020", false, 'Philips', '2019.09', '5.127.1.26581');
+const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020", false, 'Philips', '2019.09', '5.127.1.26581');
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 const ZNCZ02LM = new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45524')], true, "Mains (single phase)", "lumi.plug");
 const GLEDOPTO_2ID = new Device('Router', '0x0017880104e45724', 6540,4151, [new Endpoint(11, [0,3,4,5,6,8,768], [], '0x0017880104e45724', [], {}, [], 49246, 528), new Endpoint(12, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 260, 258), new Endpoint(13, [4096], [4096], '0x0017880104e45724', [], {}, [], 49246, 57694), new Endpoint(15, [0, 3, 4, 5, 6, 8, 768], [], '0x0017880104e45724', [], {}, [], 49246, 256)], true, "Mains (single phase)", 'GL-C-007', false, 'GLEDOPTO');
+const QBKG03LM = new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2');
 
 const groups = {
     'group_1': new Group(1, []),
@@ -139,9 +140,10 @@ const groups = {
     'group/with/slashes': new Group(99, []),
     'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
     'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]]),
-    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0]]),
+    'group_with_switch': new Group(14, [ZNCZ02LM.endpoints[0], bulb_2.endpoints[0]]),
     'gledopto_group': new Group(21, [GLEDOPTO_2ID.endpoints[3]]),
     'default_bind_group': new Group(901, []),
+    'ha_discovery_group': new Group(9, [bulb_color_2.endpoints[0], bulb_2.endpoints[0], QBKG03LM.endpoints[1]]),
 }
 
 const devices = {
@@ -162,7 +164,7 @@ const devices = {
     'ZNCZ02LM': ZNCZ02LM,
     'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
     'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
-    'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
+    'QBKG03LM':QBKG03LM,
     'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO_2ID': GLEDOPTO_2ID,
@@ -194,6 +196,8 @@ const devices = {
     'unknown': new Device('Router', '0x0017980134e45545', 6540,4151, [], true, "Mains (single phase)"),
     'temperature_sensor': new Device('EndDevice', '0x0017880104e45561', 6544,4151, [new Endpoint(1, [0,3,4,1026], [])], true, "Battery", "temperature.sensor"),
     'heating_actuator': new Device('Router', '0x0017880104e45562', 6545,4151, [new Endpoint(1, [0,3,4,513], [1026])], true, "Mains (single phase)", "heating.actuator"),
+    'bj_scene_switch': new Device('EndDevice', '0xd85def11a1002caa', 50117, 4398, [new Endpoint(10, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa', [{target: bulb_color_2.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 6, name: 'genOnOff'}}, {target: bulb_color_2.endpoints[0], cluster: {ID: 768, name: 'lightingColorCtrl'}},]), new Endpoint(11, [0,4096], [3,4,5,6,8,25,768,4096], '0xd85def11a1002caa')], true, 'Battery', 'RB01', false, 'Busch-Jaeger', '20161222', '1.2.0'),
+
 }
 
 const mock = {

test/utils.test.js

@@ -1,4 +1,4 @@
-const utils = require('../lib/util/utils.js');
+const utils = require('../lib/util/utils');
 const version = require('../package.json').version;
 const versionHerdsman = require('../node_modules/zigbee-herdsman/package.json').version;
 
@@ -20,10 +20,6 @@ describe('Utils', () => {
         });
     });
 
-    it('Convert milliseconds to seconds', () => {
-        expect(utils.millisecondsToSeconds(2000)).toBe(2);
-    })
-
     it('Object has properties', () => {
         expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b'])).toBeTruthy();
         expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b', 'd'])).toBeFalsy();
@@ -36,10 +32,10 @@ describe('Utils', () => {
         }));
 
         mockReturnValue = [false, {shortHash: '123'}]
-        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "123", "version": version});
+        expect(await utils.getZigbee2MQTTVersion()).toStrictEqual({"commitHash": "123", "version": version});
 
         mockReturnValue = [true, null]
-        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
+        expect(await utils.getZigbee2MQTTVersion()).toStrictEqual({"commitHash": expect.any(String), "version": version});
     })
 
     it('Check dependency version', async () => {
@@ -57,11 +53,6 @@ describe('Utils', () => {
         Date.prototype.getTimezoneOffset = getTimezoneOffset;
     })
 
-    it('Throw exception when formating with invalid date', async () => {
-        var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
-        expect(() => utils.formatDate(date, 'invalid', 1)).toThrowError("Unsupported type 'invalid'")
-    })
-
     it('Get key', async () => {
         expect(utils.getKey({'1': '1'}, '1', 2, null)).toBe('1');
         expect(utils.getKey({'1': '1'}, '2', 2, null)).toBe(2);

tsconfig.json

@@ -0,0 +1,24 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "esModuleInterop": true,
+        "target": "es2018",
+        "lib": ["es2018"],
+        "noImplicitAny": true,
+        "noImplicitThis": true,
+        "moduleResolution": "node",
+        "declaration": true,
+        "declarationMap": true,
+        "outDir": "dist",
+        "baseUrl": ".",
+        "allowJs": true,
+        "rootDir": "lib",
+        "inlineSourceMap": true,
+        "resolveJsonModule": true,
+        "experimentalDecorators": true
+    },
+    "include": [
+        "lib/**/*",
+        "lib/util/settings.schema.json"
+    ],
+}

update.sh

@@ -7,8 +7,12 @@ if [ -d data-backup ]; then
    exit 1
 fi
 
-echo "Stopping Zigbee2MQTT..."
-sudo systemctl stop zigbee2mqtt
+if which systemctl 2> /dev/null > /dev/null; then
+       echo "Stopping Zigbee2MQTT..."
+       sudo systemctl stop zigbee2mqtt
+else
+       echo "Skipped stopping Zigbee2MQTT, no systemctl found"
+fi
 
 echo "Creating backup of configuration..."
 cp -R data data-backup
@@ -24,7 +28,11 @@ echo "Restore configuration..."
 cp -R data-backup/* data
 rm -rf data-backup
 
-echo "Starting Zigbee2MQTT..."
-sudo systemctl start zigbee2mqtt
+if which systemctl 2> /dev/null > /dev/null; then
+       echo "Starting Zigbee2MQTT..."
+       sudo systemctl start zigbee2mqtt
+else
+       echo "Skipped starting Zigbee2MQTT, no systemctl found"
+fi
 
 echo "Done!"